fastbulk - Commit to /usr/src/test/fastbulk
[dragonfly.git] / test / fastbulk / fastbulk.c
1 /*
2  * Copyright (c) 2010 The DragonFly Project.  All rights reserved.
3  *
4  * This code is derived from software contributed to The DragonFly Project
5  * by Matthew Dillon <dillon@backplane.com>
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in
15  *    the documentation and/or other materials provided with the
16  *    distribution.
17  * 3. Neither the name of The DragonFly Project nor the names of its
18  *    contributors may be used to endorse or promote products derived
19  *    from this software without specific, prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  */
34 /*
35  * fastbulk.c
36  *
37  * fastbulk <pkgsrcdir>
38  *
39  * This program iterates all pkgsrc directories, runs 'bmake show-depends-dirs'
40  * recursively, and builds a dependency tree on the fly.
41  *
42  * As the dependency tree is being built, terminal dependencies are built
43  * and packaged on the fly (well, really it runs /tmp/track/dobuild inside
44  * the chroot).  As these builds complete additional dependencies may be
45  * satisfied and be added to the build order.  Ultimately the entire tree
46  * is built.
47  *
48  * Only one attempt is made to build any given package, no matter how many
49  * other packages depend on it.
50  */
51 #include <stdio.h>
52 #include <stdlib.h>
53 #include <unistd.h>
54 #include <string.h>
55 #include <dirent.h>
56 #include <assert.h>
57 #include <sys/file.h>
58 #include <sys/stat.h>
59 #include <sys/wait.h>
60
61 struct item;
62
63 struct depn {
64         struct depn *dnext;
65         struct item *item;
66 };
67
68 struct item {
69         enum { XWAITING, XDEPFAIL, XBUILD, XRUN, XDONE } status;
70         struct item *hnext;     /* ItemHash next */
71         struct item *bnext;     /* BuildList/RunList next */
72         struct depn *dbase;     /* packages depending on us */
73         char *rpath;            /* e.g. "shells/tcsh" */
74         char *lpath;            /* e.g. "shells.tcsh" (log path) */
75         int dcount;             /* build completion for our dependencies */
76         int xcode;              /* exit code from build */
77         pid_t pid;              /* running build */
78 };
79
80 #define ITHSIZE 1024
81 #define ITHMASK (ITHSIZE - 1)
82
83 static struct item *ItemHash[ITHSIZE];
84 static struct item *BuildList;
85 static struct item **BuildListP = &BuildList;
86 static struct item *RunList;
87
88 static void ordered_scan(const char *bpath, const char *path, size_t blen);
89 static struct item *ordered_depends(const char *bpath, const char *npath);
90 static struct item *lookupItem(const char *npath);
91 static struct item *addItem(const char *npath);
92 static void addDepn(struct item *item, struct item *xitem);
93
94 static void addBuild(struct item *item);
95 static void runBuilds(const char *bpath);
96 static struct item *waitRunning(int flags);
97 static void processCompletion(struct item *item);
98
99 static const char *neednl(void);
100 static void usage(void);
101
102 int NParallel = 1;
103 int VerboseOpt;
104 int NRunning;
105 int NeedNL;
106
107 int
108 main(int ac, char **av)
109 {
110         char *bpath;
111         size_t blen;
112         int ch;
113
114         while ((ch = getopt(ac, av, "j:v")) != -1) {
115                 switch(ch) {
116                 case 'j':
117                         NParallel = strtol(optarg, NULL, 0);
118                         break;
119                 case 'v':
120                         VerboseOpt = 1;
121                         break;
122                 default:
123                         usage();
124                         /* NOT REACHED */
125                 }
126         }
127         ac -= optind;
128         av += optind;
129
130         if (ac != 1) {
131                 fprintf(stderr, "requires base directory as first argument\n");
132                 exit(1);
133         }
134
135         /*
136          * Base dir
137          */
138         bpath = strdup(av[0]);
139         blen = strlen(bpath);
140         while (blen && bpath[blen-1] == '/')
141                 --blen;
142         bpath[blen] = 0;
143
144         /*
145          * Do recursive directory scan
146          */
147         ordered_scan(bpath, bpath, strlen(bpath));
148
149         /*
150          * Wait for all current builds to finish running, keep the pipeline
151          * full until both the BuildList and RunList have been exhausted.
152          */
153         runBuilds(bpath);
154         while (waitRunning(0) != NULL)
155                 runBuilds(bpath);
156         return(0);
157 }
158
159 /*
160  * Recursively scan the requested directory tree looking for pkgsrc
161  * stuff to build.
162  */
163 static void
164 ordered_scan(const char *bpath, const char *path, size_t blen)
165 {
166         DIR *dir;
167         struct dirent *den;
168         char *npath;
169         char *xpath;
170         struct stat st;
171
172         if ((dir = opendir(path)) != NULL) {
173                 while ((den = readdir(dir)) != NULL) {
174                         if (den->d_name[0] == '.')
175                                 continue;
176                         asprintf(&npath, "%s/%s", path, den->d_name);
177                         asprintf(&xpath, "%s/DESCR", npath);
178
179                         if (lookupItem(npath + blen + 1) == NULL &&
180                             stat(npath, &st) == 0 && S_ISDIR(st.st_mode)) {
181                                 if (stat(xpath, &st) == 0) {
182                                         ordered_depends(bpath,
183                                                         npath + blen + 1);
184                                 } else {
185                                         ordered_scan(bpath, npath, blen);
186                                 }
187                         }
188                         free(npath);
189                         free(xpath);
190                 }
191                 closedir(dir);
192         }
193 }
194
195 /*
196  * Recursively execute 'bmake show-depends-dirs' to calculate all required
197  * dependencies.
198  */
199 static struct item *
200 ordered_depends(const char *bpath, const char *npath)
201 {
202         struct item *item;
203         struct item *xitem;
204         char buf[1024];
205         FILE *fp;
206         char *cmd;
207         int len;
208
209         item = addItem(npath);
210
211         /*
212          * Retrieve and process dependencies recursively.  Note that
213          * addDepn() can modify item's status.
214          *
215          * Guard the recursion by bumping dcount to prevent the item
216          * from being processed for completion while we are still adding
217          * its dependencies.  This would normally not occur but it can
218          * if pkgsrc has a broken dependency loop.
219          */
220         ++item->dcount;
221         asprintf(&cmd, "cd %s/%s; bmake show-depends-dirs", bpath, npath);
222         fp = popen(cmd, "r");
223         while (fgets(buf, sizeof(buf), fp) != NULL) {
224                 len = strlen(buf);
225                 if (len && buf[len-1] == '\n')
226                         buf[--len] = 0;
227                 xitem = lookupItem(buf);
228                 if (xitem == NULL)
229                         xitem = ordered_depends(bpath, buf);
230                 addDepn(item, xitem);
231         }
232         pclose(fp);
233         free(cmd);
234         --item->dcount;
235
236         /*
237          * If the item has no dependencies left either add it to the
238          * build list or do completion processing (i.e. if some of the
239          * dependencies failed).
240          */
241         if (item->dcount == 0) {
242                 switch(item->status) {
243                 case XWAITING:
244                         addBuild(item);
245                         break;
246                 case XDEPFAIL:
247                         processCompletion(item);
248                         break;
249                 default:
250                         assert(0);
251                         /* NOT REACHED */
252                         break;
253                 }
254         } else {
255                 if (VerboseOpt)
256                         printf("Deferred   %s\n", item->rpath);
257         }
258         runBuilds(bpath);
259         return (item);
260 }
261
262 /*
263  * Item hashing and dependency helper routines, called during the
264  * directory scan.
265  */
266 static int
267 itemhash(const char *npath)
268 {
269         int hv = 0xA1B5F342;
270         int i;
271
272         for (i = 0; npath[i]; ++i)
273                 hv = (hv << 5) ^ (hv >> 23) ^ npath[i];
274         return(hv & ITHMASK);
275 }
276
277 static struct item *
278 lookupItem(const char *npath)
279 {
280         struct item *item;
281
282         for (item = ItemHash[itemhash(npath)]; item; item = item->hnext) {
283                 if (strcmp(npath, item->rpath) == 0)
284                         return(item);
285         }
286         return(NULL);
287 }
288
289 static struct item *
290 addItem(const char *npath)
291 {
292         struct item **itemp;
293         struct item *item = calloc(sizeof(*item), 1);
294         int i;
295
296         itemp = &ItemHash[itemhash(npath)];
297         item->status = XWAITING;
298         item->hnext = *itemp;
299         item->rpath = strdup(npath);
300         item->lpath = strdup(npath);
301         *itemp = item;
302         for (i = 0; item->lpath[i]; ++i) {
303                 if (item->lpath[i] == '/')
304                         item->lpath[i] = '.';
305         }
306
307         return(item);
308 }
309
310 /*
311  * Add a reverse dependency from the deepest point (xitem) to the
312  * packages that depend on xitem (item in this case).
313  *
314  * Caller will check dcount after it is through adding dependencies.
315  */
316 static void
317 addDepn(struct item *item, struct item *xitem)
318 {
319         struct depn *depn = calloc(sizeof(*depn), 1);
320         char *logpath3;
321         FILE *fp;
322
323         depn->item = item;
324         depn->dnext = xitem->dbase;
325         xitem->dbase = depn;
326         if (xitem->status == XDONE) {
327                 if (xitem->xcode) {
328                         assert(item->status == XWAITING ||
329                                item->status == XDEPFAIL);
330                         item->xcode = xitem->xcode;
331                         item->status = XDEPFAIL;
332                         asprintf(&logpath3,
333                                  "/tmp/logs/bad/%s", item->lpath);
334                         fp = fopen(logpath3, "a");
335                         fprintf(fp, "Dependency failed: %s\n",
336                                 xitem->rpath);
337                         fclose(fp);
338                         free(logpath3);
339                 }
340         } else {
341                 ++item->dcount;
342         }
343 }
344
345 /*
346  * Add the item to the build request list.  This routine is called
347  * after all build dependencies have been satisfied for the item.
348  * runBuilds() will pick items off of BuildList to keep the parallel
349  * build pipeline full.
350  */
351 static void
352 addBuild(struct item *item)
353 {
354         printf("%sBuildOrder %s\n", neednl(), item->rpath);
355         *BuildListP = item;
356         BuildListP = &item->bnext;
357         item->status = XBUILD;
358 }
359
360 /*
361  * Start new builds from the build list and handle build completions,
362  * which can potentialy add new items to the build list.
363  *
364  * This routine will maintain up to NParallel builds.  A new build is
365  * only started once its dependencies have completed successfully so
366  * when the bulk build starts it typically takes a little while before
367  * fastbulk can keep the parallel pipeline full.
368  */
369 static void
370 runBuilds(const char *bpath)
371 {
372         struct item *item;
373         char *logpath;
374         FILE *fp;
375         int fd;
376
377         /*
378          * Try to maintain up to NParallel builds
379          */
380         while (NRunning < NParallel && BuildList) {
381                 item = BuildList;
382                 if ((BuildList = item->bnext) == NULL)
383                         BuildListP = &BuildList;
384                 printf("%sBuildStart %s\n", neednl(), item->rpath);
385
386                 /*
387                  * When [re]running a build remove any bad log from prior
388                  * attempts.
389                  */
390                 asprintf(&logpath, "/tmp/logs/bad/%s", item->lpath);
391                 remove(logpath);
392                 free(logpath);
393
394                 asprintf(&logpath, "/tmp/logs/run/%s", item->lpath);
395
396                 item->status = XRUN;
397
398                 item->pid = fork();
399                 if (item->pid == 0) {
400                         /*
401                          * Child process - setup the log file and exec
402                          */
403                         if (chdir(bpath) < 0)
404                                 _exit(99);
405                         if (chdir(item->rpath) < 0)
406                                 _exit(99);
407
408                         fd = open(logpath, O_RDWR|O_CREAT|O_TRUNC, 0666);
409                         if (fd != 1)
410                                 dup2(fd, 1);
411                         if (fd != 2)
412                                 dup2(fd, 2);
413                         if (fd != 1 && fd != 2)
414                                 close(fd);
415                         fd = open("/dev/null", O_RDWR);
416                         if (fd != 0) {
417                                 dup2(fd, 0);
418                                 close(fd);
419                         }
420
421                         /*
422                          * we tack a 'clean' on to the repackage to clean
423                          * the work directory on success.  If a failure
424                          * occurs we leave the work directory intact.
425                          *
426                          * leaving work directories around when doing a
427                          * bulk build can fill up the filesystem very fast.
428                          */
429                         execl("/tmp/track/dobuild", "dobuild",
430                                 item->lpath, NULL);
431                         _exit(99);
432                 } else if (item->pid < 0) {
433                         /*
434                          * Parent fork() failed, log the problem and
435                          * do completion processing.
436                          */
437                         item->xcode = -98;
438                         fp = fopen(logpath, "a");
439                         fprintf(fp, "fastbulk: Unable to fork/exec bmake\n");
440                         fclose(fp);
441                         processCompletion(item);
442                 } else {
443                         /*
444                          * Parent is now tracking the running child,
445                          * add the item to the RunList.
446                          */
447                         item->bnext = RunList;
448                         RunList = item;
449                         ++NRunning;
450                 }
451                 free(logpath);
452         }
453
454         /*
455          * Process any completed builds (non-blocking)
456          */
457         while (waitRunning(WNOHANG) != NULL)
458                 ;
459 }
460
461 /*
462  * Wait for a running build to finish and process its completion.
463  * Return the build or NULL if no builds are pending.
464  *
465  * The caller should call runBuilds() in the loop to keep the build
466  * pipeline full until there is nothing left in the build list.
467  */
468 static struct item *
469 waitRunning(int flags)
470 {
471         struct item *item;
472         struct item **itemp;
473         pid_t pid;
474         int status;
475
476         if (RunList == NULL)
477                 return(NULL);
478         while ((pid = wait3(&status, flags, NULL)) < 0 && flags == 0)
479                 ;
480
481         /*
482          * NOTE! The pid may be associated with one of our popen()'s
483          *       so just ignore it if we cannot find it.
484          */
485         if (pid > 0) {
486                 status = WEXITSTATUS(status);
487                 itemp = &RunList;
488                 while ((item = *itemp) != NULL) {
489                         if (item->pid == pid)
490                                 break;
491                         itemp = &item->bnext;
492                 }
493                 if (item) {
494                         *itemp = item->bnext;
495                         item->bnext = NULL;
496                         item->xcode = status;
497                         --NRunning;
498                         processCompletion(item);
499                 }
500         } else {
501                 item = NULL;
502         }
503         return (item);
504 }
505
506 /*
507  * Process the build completion for an item.
508  */
509 static void
510 processCompletion(struct item *item)
511 {
512         struct depn *depn;
513         struct item *xitem;
514         char *logpath1;
515         char *logpath2;
516         char *logpath3;
517         FILE *fp;
518
519         /*
520          * If XRUN we have to move the logfile to the correct directory.
521          * (If XDEPFAIL the logfile is already in the correct directory).
522          */
523         if (item->status == XRUN) {
524                 asprintf(&logpath1, "/tmp/logs/run/%s", item->lpath);
525                 asprintf(&logpath2, "/tmp/logs/%s/%s",
526                          (item->xcode ? "bad" : "good"), item->lpath);
527                 rename(logpath1, logpath2);
528                 free(logpath1);
529                 free(logpath2);
530         }
531
532         printf("%sFinish %-3d %s\n", neednl(), item->xcode, item->rpath);
533         assert(item->status == XRUN || item->status == XDEPFAIL);
534         item->status = XDONE;
535         for (depn = item->dbase; depn; depn = depn->dnext) {
536                 xitem = depn->item;
537                 assert(xitem->dcount > 0);
538                 --xitem->dcount;
539                 if (xitem->status == XWAITING || xitem->status == XDEPFAIL) {
540                         /*
541                          * If our build went well add items dependent
542                          * on us to the build, otherwise fail the items
543                          * dependent on us.
544                          */
545                         if (item->xcode) {
546                                 xitem->xcode = item->xcode;
547                                 xitem->status = XDEPFAIL;
548                                 asprintf(&logpath3,
549                                          "/tmp/logs/bad/%s", xitem->lpath);
550                                 fp = fopen(logpath3, "a");
551                                 fprintf(fp, "Dependency failed: %s\n",
552                                         item->rpath);
553                                 fclose(fp);
554                                 free(logpath3);
555                         }
556                         if (xitem->dcount == 0) {
557                                 if (xitem->status == XWAITING)
558                                         addBuild(xitem);
559                                 else
560                                         processCompletion(xitem);
561                         }
562                 } else if (xitem->status == XDONE && xitem->xcode) {
563                         /*
564                          * The package depending on us has already run
565                          * (this case should not occur).
566                          *
567                          * Add this dependency failure to its log file
568                          * (which has already been renamed).
569                          */
570                         asprintf(&logpath3,
571                                  "/tmp/logs/bad/%s", xitem->lpath);
572                         fp = fopen(logpath3, "a");
573                         fprintf(fp, "Dependency failed: %s\n",
574                                 item->rpath);
575                         fclose(fp);
576                         free(logpath3);
577                 }
578         }
579 }
580
581 static const char *
582 neednl(void)
583 {
584         if (NeedNL) {
585                 NeedNL = 0;
586                 return("\n");
587         } else {
588                 return("");
589         }
590 }
591
592 static void
593 usage(void)
594 {
595         fprintf(stderr, "fastbulk [-j parallel] /usr/pkgsrc\n");
596         exit(1);
597 }