hammer - Fix core dump during remote termination of mirror-stream
[dragonfly.git] / test / fastbulk / fastbulk.c
1 /*
2  * Copyright (c) 2010 The DragonFly Project.  All rights reserved.
3  *
4  * This code is derived from software contributed to The DragonFly Project
5  * by Matthew Dillon <dillon@backplane.com>
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in
15  *    the documentation and/or other materials provided with the
16  *    distribution.
17  * 3. Neither the name of The DragonFly Project nor the names of its
18  *    contributors may be used to endorse or promote products derived
19  *    from this software without specific, prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  */
34 /*
35  * fastbulk.c
36  *
37  * fastbulk <pkgsrcdir>
38  *
39  * This program iterates all pkgsrc directories, runs 'bmake show-depends-dirs'
40  * recursively, and builds a dependency tree on the fly.
41  *
42  * As the dependency tree is being built, terminal dependencies are built
43  * and packaged on the fly (well, really it runs /tmp/track/dobuild inside
44  * the chroot).  As these builds complete additional dependencies may be
45  * satisfied and be added to the build order.  Ultimately the entire tree
46  * is built.
47  *
48  * Only one attempt is made to build any given package, no matter how many
49  * other packages depend on it.
50  */
51 #include <sys/types.h>
52 #include <sys/file.h>
53 #include <sys/stat.h>
54 #include <sys/wait.h>
55 #include <sys/ioctl.h>
56 #include <sys/time.h>
57 #include <sys/resource.h>
58
59 #include <stdio.h>
60 #include <stdlib.h>
61 #include <unistd.h>
62 #include <string.h>
63 #include <dirent.h>
64 #include <assert.h>
65
66 struct item;
67
68 struct depn {
69         struct depn *dnext;
70         struct item *item;
71 };
72
73 struct item {
74         enum { XWAITING, XDEPFAIL, XBUILD, XRUN, XDONE } status;
75         struct item *hnext;     /* ItemHash next */
76         struct item *bnext;     /* BuildList/RunList next */
77         struct depn *dbase;     /* packages depending on us */
78         char *rpath;            /* e.g. "shells/tcsh" */
79         char *lpath;            /* e.g. "shells.tcsh" (log path) */
80         int dcount;             /* build completion for our dependencies */
81         int xcode;              /* exit code from build */
82         pid_t pid;              /* running build */
83 };
84
85 #define ITHSIZE 1024
86 #define ITHMASK (ITHSIZE - 1)
87
88 static struct item *ItemHash[ITHSIZE];
89 static struct item *BuildList;
90 static struct item **BuildListP = &BuildList;
91 static struct item *RunList;
92
93 static void ordered_scan(const char *bpath, const char *path, size_t blen);
94 static struct item *ordered_depends(const char *bpath, const char *npath);
95 static struct item *lookupItem(const char *npath);
96 static struct item *addItem(const char *npath);
97 static void addDepn(struct item *item, struct item *xitem);
98
99 static void addBuild(struct item *item);
100 static void runBuilds(const char *bpath);
101 static struct item *waitRunning(int flags);
102 static void processCompletion(struct item *item);
103
104 static const char *neednl(void);
105 static void usage(void);
106
107 int NParallel = 1;
108 int VerboseOpt;
109 int NRunning;
110 int NeedNL;
111
112 int
113 main(int ac, char **av)
114 {
115         char *bpath;
116         size_t blen;
117         int ch;
118
119         while ((ch = getopt(ac, av, "j:v")) != -1) {
120                 switch(ch) {
121                 case 'j':
122                         NParallel = strtol(optarg, NULL, 0);
123                         break;
124                 case 'v':
125                         VerboseOpt = 1;
126                         break;
127                 default:
128                         usage();
129                         /* NOT REACHED */
130                 }
131         }
132         ac -= optind;
133         av += optind;
134
135         if (ac != 1) {
136                 fprintf(stderr, "requires base directory as first argument\n");
137                 exit(1);
138         }
139
140         /*
141          * Base dir
142          */
143         bpath = strdup(av[0]);
144         blen = strlen(bpath);
145         while (blen && bpath[blen-1] == '/')
146                 --blen;
147         bpath[blen] = 0;
148
149         /*
150          * Do recursive directory scan
151          */
152         ordered_scan(bpath, bpath, strlen(bpath));
153
154         /*
155          * Wait for all current builds to finish running, keep the pipeline
156          * full until both the BuildList and RunList have been exhausted.
157          */
158         runBuilds(bpath);
159         while (waitRunning(0) != NULL)
160                 runBuilds(bpath);
161         return(0);
162 }
163
164 /*
165  * Recursively scan the requested directory tree looking for pkgsrc
166  * stuff to build.
167  */
168 static void
169 ordered_scan(const char *bpath, const char *path, size_t blen)
170 {
171         DIR *dir;
172         struct dirent *den;
173         char *npath;
174         char *xpath;
175         struct stat st;
176
177         if ((dir = opendir(path)) != NULL) {
178                 while ((den = readdir(dir)) != NULL) {
179                         if (den->d_name[0] == '.')
180                                 continue;
181                         asprintf(&npath, "%s/%s", path, den->d_name);
182                         asprintf(&xpath, "%s/DESCR", npath);
183
184                         if (lookupItem(npath + blen + 1) == NULL &&
185                             stat(npath, &st) == 0 && S_ISDIR(st.st_mode)) {
186                                 if (stat(xpath, &st) == 0) {
187                                         ordered_depends(bpath,
188                                                         npath + blen + 1);
189                                 } else {
190                                         ordered_scan(bpath, npath, blen);
191                                 }
192                         }
193                         free(npath);
194                         free(xpath);
195                 }
196                 closedir(dir);
197         }
198 }
199
200 /*
201  * Recursively execute 'bmake show-depends-dirs' to calculate all required
202  * dependencies.
203  */
204 static struct item *
205 ordered_depends(const char *bpath, const char *npath)
206 {
207         struct item *item;
208         struct item *xitem;
209         char buf[1024];
210         FILE *fp;
211         char *cmd;
212         int len;
213
214         item = addItem(npath);
215
216         /*
217          * Retrieve and process dependencies recursively.  Note that
218          * addDepn() can modify item's status.
219          *
220          * Guard the recursion by bumping dcount to prevent the item
221          * from being processed for completion while we are still adding
222          * its dependencies.  This would normally not occur but it can
223          * if pkgsrc has a broken dependency loop.
224          */
225         ++item->dcount;
226         asprintf(&cmd, "cd %s/%s; bmake show-depends-dirs", bpath, npath);
227         fp = popen(cmd, "r");
228         while (fgets(buf, sizeof(buf), fp) != NULL) {
229                 len = strlen(buf);
230                 if (len && buf[len-1] == '\n')
231                         buf[--len] = 0;
232                 xitem = lookupItem(buf);
233                 if (xitem == NULL)
234                         xitem = ordered_depends(bpath, buf);
235                 addDepn(item, xitem);
236         }
237         pclose(fp);
238         free(cmd);
239         --item->dcount;
240
241         /*
242          * If the item has no dependencies left either add it to the
243          * build list or do completion processing (i.e. if some of the
244          * dependencies failed).
245          */
246         if (item->dcount == 0) {
247                 switch(item->status) {
248                 case XWAITING:
249                         addBuild(item);
250                         break;
251                 case XDEPFAIL:
252                         processCompletion(item);
253                         break;
254                 default:
255                         assert(0);
256                         /* NOT REACHED */
257                         break;
258                 }
259         } else {
260                 if (VerboseOpt)
261                         printf("Deferred   %s\n", item->rpath);
262         }
263         runBuilds(bpath);
264         return (item);
265 }
266
267 /*
268  * Item hashing and dependency helper routines, called during the
269  * directory scan.
270  */
271 static int
272 itemhash(const char *npath)
273 {
274         int hv = 0xA1B5F342;
275         int i;
276
277         for (i = 0; npath[i]; ++i)
278                 hv = (hv << 5) ^ (hv >> 23) ^ npath[i];
279         return(hv & ITHMASK);
280 }
281
282 static struct item *
283 lookupItem(const char *npath)
284 {
285         struct item *item;
286
287         for (item = ItemHash[itemhash(npath)]; item; item = item->hnext) {
288                 if (strcmp(npath, item->rpath) == 0)
289                         return(item);
290         }
291         return(NULL);
292 }
293
294 static struct item *
295 addItem(const char *npath)
296 {
297         struct item **itemp;
298         struct item *item = calloc(sizeof(*item), 1);
299         int i;
300
301         itemp = &ItemHash[itemhash(npath)];
302         item->status = XWAITING;
303         item->hnext = *itemp;
304         item->rpath = strdup(npath);
305         item->lpath = strdup(npath);
306         *itemp = item;
307         for (i = 0; item->lpath[i]; ++i) {
308                 if (item->lpath[i] == '/')
309                         item->lpath[i] = '.';
310         }
311
312         return(item);
313 }
314
315 /*
316  * Add a reverse dependency from the deepest point (xitem) to the
317  * packages that depend on xitem (item in this case).
318  *
319  * Caller will check dcount after it is through adding dependencies.
320  */
321 static void
322 addDepn(struct item *item, struct item *xitem)
323 {
324         struct depn *depn = calloc(sizeof(*depn), 1);
325         char *logpath3;
326         FILE *fp;
327
328         depn->item = item;
329         depn->dnext = xitem->dbase;
330         xitem->dbase = depn;
331         if (xitem->status == XDONE) {
332                 if (xitem->xcode) {
333                         assert(item->status == XWAITING ||
334                                item->status == XDEPFAIL);
335                         item->xcode = xitem->xcode;
336                         item->status = XDEPFAIL;
337                         asprintf(&logpath3,
338                                  "/tmp/logs/bad/%s", item->lpath);
339                         fp = fopen(logpath3, "a");
340                         fprintf(fp, "Dependency failed: %s\n",
341                                 xitem->rpath);
342                         fclose(fp);
343                         free(logpath3);
344                 }
345         } else {
346                 ++item->dcount;
347         }
348 }
349
350 /*
351  * Add the item to the build request list.  This routine is called
352  * after all build dependencies have been satisfied for the item.
353  * runBuilds() will pick items off of BuildList to keep the parallel
354  * build pipeline full.
355  */
356 static void
357 addBuild(struct item *item)
358 {
359         printf("%sBuildOrder %s\n", neednl(), item->rpath);
360         *BuildListP = item;
361         BuildListP = &item->bnext;
362         item->status = XBUILD;
363 }
364
365 /*
366  * Start new builds from the build list and handle build completions,
367  * which can potentialy add new items to the build list.
368  *
369  * This routine will maintain up to NParallel builds.  A new build is
370  * only started once its dependencies have completed successfully so
371  * when the bulk build starts it typically takes a little while before
372  * fastbulk can keep the parallel pipeline full.
373  */
374 static void
375 runBuilds(const char *bpath)
376 {
377         struct rlimit rlm;
378         struct item *item;
379         char *logpath;
380         FILE *fp;
381         int fd;
382
383         /*
384          * Try to maintain up to NParallel builds
385          */
386         while (NRunning < NParallel && BuildList) {
387                 item = BuildList;
388                 if ((BuildList = item->bnext) == NULL)
389                         BuildListP = &BuildList;
390                 printf("%sBuildStart %s\n", neednl(), item->rpath);
391
392                 /*
393                  * When [re]running a build remove any bad log from prior
394                  * attempts.
395                  */
396                 asprintf(&logpath, "/tmp/logs/bad/%s", item->lpath);
397                 remove(logpath);
398                 free(logpath);
399
400                 asprintf(&logpath, "/tmp/logs/run/%s", item->lpath);
401
402                 item->status = XRUN;
403
404                 item->pid = fork();
405                 if (item->pid == 0) {
406                         /*
407                          * Child process - setup the log file and exec
408                          */
409                         if (chdir(bpath) < 0)
410                                 _exit(99);
411                         if (chdir(item->rpath) < 0)
412                                 _exit(99);
413
414                         /*
415                          * Connect log file up, disconnect tty (in case a
416                          * 'patch' command tries to ask for help).
417                          */
418                         fd = open(logpath, O_RDWR|O_CREAT|O_TRUNC, 0666);
419                         if (fd != 1)
420                                 dup2(fd, 1);
421                         if (fd != 2)
422                                 dup2(fd, 2);
423                         if (fd != 1 && fd != 2)
424                                 close(fd);
425                         fd = open("/dev/null", O_RDWR);
426                         if (fd != 0) {
427                                 dup2(fd, 0);
428                                 close(fd);
429                         }
430
431                         /*
432                          * Set resource limits:
433                          *
434                          *      ~2 hours cpu    - prevent runaways from stalling
435                          *                        the build.
436                          *
437                          *      ~2GB file size  - prevent endless growing log files.
438                          *
439                          *      ~5GB footprint  - prevent processes w/ out of
440                          *                        control memory usage.
441                          *
442                          *      ~0 core         - No core dumps cluttering
443                          *                        directories, please.
444                          */
445                         if (getrlimit(RLIMIT_CPU, &rlm) == 0 &&
446                             rlm.rlim_cur > 2 * 60 * 60) {
447                                 rlm.rlim_cur = 2 * 60 * 60;
448                                 setrlimit(RLIMIT_CPU, &rlm);
449                         }
450                         if (getrlimit(RLIMIT_FSIZE, &rlm) == 0 &&
451                             rlm.rlim_cur > 2LL * 1024 * 1024 * 1024) {
452                                 rlm.rlim_cur = 2LL * 1024 * 1024 * 1024;
453                                 setrlimit(RLIMIT_FSIZE, &rlm);
454                         }
455                         if (getrlimit(RLIMIT_AS, &rlm) == 0 &&
456                             rlm.rlim_cur > 5LL * 1024 * 1024 * 1024) {
457                                 rlm.rlim_cur = 5LL * 1024 * 1024 * 1024;
458                                 setrlimit(RLIMIT_AS, &rlm);
459                         }
460                         if (getrlimit(RLIMIT_CORE, &rlm) == 0 &&
461                             rlm.rlim_cur > 0) {
462                                 rlm.rlim_cur = 0;
463                                 setrlimit(RLIMIT_CORE, &rlm);
464                         }
465
466                         /*
467                          * Disconnect tty so utilities which try to ask
468                          * for help (like patch) or Y/N answers on /dev/tty
469                          * do not stall.
470                          */
471                         fd = open("/dev/tty", O_RDWR);
472                         if (fd >= 0) {
473                                 ioctl(fd, TIOCNOTTY, 0);
474                                 close(fd);
475                         }
476                         setsid();
477
478                         /*
479                          * we tack a 'clean' on to the repackage to clean
480                          * the work directory on success.  If a failure
481                          * occurs we leave the work directory intact.
482                          *
483                          * leaving work directories around when doing a
484                          * bulk build can fill up the filesystem very fast.
485                          */
486                         execl("/tmp/track/dobuild", "dobuild",
487                                 item->lpath, NULL);
488                         _exit(99);
489                 } else if (item->pid < 0) {
490                         /*
491                          * Parent fork() failed, log the problem and
492                          * do completion processing.
493                          */
494                         item->xcode = -98;
495                         fp = fopen(logpath, "a");
496                         fprintf(fp, "fastbulk: Unable to fork/exec bmake\n");
497                         fclose(fp);
498                         processCompletion(item);
499                 } else {
500                         /*
501                          * Parent is now tracking the running child,
502                          * add the item to the RunList.
503                          */
504                         item->bnext = RunList;
505                         RunList = item;
506                         ++NRunning;
507                 }
508                 free(logpath);
509         }
510
511         /*
512          * Process any completed builds (non-blocking)
513          */
514         while (waitRunning(WNOHANG) != NULL)
515                 ;
516 }
517
518 /*
519  * Wait for a running build to finish and process its completion.
520  * Return the build or NULL if no builds are pending.
521  *
522  * The caller should call runBuilds() in the loop to keep the build
523  * pipeline full until there is nothing left in the build list.
524  */
525 static struct item *
526 waitRunning(int flags)
527 {
528         struct item *item;
529         struct item **itemp;
530         pid_t pid;
531         int status;
532
533         if (RunList == NULL)
534                 return(NULL);
535         while ((pid = wait3(&status, flags, NULL)) < 0 && flags == 0)
536                 ;
537
538         /*
539          * NOTE! The pid may be associated with one of our popen()'s
540          *       so just ignore it if we cannot find it.
541          */
542         if (pid > 0) {
543                 status = WEXITSTATUS(status);
544                 itemp = &RunList;
545                 while ((item = *itemp) != NULL) {
546                         if (item->pid == pid)
547                                 break;
548                         itemp = &item->bnext;
549                 }
550                 if (item) {
551                         *itemp = item->bnext;
552                         item->bnext = NULL;
553                         item->xcode = status;
554                         --NRunning;
555                         processCompletion(item);
556                 }
557         } else {
558                 item = NULL;
559         }
560         return (item);
561 }
562
563 /*
564  * Process the build completion for an item.
565  */
566 static void
567 processCompletion(struct item *item)
568 {
569         struct depn *depn;
570         struct item *xitem;
571         char *logpath1;
572         char *logpath2;
573         char *logpath3;
574         FILE *fp;
575
576         /*
577          * If XRUN we have to move the logfile to the correct directory.
578          * (If XDEPFAIL the logfile is already in the correct directory).
579          */
580         if (item->status == XRUN) {
581                 asprintf(&logpath1, "/tmp/logs/run/%s", item->lpath);
582                 asprintf(&logpath2, "/tmp/logs/%s/%s",
583                          (item->xcode ? "bad" : "good"), item->lpath);
584                 rename(logpath1, logpath2);
585                 free(logpath1);
586                 free(logpath2);
587         }
588
589         printf("%sFinish %-3d %s\n", neednl(), item->xcode, item->rpath);
590         assert(item->status == XRUN || item->status == XDEPFAIL);
591         item->status = XDONE;
592         for (depn = item->dbase; depn; depn = depn->dnext) {
593                 xitem = depn->item;
594                 assert(xitem->dcount > 0);
595                 --xitem->dcount;
596                 if (xitem->status == XWAITING || xitem->status == XDEPFAIL) {
597                         /*
598                          * If our build went well add items dependent
599                          * on us to the build, otherwise fail the items
600                          * dependent on us.
601                          */
602                         if (item->xcode) {
603                                 xitem->xcode = item->xcode;
604                                 xitem->status = XDEPFAIL;
605                                 asprintf(&logpath3,
606                                          "/tmp/logs/bad/%s", xitem->lpath);
607                                 fp = fopen(logpath3, "a");
608                                 fprintf(fp, "Dependency failed: %s\n",
609                                         item->rpath);
610                                 fclose(fp);
611                                 free(logpath3);
612                         }
613                         if (xitem->dcount == 0) {
614                                 if (xitem->status == XWAITING)
615                                         addBuild(xitem);
616                                 else
617                                         processCompletion(xitem);
618                         }
619                 } else if (xitem->status == XDONE && xitem->xcode) {
620                         /*
621                          * The package depending on us has already run
622                          * (this case should not occur).
623                          *
624                          * Add this dependency failure to its log file
625                          * (which has already been renamed).
626                          */
627                         asprintf(&logpath3,
628                                  "/tmp/logs/bad/%s", xitem->lpath);
629                         fp = fopen(logpath3, "a");
630                         fprintf(fp, "Dependency failed: %s\n",
631                                 item->rpath);
632                         fclose(fp);
633                         free(logpath3);
634                 }
635         }
636 }
637
638 static const char *
639 neednl(void)
640 {
641         if (NeedNL) {
642                 NeedNL = 0;
643                 return("\n");
644         } else {
645                 return("");
646         }
647 }
648
649 static void
650 usage(void)
651 {
652         fprintf(stderr, "fastbulk [-j parallel] /usr/pkgsrc\n");
653         exit(1);
654 }