2 * Copyright (c) 2010 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 * fastbulk <pkgsrcdir>
39 * This program iterates all pkgsrc directories, runs 'bmake show-depends-dirs'
40 * recursively, and builds a dependency tree on the fly.
42 * As the dependency tree is being built, terminal dependencies are built
43 * and packaged on the fly (well, really it runs /tmp/track/dobuild inside
44 * the chroot). As these builds complete additional dependencies may be
45 * satisfied and be added to the build order. Ultimately the entire tree
48 * Only one attempt is made to build any given package, no matter how many
49 * other packages depend on it.
51 #include <sys/types.h>
55 #include <sys/ioctl.h>
57 #include <sys/resource.h>
74 enum { XWAITING, XDEPFAIL, XBUILD, XRUN, XDONE } status;
75 struct item *hnext; /* ItemHash next */
76 struct item *bnext; /* BuildList/RunList next */
77 struct depn *dbase; /* packages depending on us */
78 char *rpath; /* e.g. "shells/tcsh" */
79 char *lpath; /* e.g. "shells.tcsh" (log path) */
80 int dcount; /* build completion for our dependencies */
81 int xcode; /* exit code from build */
82 pid_t pid; /* running build */
86 #define ITHMASK (ITHSIZE - 1)
88 static struct item *ItemHash[ITHSIZE];
89 static struct item *BuildList;
90 static struct item **BuildListP = &BuildList;
91 static struct item *RunList;
93 static void ordered_scan(const char *bpath, const char *path, size_t blen);
94 static struct item *ordered_depends(const char *bpath, const char *npath);
95 static struct item *lookupItem(const char *npath);
96 static struct item *addItem(const char *npath);
97 static void addDepn(struct item *item, struct item *xitem);
99 static void addBuild(struct item *item);
100 static void runBuilds(const char *bpath);
101 static struct item *waitRunning(int flags);
102 static void processCompletion(struct item *item);
104 static const char *neednl(void);
105 static void usage(void);
113 main(int ac, char **av)
119 while ((ch = getopt(ac, av, "j:v")) != -1) {
122 NParallel = strtol(optarg, NULL, 0);
136 fprintf(stderr, "requires base directory as first argument\n");
143 bpath = strdup(av[0]);
144 blen = strlen(bpath);
145 while (blen && bpath[blen-1] == '/')
150 * Do recursive directory scan
152 ordered_scan(bpath, bpath, strlen(bpath));
155 * Wait for all current builds to finish running, keep the pipeline
156 * full until both the BuildList and RunList have been exhausted.
159 while (waitRunning(0) != NULL)
165 * Recursively scan the requested directory tree looking for pkgsrc
169 ordered_scan(const char *bpath, const char *path, size_t blen)
177 if ((dir = opendir(path)) != NULL) {
178 while ((den = readdir(dir)) != NULL) {
179 if (den->d_name[0] == '.')
181 asprintf(&npath, "%s/%s", path, den->d_name);
182 asprintf(&xpath, "%s/DESCR", npath);
184 if (lookupItem(npath + blen + 1) == NULL &&
185 stat(npath, &st) == 0 && S_ISDIR(st.st_mode)) {
186 if (stat(xpath, &st) == 0) {
187 ordered_depends(bpath,
190 ordered_scan(bpath, npath, blen);
201 * Recursively execute 'bmake show-depends-dirs' to calculate all required
205 ordered_depends(const char *bpath, const char *npath)
214 item = addItem(npath);
217 * Retrieve and process dependencies recursively. Note that
218 * addDepn() can modify item's status.
220 * Guard the recursion by bumping dcount to prevent the item
221 * from being processed for completion while we are still adding
222 * its dependencies. This would normally not occur but it can
223 * if pkgsrc has a broken dependency loop.
226 asprintf(&cmd, "cd %s/%s; bmake show-depends-dirs", bpath, npath);
227 fp = popen(cmd, "r");
228 while (fgets(buf, sizeof(buf), fp) != NULL) {
230 if (len && buf[len-1] == '\n')
232 xitem = lookupItem(buf);
234 xitem = ordered_depends(bpath, buf);
235 addDepn(item, xitem);
242 * If the item has no dependencies left either add it to the
243 * build list or do completion processing (i.e. if some of the
244 * dependencies failed).
246 if (item->dcount == 0) {
247 switch(item->status) {
252 processCompletion(item);
261 printf("Deferred %s\n", item->rpath);
268 * Item hashing and dependency helper routines, called during the
272 itemhash(const char *npath)
277 for (i = 0; npath[i]; ++i)
278 hv = (hv << 5) ^ (hv >> 23) ^ npath[i];
279 return(hv & ITHMASK);
283 lookupItem(const char *npath)
287 for (item = ItemHash[itemhash(npath)]; item; item = item->hnext) {
288 if (strcmp(npath, item->rpath) == 0)
295 addItem(const char *npath)
298 struct item *item = calloc(sizeof(*item), 1);
301 itemp = &ItemHash[itemhash(npath)];
302 item->status = XWAITING;
303 item->hnext = *itemp;
304 item->rpath = strdup(npath);
305 item->lpath = strdup(npath);
307 for (i = 0; item->lpath[i]; ++i) {
308 if (item->lpath[i] == '/')
309 item->lpath[i] = '.';
316 * Add a reverse dependency from the deepest point (xitem) to the
317 * packages that depend on xitem (item in this case).
319 * Caller will check dcount after it is through adding dependencies.
322 addDepn(struct item *item, struct item *xitem)
324 struct depn *depn = calloc(sizeof(*depn), 1);
329 depn->dnext = xitem->dbase;
331 if (xitem->status == XDONE) {
333 assert(item->status == XWAITING ||
334 item->status == XDEPFAIL);
335 item->xcode = xitem->xcode;
336 item->status = XDEPFAIL;
338 "/tmp/logs/bad/%s", item->lpath);
339 fp = fopen(logpath3, "a");
340 fprintf(fp, "Dependency failed: %s\n",
351 * Add the item to the build request list. This routine is called
352 * after all build dependencies have been satisfied for the item.
353 * runBuilds() will pick items off of BuildList to keep the parallel
354 * build pipeline full.
357 addBuild(struct item *item)
359 printf("%sBuildOrder %s\n", neednl(), item->rpath);
361 BuildListP = &item->bnext;
362 item->status = XBUILD;
366 * Start new builds from the build list and handle build completions,
367 * which can potentialy add new items to the build list.
369 * This routine will maintain up to NParallel builds. A new build is
370 * only started once its dependencies have completed successfully so
371 * when the bulk build starts it typically takes a little while before
372 * fastbulk can keep the parallel pipeline full.
375 runBuilds(const char *bpath)
384 * Try to maintain up to NParallel builds
386 while (NRunning < NParallel && BuildList) {
388 if ((BuildList = item->bnext) == NULL)
389 BuildListP = &BuildList;
390 printf("%sBuildStart %s\n", neednl(), item->rpath);
393 * When [re]running a build remove any bad log from prior
396 asprintf(&logpath, "/tmp/logs/bad/%s", item->lpath);
400 asprintf(&logpath, "/tmp/logs/run/%s", item->lpath);
405 if (item->pid == 0) {
407 * Child process - setup the log file and exec
409 if (chdir(bpath) < 0)
411 if (chdir(item->rpath) < 0)
415 * Connect log file up, disconnect tty (in case a
416 * 'patch' command tries to ask for help).
418 fd = open(logpath, O_RDWR|O_CREAT|O_TRUNC, 0666);
423 if (fd != 1 && fd != 2)
425 fd = open("/dev/null", O_RDWR);
432 * Set resource limits:
434 * ~2 hours cpu - prevent runaways from stalling
437 * ~2GB file size - prevent endless growing log files.
439 * ~5GB footprint - prevent processes w/ out of
440 * control memory usage.
442 * ~0 core - No core dumps cluttering
443 * directories, please.
445 if (getrlimit(RLIMIT_CPU, &rlm) == 0 &&
446 rlm.rlim_cur > 2 * 60 * 60) {
447 rlm.rlim_cur = 2 * 60 * 60;
448 setrlimit(RLIMIT_CPU, &rlm);
450 if (getrlimit(RLIMIT_FSIZE, &rlm) == 0 &&
451 rlm.rlim_cur > 2LL * 1024 * 1024 * 1024) {
452 rlm.rlim_cur = 2LL * 1024 * 1024 * 1024;
453 setrlimit(RLIMIT_FSIZE, &rlm);
455 if (getrlimit(RLIMIT_AS, &rlm) == 0 &&
456 rlm.rlim_cur > 5LL * 1024 * 1024 * 1024) {
457 rlm.rlim_cur = 5LL * 1024 * 1024 * 1024;
458 setrlimit(RLIMIT_AS, &rlm);
460 if (getrlimit(RLIMIT_CORE, &rlm) == 0 &&
463 setrlimit(RLIMIT_CORE, &rlm);
467 * Disconnect tty so utilities which try to ask
468 * for help (like patch) or Y/N answers on /dev/tty
471 fd = open("/dev/tty", O_RDWR);
473 ioctl(fd, TIOCNOTTY, 0);
479 * we tack a 'clean' on to the repackage to clean
480 * the work directory on success. If a failure
481 * occurs we leave the work directory intact.
483 * leaving work directories around when doing a
484 * bulk build can fill up the filesystem very fast.
486 execl("/tmp/track/dobuild", "dobuild",
489 } else if (item->pid < 0) {
491 * Parent fork() failed, log the problem and
492 * do completion processing.
495 fp = fopen(logpath, "a");
496 fprintf(fp, "fastbulk: Unable to fork/exec bmake\n");
498 processCompletion(item);
501 * Parent is now tracking the running child,
502 * add the item to the RunList.
504 item->bnext = RunList;
512 * Process any completed builds (non-blocking)
514 while (waitRunning(WNOHANG) != NULL)
519 * Wait for a running build to finish and process its completion.
520 * Return the build or NULL if no builds are pending.
522 * The caller should call runBuilds() in the loop to keep the build
523 * pipeline full until there is nothing left in the build list.
526 waitRunning(int flags)
535 while ((pid = wait3(&status, flags, NULL)) < 0 && flags == 0)
539 * NOTE! The pid may be associated with one of our popen()'s
540 * so just ignore it if we cannot find it.
543 status = WEXITSTATUS(status);
545 while ((item = *itemp) != NULL) {
546 if (item->pid == pid)
548 itemp = &item->bnext;
551 *itemp = item->bnext;
553 item->xcode = status;
555 processCompletion(item);
564 * Process the build completion for an item.
567 processCompletion(struct item *item)
577 * If XRUN we have to move the logfile to the correct directory.
578 * (If XDEPFAIL the logfile is already in the correct directory).
580 if (item->status == XRUN) {
581 asprintf(&logpath1, "/tmp/logs/run/%s", item->lpath);
582 asprintf(&logpath2, "/tmp/logs/%s/%s",
583 (item->xcode ? "bad" : "good"), item->lpath);
584 rename(logpath1, logpath2);
589 printf("%sFinish %-3d %s\n", neednl(), item->xcode, item->rpath);
590 assert(item->status == XRUN || item->status == XDEPFAIL);
591 item->status = XDONE;
592 for (depn = item->dbase; depn; depn = depn->dnext) {
594 assert(xitem->dcount > 0);
596 if (xitem->status == XWAITING || xitem->status == XDEPFAIL) {
598 * If our build went well add items dependent
599 * on us to the build, otherwise fail the items
603 xitem->xcode = item->xcode;
604 xitem->status = XDEPFAIL;
606 "/tmp/logs/bad/%s", xitem->lpath);
607 fp = fopen(logpath3, "a");
608 fprintf(fp, "Dependency failed: %s\n",
613 if (xitem->dcount == 0) {
614 if (xitem->status == XWAITING)
617 processCompletion(xitem);
619 } else if (xitem->status == XDONE && xitem->xcode) {
621 * The package depending on us has already run
622 * (this case should not occur).
624 * Add this dependency failure to its log file
625 * (which has already been renamed).
628 "/tmp/logs/bad/%s", xitem->lpath);
629 fp = fopen(logpath3, "a");
630 fprintf(fp, "Dependency failed: %s\n",
652 fprintf(stderr, "fastbulk [-j parallel] /usr/pkgsrc\n");