2 * Copyright (c) 2010 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 * fastbulk <pkgsrcdir>
39 * This program iterates all pkgsrc directories, runs 'bmake show-depends-dirs'
40 * recursively, and builds a dependency tree on the fly.
42 * As the dependency tree is being built, terminal dependencies are built
43 * and packaged on the fly (well, really it runs /tmp/track/dobuild inside
44 * the chroot). As these builds complete additional dependencies may be
45 * satisfied and be added to the build order. Ultimately the entire tree
48 * Only one attempt is made to build any given package, no matter how many
49 * other packages depend on it.
69 enum { XWAITING, XDEPFAIL, XBUILD, XRUN, XDONE } status;
70 struct item *hnext; /* ItemHash next */
71 struct item *bnext; /* BuildList/RunList next */
72 struct depn *dbase; /* packages depending on us */
73 char *rpath; /* e.g. "shells/tcsh" */
74 char *lpath; /* e.g. "shells.tcsh" (log path) */
75 int dcount; /* build completion for our dependencies */
76 int xcode; /* exit code from build */
77 pid_t pid; /* running build */
81 #define ITHMASK (ITHSIZE - 1)
83 static struct item *ItemHash[ITHSIZE];
84 static struct item *BuildList;
85 static struct item **BuildListP = &BuildList;
86 static struct item *RunList;
88 static void ordered_scan(const char *bpath, const char *path, size_t blen);
89 static struct item *ordered_depends(const char *bpath, const char *npath);
90 static struct item *lookupItem(const char *npath);
91 static struct item *addItem(const char *npath);
92 static void addDepn(struct item *item, struct item *xitem);
94 static void addBuild(struct item *item);
95 static void runBuilds(const char *bpath);
96 static struct item *waitRunning(int flags);
97 static void processCompletion(struct item *item);
99 static const char *neednl(void);
100 static void usage(void);
108 main(int ac, char **av)
114 while ((ch = getopt(ac, av, "j:v")) != -1) {
117 NParallel = strtol(optarg, NULL, 0);
131 fprintf(stderr, "requires base directory as first argument\n");
138 bpath = strdup(av[0]);
139 blen = strlen(bpath);
140 while (blen && bpath[blen-1] == '/')
145 * Do recursive directory scan
147 ordered_scan(bpath, bpath, strlen(bpath));
150 * Wait for all current builds to finish running, keep the pipeline
151 * full until both the BuildList and RunList have been exhausted.
154 while (waitRunning(0) != NULL)
160 * Recursively scan the requested directory tree looking for pkgsrc
164 ordered_scan(const char *bpath, const char *path, size_t blen)
172 if ((dir = opendir(path)) != NULL) {
173 while ((den = readdir(dir)) != NULL) {
174 if (den->d_name[0] == '.')
176 asprintf(&npath, "%s/%s", path, den->d_name);
177 asprintf(&xpath, "%s/DESCR", npath);
179 if (lookupItem(npath + blen + 1) == NULL &&
180 stat(npath, &st) == 0 && S_ISDIR(st.st_mode)) {
181 if (stat(xpath, &st) == 0) {
182 ordered_depends(bpath,
185 ordered_scan(bpath, npath, blen);
196 * Recursively execute 'bmake show-depends-dirs' to calculate all required
200 ordered_depends(const char *bpath, const char *npath)
209 item = addItem(npath);
212 * Retrieve and process dependencies recursively. Note that
213 * addDepn() can modify item's status.
215 * Guard the recursion by bumping dcount to prevent the item
216 * from being processed for completion while we are still adding
217 * its dependencies. This would normally not occur but it can
218 * if pkgsrc has a broken dependency loop.
221 asprintf(&cmd, "cd %s/%s; bmake show-depends-dirs", bpath, npath);
222 fp = popen(cmd, "r");
223 while (fgets(buf, sizeof(buf), fp) != NULL) {
225 if (len && buf[len-1] == '\n')
227 xitem = lookupItem(buf);
229 xitem = ordered_depends(bpath, buf);
230 addDepn(item, xitem);
237 * If the item has no dependencies left either add it to the
238 * build list or do completion processing (i.e. if some of the
239 * dependencies failed).
241 if (item->dcount == 0) {
242 switch(item->status) {
247 processCompletion(item);
256 printf("Deferred %s\n", item->rpath);
263 * Item hashing and dependency helper routines, called during the
267 itemhash(const char *npath)
272 for (i = 0; npath[i]; ++i)
273 hv = (hv << 5) ^ (hv >> 23) ^ npath[i];
274 return(hv & ITHMASK);
278 lookupItem(const char *npath)
282 for (item = ItemHash[itemhash(npath)]; item; item = item->hnext) {
283 if (strcmp(npath, item->rpath) == 0)
290 addItem(const char *npath)
293 struct item *item = calloc(sizeof(*item), 1);
296 itemp = &ItemHash[itemhash(npath)];
297 item->status = XWAITING;
298 item->hnext = *itemp;
299 item->rpath = strdup(npath);
300 item->lpath = strdup(npath);
302 for (i = 0; item->lpath[i]; ++i) {
303 if (item->lpath[i] == '/')
304 item->lpath[i] = '.';
311 * Add a reverse dependency from the deepest point (xitem) to the
312 * packages that depend on xitem (item in this case).
314 * Caller will check dcount after it is through adding dependencies.
317 addDepn(struct item *item, struct item *xitem)
319 struct depn *depn = calloc(sizeof(*depn), 1);
324 depn->dnext = xitem->dbase;
326 if (xitem->status == XDONE) {
328 assert(item->status == XWAITING ||
329 item->status == XDEPFAIL);
330 item->xcode = xitem->xcode;
331 item->status = XDEPFAIL;
333 "/tmp/logs/bad/%s", item->lpath);
334 fp = fopen(logpath3, "a");
335 fprintf(fp, "Dependency failed: %s\n",
346 * Add the item to the build request list. This routine is called
347 * after all build dependencies have been satisfied for the item.
348 * runBuilds() will pick items off of BuildList to keep the parallel
349 * build pipeline full.
352 addBuild(struct item *item)
354 printf("%sBuildOrder %s\n", neednl(), item->rpath);
356 BuildListP = &item->bnext;
357 item->status = XBUILD;
361 * Start new builds from the build list and handle build completions,
362 * which can potentialy add new items to the build list.
364 * This routine will maintain up to NParallel builds. A new build is
365 * only started once its dependencies have completed successfully so
366 * when the bulk build starts it typically takes a little while before
367 * fastbulk can keep the parallel pipeline full.
370 runBuilds(const char *bpath)
378 * Try to maintain up to NParallel builds
380 while (NRunning < NParallel && BuildList) {
382 if ((BuildList = item->bnext) == NULL)
383 BuildListP = &BuildList;
384 printf("%sBuildStart %s\n", neednl(), item->rpath);
387 * When [re]running a build remove any bad log from prior
390 asprintf(&logpath, "/tmp/logs/bad/%s", item->lpath);
394 asprintf(&logpath, "/tmp/logs/run/%s", item->lpath);
399 if (item->pid == 0) {
401 * Child process - setup the log file and exec
403 if (chdir(bpath) < 0)
405 if (chdir(item->rpath) < 0)
408 fd = open(logpath, O_RDWR|O_CREAT|O_TRUNC, 0666);
413 if (fd != 1 && fd != 2)
415 fd = open("/dev/null", O_RDWR);
422 * we tack a 'clean' on to the repackage to clean
423 * the work directory on success. If a failure
424 * occurs we leave the work directory intact.
426 * leaving work directories around when doing a
427 * bulk build can fill up the filesystem very fast.
429 execl("/tmp/track/dobuild", "dobuild",
432 } else if (item->pid < 0) {
434 * Parent fork() failed, log the problem and
435 * do completion processing.
438 fp = fopen(logpath, "a");
439 fprintf(fp, "fastbulk: Unable to fork/exec bmake\n");
441 processCompletion(item);
444 * Parent is now tracking the running child,
445 * add the item to the RunList.
447 item->bnext = RunList;
455 * Process any completed builds (non-blocking)
457 while (waitRunning(WNOHANG) != NULL)
462 * Wait for a running build to finish and process its completion.
463 * Return the build or NULL if no builds are pending.
465 * The caller should call runBuilds() in the loop to keep the build
466 * pipeline full until there is nothing left in the build list.
469 waitRunning(int flags)
478 while ((pid = wait3(&status, flags, NULL)) < 0 && flags == 0)
482 * NOTE! The pid may be associated with one of our popen()'s
483 * so just ignore it if we cannot find it.
486 status = WEXITSTATUS(status);
488 while ((item = *itemp) != NULL) {
489 if (item->pid == pid)
491 itemp = &item->bnext;
494 *itemp = item->bnext;
496 item->xcode = status;
498 processCompletion(item);
507 * Process the build completion for an item.
510 processCompletion(struct item *item)
520 * If XRUN we have to move the logfile to the correct directory.
521 * (If XDEPFAIL the logfile is already in the correct directory).
523 if (item->status == XRUN) {
524 asprintf(&logpath1, "/tmp/logs/run/%s", item->lpath);
525 asprintf(&logpath2, "/tmp/logs/%s/%s",
526 (item->xcode ? "bad" : "good"), item->lpath);
527 rename(logpath1, logpath2);
532 printf("%sFinish %-3d %s\n", neednl(), item->xcode, item->rpath);
533 assert(item->status == XRUN || item->status == XDEPFAIL);
534 item->status = XDONE;
535 for (depn = item->dbase; depn; depn = depn->dnext) {
537 assert(xitem->dcount > 0);
539 if (xitem->status == XWAITING || xitem->status == XDEPFAIL) {
541 * If our build went well add items dependent
542 * on us to the build, otherwise fail the items
546 xitem->xcode = item->xcode;
547 xitem->status = XDEPFAIL;
549 "/tmp/logs/bad/%s", xitem->lpath);
550 fp = fopen(logpath3, "a");
551 fprintf(fp, "Dependency failed: %s\n",
556 if (xitem->dcount == 0) {
557 if (xitem->status == XWAITING)
560 processCompletion(xitem);
562 } else if (xitem->status == XDONE && xitem->xcode) {
564 * The package depending on us has already run
565 * (this case should not occur).
567 * Add this dependency failure to its log file
568 * (which has already been renamed).
571 "/tmp/logs/bad/%s", xitem->lpath);
572 fp = fopen(logpath3, "a");
573 fprintf(fp, "Dependency failed: %s\n",
595 fprintf(stderr, "fastbulk [-j parallel] /usr/pkgsrc\n");