test/fastbulk/fastbulk.c

   1 /*
   2  * Copyright (c) 2010 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34 /*
  35  * fastbulk.c
  36  *
  37  * fastbulk <pkgsrcdir>
  38  *
  39  * This program iterates all pkgsrc directories, runs 'bmake show-depends-dirs'
  40  * recursively, and builds a dependency tree on the fly.
  41  *
  42  * As the dependency tree is being built, terminal dependencies are built
  43  * and packaged on the fly (well, really it runs /tmp/track/dobuild inside
  44  * the chroot).  As these builds complete additional dependencies may be
  45  * satisfied and be added to the build order.  Ultimately the entire tree
  46  * is built.
  47  *
  48  * Only one attempt is made to build any given package, no matter how many
  49  * other packages depend on it.
  50  */
  51 #include <stdio.h>
  52 #include <stdlib.h>
  53 #include <unistd.h>
  54 #include <string.h>
  55 #include <dirent.h>
  56 #include <assert.h>
  57 #include <sys/file.h>
  58 #include <sys/stat.h>
  59 #include <sys/wait.h>
  60
  61 struct item;
  62
  63 struct depn {
  64         struct depn *dnext;
  65         struct item *item;
  66 };
  67
  68 struct item {
  69         enum { XWAITING, XDEPFAIL, XBUILD, XRUN, XDONE } status;
  70         struct item *hnext;     /* ItemHash next */
  71         struct item *bnext;     /* BuildList/RunList next */
  72         struct depn *dbase;     /* packages depending on us */
  73         char *rpath;            /* e.g. "shells/tcsh" */
  74         char *lpath;            /* e.g. "shells.tcsh" (log path) */
  75         int dcount;             /* build completion for our dependencies */
  76         int xcode;              /* exit code from build */
  77         pid_t pid;              /* running build */
  78 };
  79
  80 #define ITHSIZE 1024
  81 #define ITHMASK (ITHSIZE - 1)
  82
  83 static struct item *ItemHash[ITHSIZE];
  84 static struct item *BuildList;
  85 static struct item **BuildListP = &BuildList;
  86 static struct item *RunList;
  87
  88 static void ordered_scan(const char *bpath, const char *path, size_t blen);
  89 static struct item *ordered_depends(const char *bpath, const char *npath);
  90 static struct item *lookupItem(const char *npath);
  91 static struct item *addItem(const char *npath);
  92 static void addDepn(struct item *item, struct item *xitem);
  93
  94 static void addBuild(struct item *item);
  95 static void runBuilds(const char *bpath);
  96 static struct item *waitRunning(int flags);
  97 static void processCompletion(struct item *item);
  98
  99 static const char *neednl(void);
 100 static void usage(void);
 101
 102 int NParallel = 1;
 103 int VerboseOpt;
 104 int NRunning;
 105 int NeedNL;
 106
 107 int
 108 main(int ac, char **av)
 109 {
 110         char *bpath;
 111         size_t blen;
 112         int ch;
 113
 114         while ((ch = getopt(ac, av, "j:v")) != -1) {
 115                 switch(ch) {
 116                 case 'j':
 117                         NParallel = strtol(optarg, NULL, 0);
 118                         break;
 119                 case 'v':
 120                         VerboseOpt = 1;
 121                         break;
 122                 default:
 123                         usage();
 124                         /* NOT REACHED */
 125                 }
 126         }
 127         ac -= optind;
 128         av += optind;
 129
 130         if (ac != 1) {
 131                 fprintf(stderr, "requires base directory as first argument\n");
 132                 exit(1);
 133         }
 134
 135         /*
 136          * Base dir
 137          */
 138         bpath = strdup(av[0]);
 139         blen = strlen(bpath);
 140         while (blen && bpath[blen-1] == '/')
 141                 --blen;
 142         bpath[blen] = 0;
 143
 144         /*
 145          * Do recursive directory scan
 146          */
 147         ordered_scan(bpath, bpath, strlen(bpath));
 148
 149         /*
 150          * Wait for all current builds to finish running, keep the pipeline
 151          * full until both the BuildList and RunList have been exhausted.
 152          */
 153         runBuilds(bpath);
 154         while (waitRunning(0) != NULL)
 155                 runBuilds(bpath);
 156         return(0);
 157 }
 158
 159 /*
 160  * Recursively scan the requested directory tree looking for pkgsrc
 161  * stuff to build.
 162  */
 163 static void
 164 ordered_scan(const char *bpath, const char *path, size_t blen)
 165 {
 166         DIR *dir;
 167         struct dirent *den;
 168         char *npath;
 169         char *xpath;
 170         struct stat st;
 171
 172         if ((dir = opendir(path)) != NULL) {
 173                 while ((den = readdir(dir)) != NULL) {
 174                         if (den->d_name[0] == '.')
 175                                 continue;
 176                         asprintf(&npath, "%s/%s", path, den->d_name);
 177                         asprintf(&xpath, "%s/DESCR", npath);
 178
 179                         if (lookupItem(npath + blen + 1) == NULL &&
 180                             stat(npath, &st) == 0 && S_ISDIR(st.st_mode)) {
 181                                 if (stat(xpath, &st) == 0) {
 182                                         ordered_depends(bpath,
 183                                                         npath + blen + 1);
 184                                 } else {
 185                                         ordered_scan(bpath, npath, blen);
 186                                 }
 187                         }
 188                         free(npath);
 189                         free(xpath);
 190                 }
 191                 closedir(dir);
 192         }
 193 }
 194
 195 /*
 196  * Recursively execute 'bmake show-depends-dirs' to calculate all required
 197  * dependencies.
 198  */
 199 static struct item *
 200 ordered_depends(const char *bpath, const char *npath)
 201 {
 202         struct item *item;
 203         struct item *xitem;
 204         char buf[1024];
 205         FILE *fp;
 206         char *cmd;
 207         int len;
 208
 209         item = addItem(npath);
 210
 211         /*
 212          * Retrieve and process dependencies recursively.  Note that
 213          * addDepn() can modify item's status.
 214          *
 215          * Guard the recursion by bumping dcount to prevent the item
 216          * from being processed for completion while we are still adding
 217          * its dependencies.  This would normally not occur but it can
 218          * if pkgsrc has a broken dependency loop.
 219          */
 220         ++item->dcount;
 221         asprintf(&cmd, "cd %s/%s; bmake show-depends-dirs", bpath, npath);
 222         fp = popen(cmd, "r");
 223         while (fgets(buf, sizeof(buf), fp) != NULL) {
 224                 len = strlen(buf);
 225                 if (len && buf[len-1] == '\n')
 226                         buf[--len] = 0;
 227                 xitem = lookupItem(buf);
 228                 if (xitem == NULL)
 229                         xitem = ordered_depends(bpath, buf);
 230                 addDepn(item, xitem);
 231         }
 232         pclose(fp);
 233         free(cmd);
 234         --item->dcount;
 235
 236         /*
 237          * If the item has no dependencies left either add it to the
 238          * build list or do completion processing (i.e. if some of the
 239          * dependencies failed).
 240          */
 241         if (item->dcount == 0) {
 242                 switch(item->status) {
 243                 case XWAITING:
 244                         addBuild(item);
 245                         break;
 246                 case XDEPFAIL:
 247                         processCompletion(item);
 248                         break;
 249                 default:
 250                         assert(0);
 251                         /* NOT REACHED */
 252                         break;
 253                 }
 254         } else {
 255                 if (VerboseOpt)
 256                         printf("Deferred   %s\n", item->rpath);
 257         }
 258         runBuilds(bpath);
 259         return (item);
 260 }
 261
 262 /*
 263  * Item hashing and dependency helper routines, called during the
 264  * directory scan.
 265  */
 266 static int
 267 itemhash(const char *npath)
 268 {
 269         int hv = 0xA1B5F342;
 270         int i;
 271
 272         for (i = 0; npath[i]; ++i)
 273                 hv = (hv << 5) ^ (hv >> 23) ^ npath[i];
 274         return(hv & ITHMASK);
 275 }
 276
 277 static struct item *
 278 lookupItem(const char *npath)
 279 {
 280         struct item *item;
 281
 282         for (item = ItemHash[itemhash(npath)]; item; item = item->hnext) {
 283                 if (strcmp(npath, item->rpath) == 0)
 284                         return(item);
 285         }
 286         return(NULL);
 287 }
 288
 289 static struct item *
 290 addItem(const char *npath)
 291 {
 292         struct item **itemp;
 293         struct item *item = calloc(sizeof(*item), 1);
 294         int i;
 295
 296         itemp = &ItemHash[itemhash(npath)];
 297         item->status = XWAITING;
 298         item->hnext = *itemp;
 299         item->rpath = strdup(npath);
 300         item->lpath = strdup(npath);
 301         *itemp = item;
 302         for (i = 0; item->lpath[i]; ++i) {
 303                 if (item->lpath[i] == '/')
 304                         item->lpath[i] = '.';
 305         }
 306
 307         return(item);
 308 }
 309
 310 /*
 311  * Add a reverse dependency from the deepest point (xitem) to the
 312  * packages that depend on xitem (item in this case).
 313  *
 314  * Caller will check dcount after it is through adding dependencies.
 315  */
 316 static void
 317 addDepn(struct item *item, struct item *xitem)
 318 {
 319         struct depn *depn = calloc(sizeof(*depn), 1);
 320         char *logpath3;
 321         FILE *fp;
 322
 323         depn->item = item;
 324         depn->dnext = xitem->dbase;
 325         xitem->dbase = depn;
 326         if (xitem->status == XDONE) {
 327                 if (xitem->xcode) {
 328                         assert(item->status == XWAITING ||
 329                                item->status == XDEPFAIL);
 330                         item->xcode = xitem->xcode;
 331                         item->status = XDEPFAIL;
 332                         asprintf(&logpath3,
 333                                  "/tmp/logs/bad/%s", item->lpath);
 334                         fp = fopen(logpath3, "a");
 335                         fprintf(fp, "Dependency failed: %s\n",
 336                                 xitem->rpath);
 337                         fclose(fp);
 338                         free(logpath3);
 339                 }
 340         } else {
 341                 ++item->dcount;
 342         }
 343 }
 344
 345 /*
 346  * Add the item to the build request list.  This routine is called
 347  * after all build dependencies have been satisfied for the item.
 348  * runBuilds() will pick items off of BuildList to keep the parallel
 349  * build pipeline full.
 350  */
 351 static void
 352 addBuild(struct item *item)
 353 {
 354         printf("%sBuildOrder %s\n", neednl(), item->rpath);
 355         *BuildListP = item;
 356         BuildListP = &item->bnext;
 357         item->status = XBUILD;
 358 }
 359
 360 /*
 361  * Start new builds from the build list and handle build completions,
 362  * which can potentialy add new items to the build list.
 363  *
 364  * This routine will maintain up to NParallel builds.  A new build is
 365  * only started once its dependencies have completed successfully so
 366  * when the bulk build starts it typically takes a little while before
 367  * fastbulk can keep the parallel pipeline full.
 368  */
 369 static void
 370 runBuilds(const char *bpath)
 371 {
 372         struct item *item;
 373         char *logpath;
 374         FILE *fp;
 375         int fd;
 376
 377         /*
 378          * Try to maintain up to NParallel builds
 379          */
 380         while (NRunning < NParallel && BuildList) {
 381                 item = BuildList;
 382                 if ((BuildList = item->bnext) == NULL)
 383                         BuildListP = &BuildList;
 384                 printf("%sBuildStart %s\n", neednl(), item->rpath);
 385
 386                 /*
 387                  * When [re]running a build remove any bad log from prior
 388                  * attempts.
 389                  */
 390                 asprintf(&logpath, "/tmp/logs/bad/%s", item->lpath);
 391                 remove(logpath);
 392                 free(logpath);
 393
 394                 asprintf(&logpath, "/tmp/logs/run/%s", item->lpath);
 395
 396                 item->status = XRUN;
 397
 398                 item->pid = fork();
 399                 if (item->pid == 0) {
 400                         /*
 401                          * Child process - setup the log file and exec
 402                          */
 403                         if (chdir(bpath) < 0)
 404                                 _exit(99);
 405                         if (chdir(item->rpath) < 0)
 406                                 _exit(99);
 407
 408                         fd = open(logpath, O_RDWR|O_CREAT|O_TRUNC, 0666);
 409                         if (fd != 1)
 410                                 dup2(fd, 1);
 411                         if (fd != 2)
 412                                 dup2(fd, 2);
 413                         if (fd != 1 && fd != 2)
 414                                 close(fd);
 415                         fd = open("/dev/null", O_RDWR);
 416                         if (fd != 0) {
 417                                 dup2(fd, 0);
 418                                 close(fd);
 419                         }
 420
 421                         /*
 422                          * we tack a 'clean' on to the repackage to clean
 423                          * the work directory on success.  If a failure
 424                          * occurs we leave the work directory intact.
 425                          *
 426                          * leaving work directories around when doing a
 427                          * bulk build can fill up the filesystem very fast.
 428                          */
 429                         execl("/tmp/track/dobuild", "dobuild",
 430                                 item->lpath, NULL);
 431                         _exit(99);
 432                 } else if (item->pid < 0) {
 433                         /*
 434                          * Parent fork() failed, log the problem and
 435                          * do completion processing.
 436                          */
 437                         item->xcode = -98;
 438                         fp = fopen(logpath, "a");
 439                         fprintf(fp, "fastbulk: Unable to fork/exec bmake\n");
 440                         fclose(fp);
 441                         processCompletion(item);
 442                 } else {
 443                         /*
 444                          * Parent is now tracking the running child,
 445                          * add the item to the RunList.
 446                          */
 447                         item->bnext = RunList;
 448                         RunList = item;
 449                         ++NRunning;
 450                 }
 451                 free(logpath);
 452         }
 453
 454         /*
 455          * Process any completed builds (non-blocking)
 456          */
 457         while (waitRunning(WNOHANG) != NULL)
 458                 ;
 459 }
 460
 461 /*
 462  * Wait for a running build to finish and process its completion.
 463  * Return the build or NULL if no builds are pending.
 464  *
 465  * The caller should call runBuilds() in the loop to keep the build
 466  * pipeline full until there is nothing left in the build list.
 467  */
 468 static struct item *
 469 waitRunning(int flags)
 470 {
 471         struct item *item;
 472         struct item **itemp;
 473         pid_t pid;
 474         int status;
 475
 476         if (RunList == NULL)
 477                 return(NULL);
 478         while ((pid = wait3(&status, flags, NULL)) < 0 && flags == 0)
 479                 ;
 480
 481         /*
 482          * NOTE! The pid may be associated with one of our popen()'s
 483          *       so just ignore it if we cannot find it.
 484          */
 485         if (pid > 0) {
 486                 status = WEXITSTATUS(status);
 487                 itemp = &RunList;
 488                 while ((item = *itemp) != NULL) {
 489                         if (item->pid == pid)
 490                                 break;
 491                         itemp = &item->bnext;
 492                 }
 493                 if (item) {
 494                         *itemp = item->bnext;
 495                         item->bnext = NULL;
 496                         item->xcode = status;
 497                         --NRunning;
 498                         processCompletion(item);
 499                 }
 500         } else {
 501                 item = NULL;
 502         }
 503         return (item);
 504 }
 505
 506 /*
 507  * Process the build completion for an item.
 508  */
 509 static void
 510 processCompletion(struct item *item)
 511 {
 512         struct depn *depn;
 513         struct item *xitem;
 514         char *logpath1;
 515         char *logpath2;
 516         char *logpath3;
 517         FILE *fp;
 518
 519         /*
 520          * If XRUN we have to move the logfile to the correct directory.
 521          * (If XDEPFAIL the logfile is already in the correct directory).
 522          */
 523         if (item->status == XRUN) {
 524                 asprintf(&logpath1, "/tmp/logs/run/%s", item->lpath);
 525                 asprintf(&logpath2, "/tmp/logs/%s/%s",
 526                          (item->xcode ? "bad" : "good"), item->lpath);
 527                 rename(logpath1, logpath2);
 528                 free(logpath1);
 529                 free(logpath2);
 530         }
 531
 532         printf("%sFinish %-3d %s\n", neednl(), item->xcode, item->rpath);
 533         assert(item->status == XRUN || item->status == XDEPFAIL);
 534         item->status = XDONE;
 535         for (depn = item->dbase; depn; depn = depn->dnext) {
 536                 xitem = depn->item;
 537                 assert(xitem->dcount > 0);
 538                 --xitem->dcount;
 539                 if (xitem->status == XWAITING || xitem->status == XDEPFAIL) {
 540                         /*
 541                          * If our build went well add items dependent
 542                          * on us to the build, otherwise fail the items
 543                          * dependent on us.
 544                          */
 545                         if (item->xcode) {
 546                                 xitem->xcode = item->xcode;
 547                                 xitem->status = XDEPFAIL;
 548                                 asprintf(&logpath3,
 549                                          "/tmp/logs/bad/%s", xitem->lpath);
 550                                 fp = fopen(logpath3, "a");
 551                                 fprintf(fp, "Dependency failed: %s\n",
 552                                         item->rpath);
 553                                 fclose(fp);
 554                                 free(logpath3);
 555                         }
 556                         if (xitem->dcount == 0) {
 557                                 if (xitem->status == XWAITING)
 558                                         addBuild(xitem);
 559                                 else
 560                                         processCompletion(xitem);
 561                         }
 562                 } else if (xitem->status == XDONE && xitem->xcode) {
 563                         /*
 564                          * The package depending on us has already run
 565                          * (this case should not occur).
 566                          *
 567                          * Add this dependency failure to its log file
 568                          * (which has already been renamed).
 569                          */
 570                         asprintf(&logpath3,
 571                                  "/tmp/logs/bad/%s", xitem->lpath);
 572                         fp = fopen(logpath3, "a");
 573                         fprintf(fp, "Dependency failed: %s\n",
 574                                 item->rpath);
 575                         fclose(fp);
 576                         free(logpath3);
 577                 }
 578         }
 579 }
 580
 581 static const char *
 582 neednl(void)
 583 {
 584         if (NeedNL) {
 585                 NeedNL = 0;
 586                 return("\n");
 587         } else {
 588                 return("");
 589         }
 590 }
 591
 592 static void
 593 usage(void)
 594 {
 595         fprintf(stderr, "fastbulk [-j parallel] /usr/pkgsrc\n");
 596         exit(1);
 597 }