test/fastbulk/fastbulk.c

   1 /*
   2  * Copyright (c) 2010 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34 /*
  35  * fastbulk.c
  36  *
  37  * fastbulk <pkgsrcdir>
  38  *
  39  * This program iterates all pkgsrc directories, runs 'bmake show-depends-dirs'
  40  * recursively, and builds a dependency tree on the fly.
  41  *
  42  * As the dependency tree is being built, terminal dependencies are built
  43  * and packaged on the fly (well, really it runs /tmp/track/dobuild inside
  44  * the chroot).  As these builds complete additional dependencies may be
  45  * satisfied and be added to the build order.  Ultimately the entire tree
  46  * is built.
  47  *
  48  * Only one attempt is made to build any given package, no matter how many
  49  * other packages depend on it.
  50  */
  51 #include <sys/types.h>
  52 #include <sys/file.h>
  53 #include <sys/stat.h>
  54 #include <sys/wait.h>
  55 #include <sys/ioctl.h>
  56 #include <sys/time.h>
  57 #include <sys/resource.h>
  58
  59 #include <stdio.h>
  60 #include <stdlib.h>
  61 #include <unistd.h>
  62 #include <string.h>
  63 #include <dirent.h>
  64 #include <assert.h>
  65
  66 struct item;
  67
  68 struct depn {
  69         struct depn *dnext;
  70         struct item *item;
  71 };
  72
  73 struct item {
  74         enum { XWAITING, XDEPFAIL, XBUILD, XRUN, XDONE } status;
  75         struct item *hnext;     /* ItemHash next */
  76         struct item *bnext;     /* BuildList/RunList next */
  77         struct depn *dbase;     /* packages depending on us */
  78         char *rpath;            /* e.g. "shells/tcsh" */
  79         char *lpath;            /* e.g. "shells.tcsh" (log path) */
  80         int dcount;             /* build completion for our dependencies */
  81         int xcode;              /* exit code from build */
  82         pid_t pid;              /* running build */
  83 };
  84
  85 #define ITHSIZE 1024
  86 #define ITHMASK (ITHSIZE - 1)
  87
  88 static struct item *ItemHash[ITHSIZE];
  89 static struct item *BuildList;
  90 static struct item **BuildListP = &BuildList;
  91 static struct item *RunList;
  92
  93 static void ordered_scan(const char *bpath, const char *path, size_t blen);
  94 static struct item *ordered_depends(const char *bpath, const char *npath);
  95 static struct item *lookupItem(const char *npath);
  96 static struct item *addItem(const char *npath);
  97 static void addDepn(struct item *item, struct item *xitem);
  98
  99 static void addBuild(struct item *item);
 100 static void runBuilds(const char *bpath);
 101 static struct item *waitRunning(int flags);
 102 static void processCompletion(struct item *item);
 103
 104 static const char *neednl(void);
 105 static void usage(void);
 106
 107 int NParallel = 1;
 108 int VerboseOpt;
 109 int NRunning;
 110 int NeedNL;
 111
 112 int
 113 main(int ac, char **av)
 114 {
 115         char *bpath;
 116         size_t blen;
 117         int ch;
 118
 119         while ((ch = getopt(ac, av, "j:v")) != -1) {
 120                 switch(ch) {
 121                 case 'j':
 122                         NParallel = strtol(optarg, NULL, 0);
 123                         break;
 124                 case 'v':
 125                         VerboseOpt = 1;
 126                         break;
 127                 default:
 128                         usage();
 129                         /* NOT REACHED */
 130                 }
 131         }
 132         ac -= optind;
 133         av += optind;
 134
 135         if (ac != 1) {
 136                 fprintf(stderr, "requires base directory as first argument\n");
 137                 exit(1);
 138         }
 139
 140         /*
 141          * Base dir
 142          */
 143         bpath = strdup(av[0]);
 144         blen = strlen(bpath);
 145         while (blen && bpath[blen-1] == '/')
 146                 --blen;
 147         bpath[blen] = 0;
 148
 149         /*
 150          * Do recursive directory scan
 151          */
 152         ordered_scan(bpath, bpath, strlen(bpath));
 153
 154         /*
 155          * Wait for all current builds to finish running, keep the pipeline
 156          * full until both the BuildList and RunList have been exhausted.
 157          */
 158         runBuilds(bpath);
 159         while (waitRunning(0) != NULL)
 160                 runBuilds(bpath);
 161         return(0);
 162 }
 163
 164 /*
 165  * Recursively scan the requested directory tree looking for pkgsrc
 166  * stuff to build.
 167  */
 168 static void
 169 ordered_scan(const char *bpath, const char *path, size_t blen)
 170 {
 171         DIR *dir;
 172         struct dirent *den;
 173         char *npath;
 174         char *xpath;
 175         struct stat st;
 176
 177         if ((dir = opendir(path)) != NULL) {
 178                 while ((den = readdir(dir)) != NULL) {
 179                         if (den->d_name[0] == '.')
 180                                 continue;
 181                         asprintf(&npath, "%s/%s", path, den->d_name);
 182                         asprintf(&xpath, "%s/DESCR", npath);
 183
 184                         if (lookupItem(npath + blen + 1) == NULL &&
 185                             stat(npath, &st) == 0 && S_ISDIR(st.st_mode)) {
 186                                 if (stat(xpath, &st) == 0) {
 187                                         ordered_depends(bpath,
 188                                                         npath + blen + 1);
 189                                 } else {
 190                                         ordered_scan(bpath, npath, blen);
 191                                 }
 192                         }
 193                         free(npath);
 194                         free(xpath);
 195                 }
 196                 closedir(dir);
 197         }
 198 }
 199
 200 /*
 201  * Recursively execute 'bmake show-depends-dirs' to calculate all required
 202  * dependencies.
 203  */
 204 static struct item *
 205 ordered_depends(const char *bpath, const char *npath)
 206 {
 207         struct item *item;
 208         struct item *xitem;
 209         char buf[1024];
 210         FILE *fp;
 211         char *cmd;
 212         int len;
 213
 214         item = addItem(npath);
 215
 216         /*
 217          * Retrieve and process dependencies recursively.  Note that
 218          * addDepn() can modify item's status.
 219          *
 220          * Guard the recursion by bumping dcount to prevent the item
 221          * from being processed for completion while we are still adding
 222          * its dependencies.  This would normally not occur but it can
 223          * if pkgsrc has a broken dependency loop.
 224          */
 225         ++item->dcount;
 226         asprintf(&cmd, "cd %s/%s; bmake show-depends-dirs", bpath, npath);
 227         fp = popen(cmd, "r");
 228         while (fgets(buf, sizeof(buf), fp) != NULL) {
 229                 len = strlen(buf);
 230                 if (len && buf[len-1] == '\n')
 231                         buf[--len] = 0;
 232                 xitem = lookupItem(buf);
 233                 if (xitem == NULL)
 234                         xitem = ordered_depends(bpath, buf);
 235                 addDepn(item, xitem);
 236         }
 237         pclose(fp);
 238         free(cmd);
 239         --item->dcount;
 240
 241         /*
 242          * If the item has no dependencies left either add it to the
 243          * build list or do completion processing (i.e. if some of the
 244          * dependencies failed).
 245          */
 246         if (item->dcount == 0) {
 247                 switch(item->status) {
 248                 case XWAITING:
 249                         addBuild(item);
 250                         break;
 251                 case XDEPFAIL:
 252                         processCompletion(item);
 253                         break;
 254                 default:
 255                         assert(0);
 256                         /* NOT REACHED */
 257                         break;
 258                 }
 259         } else {
 260                 if (VerboseOpt)
 261                         printf("Deferred   %s\n", item->rpath);
 262         }
 263         runBuilds(bpath);
 264         return (item);
 265 }
 266
 267 /*
 268  * Item hashing and dependency helper routines, called during the
 269  * directory scan.
 270  */
 271 static int
 272 itemhash(const char *npath)
 273 {
 274         int hv = 0xA1B5F342;
 275         int i;
 276
 277         for (i = 0; npath[i]; ++i)
 278                 hv = (hv << 5) ^ (hv >> 23) ^ npath[i];
 279         return(hv & ITHMASK);
 280 }
 281
 282 static struct item *
 283 lookupItem(const char *npath)
 284 {
 285         struct item *item;
 286
 287         for (item = ItemHash[itemhash(npath)]; item; item = item->hnext) {
 288                 if (strcmp(npath, item->rpath) == 0)
 289                         return(item);
 290         }
 291         return(NULL);
 292 }
 293
 294 static struct item *
 295 addItem(const char *npath)
 296 {
 297         struct item **itemp;
 298         struct item *item = calloc(sizeof(*item), 1);
 299         int i;
 300
 301         itemp = &ItemHash[itemhash(npath)];
 302         item->status = XWAITING;
 303         item->hnext = *itemp;
 304         item->rpath = strdup(npath);
 305         item->lpath = strdup(npath);
 306         *itemp = item;
 307         for (i = 0; item->lpath[i]; ++i) {
 308                 if (item->lpath[i] == '/')
 309                         item->lpath[i] = '.';
 310         }
 311
 312         return(item);
 313 }
 314
 315 /*
 316  * Add a reverse dependency from the deepest point (xitem) to the
 317  * packages that depend on xitem (item in this case).
 318  *
 319  * Caller will check dcount after it is through adding dependencies.
 320  */
 321 static void
 322 addDepn(struct item *item, struct item *xitem)
 323 {
 324         struct depn *depn = calloc(sizeof(*depn), 1);
 325         char *logpath3;
 326         FILE *fp;
 327
 328         depn->item = item;
 329         depn->dnext = xitem->dbase;
 330         xitem->dbase = depn;
 331         if (xitem->status == XDONE) {
 332                 if (xitem->xcode) {
 333                         assert(item->status == XWAITING ||
 334                                item->status == XDEPFAIL);
 335                         item->xcode = xitem->xcode;
 336                         item->status = XDEPFAIL;
 337                         asprintf(&logpath3,
 338                                  "/tmp/logs/bad/%s", item->lpath);
 339                         fp = fopen(logpath3, "a");
 340                         fprintf(fp, "Dependency failed: %s\n",
 341                                 xitem->rpath);
 342                         fclose(fp);
 343                         free(logpath3);
 344                 }
 345         } else {
 346                 ++item->dcount;
 347         }
 348 }
 349
 350 /*
 351  * Add the item to the build request list.  This routine is called
 352  * after all build dependencies have been satisfied for the item.
 353  * runBuilds() will pick items off of BuildList to keep the parallel
 354  * build pipeline full.
 355  */
 356 static void
 357 addBuild(struct item *item)
 358 {
 359         printf("%sBuildOrder %s\n", neednl(), item->rpath);
 360         *BuildListP = item;
 361         BuildListP = &item->bnext;
 362         item->status = XBUILD;
 363 }
 364
 365 /*
 366  * Start new builds from the build list and handle build completions,
 367  * which can potentialy add new items to the build list.
 368  *
 369  * This routine will maintain up to NParallel builds.  A new build is
 370  * only started once its dependencies have completed successfully so
 371  * when the bulk build starts it typically takes a little while before
 372  * fastbulk can keep the parallel pipeline full.
 373  */
 374 static void
 375 runBuilds(const char *bpath)
 376 {
 377         struct rlimit rlm;
 378         struct item *item;
 379         char *logpath;
 380         FILE *fp;
 381         int fd;
 382
 383         /*
 384          * Try to maintain up to NParallel builds
 385          */
 386         while (NRunning < NParallel && BuildList) {
 387                 item = BuildList;
 388                 if ((BuildList = item->bnext) == NULL)
 389                         BuildListP = &BuildList;
 390                 printf("%sBuildStart %s\n", neednl(), item->rpath);
 391
 392                 /*
 393                  * When [re]running a build remove any bad log from prior
 394                  * attempts.
 395                  */
 396                 asprintf(&logpath, "/tmp/logs/bad/%s", item->lpath);
 397                 remove(logpath);
 398                 free(logpath);
 399
 400                 asprintf(&logpath, "/tmp/logs/run/%s", item->lpath);
 401
 402                 item->status = XRUN;
 403
 404                 item->pid = fork();
 405                 if (item->pid == 0) {
 406                         /*
 407                          * Child process - setup the log file and exec
 408                          */
 409                         if (chdir(bpath) < 0)
 410                                 _exit(99);
 411                         if (chdir(item->rpath) < 0)
 412                                 _exit(99);
 413
 414                         /*
 415                          * Connect log file up, disconnect tty (in case a
 416                          * 'patch' command tries to ask for help).
 417                          */
 418                         fd = open(logpath, O_RDWR|O_CREAT|O_TRUNC, 0666);
 419                         if (fd != 1)
 420                                 dup2(fd, 1);
 421                         if (fd != 2)
 422                                 dup2(fd, 2);
 423                         if (fd != 1 && fd != 2)
 424                                 close(fd);
 425                         fd = open("/dev/null", O_RDWR);
 426                         if (fd != 0) {
 427                                 dup2(fd, 0);
 428                                 close(fd);
 429                         }
 430
 431                         /*
 432                          * Set resource limits:
 433                          *
 434                          *      ~2 hours cpu    - prevent runaways from stalling
 435                          *                        the build.
 436                          *
 437                          *      ~2GB file size  - prevent endless growing log files.
 438                          *
 439                          *      ~5GB footprint  - prevent processes w/ out of
 440                          *                        control memory usage.
 441                          *
 442                          *      ~0 core         - No core dumps cluttering
 443                          *                        directories, please.
 444                          */
 445                         if (getrlimit(RLIMIT_CPU, &rlm) == 0 &&
 446                             rlm.rlim_cur > 2 * 60 * 60) {
 447                                 rlm.rlim_cur = 2 * 60 * 60;
 448                                 setrlimit(RLIMIT_CPU, &rlm);
 449                         }
 450                         if (getrlimit(RLIMIT_FSIZE, &rlm) == 0 &&
 451                             rlm.rlim_cur > 2LL * 1024 * 1024 * 1024) {
 452                                 rlm.rlim_cur = 2LL * 1024 * 1024 * 1024;
 453                                 setrlimit(RLIMIT_FSIZE, &rlm);
 454                         }
 455                         if (getrlimit(RLIMIT_AS, &rlm) == 0 &&
 456                             rlm.rlim_cur > 5LL * 1024 * 1024 * 1024) {
 457                                 rlm.rlim_cur = 5LL * 1024 * 1024 * 1024;
 458                                 setrlimit(RLIMIT_AS, &rlm);
 459                         }
 460                         if (getrlimit(RLIMIT_CORE, &rlm) == 0 &&
 461                             rlm.rlim_cur > 0) {
 462                                 rlm.rlim_cur = 0;
 463                                 setrlimit(RLIMIT_CORE, &rlm);
 464                         }
 465
 466                         /*
 467                          * Disconnect tty so utilities which try to ask
 468                          * for help (like patch) or Y/N answers on /dev/tty
 469                          * do not stall.
 470                          */
 471                         fd = open("/dev/tty", O_RDWR);
 472                         if (fd >= 0) {
 473                                 ioctl(fd, TIOCNOTTY, 0);
 474                                 close(fd);
 475                         }
 476                         setsid();
 477
 478                         /*
 479                          * we tack a 'clean' on to the repackage to clean
 480                          * the work directory on success.  If a failure
 481                          * occurs we leave the work directory intact.
 482                          *
 483                          * leaving work directories around when doing a
 484                          * bulk build can fill up the filesystem very fast.
 485                          */
 486                         execl("/tmp/track/dobuild", "dobuild",
 487                                 item->lpath, NULL);
 488                         _exit(99);
 489                 } else if (item->pid < 0) {
 490                         /*
 491                          * Parent fork() failed, log the problem and
 492                          * do completion processing.
 493                          */
 494                         item->xcode = -98;
 495                         fp = fopen(logpath, "a");
 496                         fprintf(fp, "fastbulk: Unable to fork/exec bmake\n");
 497                         fclose(fp);
 498                         processCompletion(item);
 499                 } else {
 500                         /*
 501                          * Parent is now tracking the running child,
 502                          * add the item to the RunList.
 503                          */
 504                         item->bnext = RunList;
 505                         RunList = item;
 506                         ++NRunning;
 507                 }
 508                 free(logpath);
 509         }
 510
 511         /*
 512          * Process any completed builds (non-blocking)
 513          */
 514         while (waitRunning(WNOHANG) != NULL)
 515                 ;
 516 }
 517
 518 /*
 519  * Wait for a running build to finish and process its completion.
 520  * Return the build or NULL if no builds are pending.
 521  *
 522  * The caller should call runBuilds() in the loop to keep the build
 523  * pipeline full until there is nothing left in the build list.
 524  */
 525 static struct item *
 526 waitRunning(int flags)
 527 {
 528         struct item *item;
 529         struct item **itemp;
 530         pid_t pid;
 531         int status;
 532
 533         if (RunList == NULL)
 534                 return(NULL);
 535         while ((pid = wait3(&status, flags, NULL)) < 0 && flags == 0)
 536                 ;
 537
 538         /*
 539          * NOTE! The pid may be associated with one of our popen()'s
 540          *       so just ignore it if we cannot find it.
 541          */
 542         if (pid > 0) {
 543                 status = WEXITSTATUS(status);
 544                 itemp = &RunList;
 545                 while ((item = *itemp) != NULL) {
 546                         if (item->pid == pid)
 547                                 break;
 548                         itemp = &item->bnext;
 549                 }
 550                 if (item) {
 551                         *itemp = item->bnext;
 552                         item->bnext = NULL;
 553                         item->xcode = status;
 554                         --NRunning;
 555                         processCompletion(item);
 556                 }
 557         } else {
 558                 item = NULL;
 559         }
 560         return (item);
 561 }
 562
 563 /*
 564  * Process the build completion for an item.
 565  */
 566 static void
 567 processCompletion(struct item *item)
 568 {
 569         struct depn *depn;
 570         struct item *xitem;
 571         char *logpath1;
 572         char *logpath2;
 573         char *logpath3;
 574         FILE *fp;
 575
 576         /*
 577          * If XRUN we have to move the logfile to the correct directory.
 578          * (If XDEPFAIL the logfile is already in the correct directory).
 579          */
 580         if (item->status == XRUN) {
 581                 asprintf(&logpath1, "/tmp/logs/run/%s", item->lpath);
 582                 asprintf(&logpath2, "/tmp/logs/%s/%s",
 583                          (item->xcode ? "bad" : "good"), item->lpath);
 584                 rename(logpath1, logpath2);
 585                 free(logpath1);
 586                 free(logpath2);
 587         }
 588
 589         printf("%sFinish %-3d %s\n", neednl(), item->xcode, item->rpath);
 590         assert(item->status == XRUN || item->status == XDEPFAIL);
 591         item->status = XDONE;
 592         for (depn = item->dbase; depn; depn = depn->dnext) {
 593                 xitem = depn->item;
 594                 assert(xitem->dcount > 0);
 595                 --xitem->dcount;
 596                 if (xitem->status == XWAITING || xitem->status == XDEPFAIL) {
 597                         /*
 598                          * If our build went well add items dependent
 599                          * on us to the build, otherwise fail the items
 600                          * dependent on us.
 601                          */
 602                         if (item->xcode) {
 603                                 xitem->xcode = item->xcode;
 604                                 xitem->status = XDEPFAIL;
 605                                 asprintf(&logpath3,
 606                                          "/tmp/logs/bad/%s", xitem->lpath);
 607                                 fp = fopen(logpath3, "a");
 608                                 fprintf(fp, "Dependency failed: %s\n",
 609                                         item->rpath);
 610                                 fclose(fp);
 611                                 free(logpath3);
 612                         }
 613                         if (xitem->dcount == 0) {
 614                                 if (xitem->status == XWAITING)
 615                                         addBuild(xitem);
 616                                 else
 617                                         processCompletion(xitem);
 618                         }
 619                 } else if (xitem->status == XDONE && xitem->xcode) {
 620                         /*
 621                          * The package depending on us has already run
 622                          * (this case should not occur).
 623                          *
 624                          * Add this dependency failure to its log file
 625                          * (which has already been renamed).
 626                          */
 627                         asprintf(&logpath3,
 628                                  "/tmp/logs/bad/%s", xitem->lpath);
 629                         fp = fopen(logpath3, "a");
 630                         fprintf(fp, "Dependency failed: %s\n",
 631                                 item->rpath);
 632                         fclose(fp);
 633                         free(logpath3);
 634                 }
 635         }
 636 }
 637
 638 static const char *
 639 neednl(void)
 640 {
 641         if (NeedNL) {
 642                 NeedNL = 0;
 643                 return("\n");
 644         } else {
 645                 return("");
 646         }
 647 }
 648
 649 static void
 650 usage(void)
 651 {
 652         fprintf(stderr, "fastbulk [-j parallel] /usr/pkgsrc\n");
 653         exit(1);
 654 }