usr.bin/dsynth/build.c

   1 /*
   2  * Copyright (c) 2019 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * This code uses concepts and configuration based on 'synth', by
   8  * John R. Marino <draco@marino.st>, which was written in ada.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  *
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in
  18  *    the documentation and/or other materials provided with the
  19  *    distribution.
  20  * 3. Neither the name of The DragonFly Project nor the names of its
  21  *    contributors may be used to endorse or promote products derived
  22  *    from this software without specific, prior written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  25  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  26  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  27  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  28  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  29  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  30  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  31  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  32  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  33  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  34  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  35  * SUCH DAMAGE.
  36  */
  37 #include "dsynth.h"
  38
  39 worker_t WorkerAry[MAXWORKERS];
  40 int BuildInitialized;
  41 int RunningWorkers;
  42 int DynamicMaxWorkers;
  43 int FailedWorkers;
  44 long RunningPkgDepSize;
  45 pthread_mutex_t WorkerMutex;
  46 pthread_cond_t WorkerCond;
  47
  48 static int build_find_leaves(pkg_t *parent, pkg_t *pkg,
  49                         pkg_t ***build_tailp, int *app, int *hasworkp,
  50                         int depth, int first, int first_one_only);
  51 static int buildCalculateDepiDepth(pkg_t *pkg);
  52 static void build_clear_trav(pkg_t *pkg);
  53 static void startbuild(pkg_t **build_listp, pkg_t ***build_tailp);
  54 static int qsort_depi(const void *pkg1, const void *pkg2);
  55 static int qsort_idep(const void *pkg1, const void *pkg2);
  56 static void startworker(pkg_t *pkg, worker_t *work);
  57 static void cleanworker(worker_t *work);
  58 static void waitbuild(int whilematch, int dynamicmax);
  59 static void workercomplete(worker_t *work);
  60 static void *childBuilderThread(void *arg);
  61 static int childInstallPkgDeps(worker_t *work);
  62 static size_t childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list,
  63                         int undoit, int depth, int first_one_only);
  64 static void dophase(worker_t *work, wmsg_t *wmsg,
  65                         int wdog, int phaseid, const char *phase);
  66 static void phaseReapAll(void);
  67 static void phaseTerminateSignal(int sig);
  68 static char *buildskipreason(pkglink_t *parent, pkg_t *pkg);
  69 static int buildskipcount_dueto(pkg_t *pkg, int mode);
  70 static int mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg,
  71                         time_t *wdog_timep);
  72 static void doHook(pkg_t *pkg, const char *id, const char *path, int waitfor);
  73 static void childHookRun(bulk_t *bulk);
  74 static void adjloadavg(double *dload);
  75
  76 static worker_t *SigWork;
  77 static int MasterPtyFd = -1;
  78 static int CopyFileFd = -1;
  79 static pid_t SigPid;
  80 static const char *WorkerFlavorPrt = "";        /* "" or "@flavor" */
  81
  82 #define MPTY_FAILED     -2
  83 #define MPTY_AGAIN      -1
  84 #define MPTY_EOF        0
  85 #define MPTY_DATA       1
  86
  87 int BuildTotal;
  88 int BuildCount;
  89 int BuildSkipCount;
  90 int BuildIgnoreCount;
  91 int BuildFailCount;
  92 int BuildSuccessCount;
  93
  94 /*
  95  * Initialize the WorkerAry[]
  96  */
  97 void
  98 DoInitBuild(int slot_override)
  99 {
 100         worker_t *work;
 101         struct stat st;
 102         int i;
 103
 104         ddassert(slot_override < 0 || MaxWorkers == 1);
 105
 106         bzero(WorkerAry, MaxWorkers * sizeof(worker_t));
 107         pthread_mutex_init(&WorkerMutex, NULL);
 108
 109         for (i = 0; i < MaxWorkers; ++i) {
 110                 work = &WorkerAry[i];
 111                 work->index = (slot_override >= 0) ? slot_override : i;
 112                 work->state = WORKER_NONE;
 113                 asprintf(&work->basedir, "%s/SL%02d", BuildBase, work->index);
 114                 pthread_cond_init(&work->cond, NULL);
 115         }
 116         BuildCount = 0;
 117
 118         /*
 119          * Create required sub-directories. The base directories must already
 120          * exist as a dsynth configuration safety.
 121          */
 122         if (stat(RepositoryPath, &st) < 0) {
 123                 if (mkdir(RepositoryPath, 0755) < 0)
 124                         dfatal("Cannot mkdir %s\n", RepositoryPath);
 125         }
 126
 127         BuildInitialized = 1;
 128
 129         /*
 130          * slow-start (increases at a rate of 1 per 5 seconds)
 131          */
 132         if (SlowStartOpt > MaxWorkers)
 133                 DynamicMaxWorkers = MaxWorkers;
 134         else if (SlowStartOpt > 0)
 135                 DynamicMaxWorkers = SlowStartOpt;
 136         else
 137                 DynamicMaxWorkers = MaxWorkers;
 138 }
 139
 140 /*
 141  * Called by the frontend to clean-up any hanging mounts.
 142  */
 143 void
 144 DoCleanBuild(int resetlogs)
 145 {
 146         int i;
 147
 148         ddassert(BuildInitialized);
 149
 150         if (resetlogs)
 151                 dlogreset();
 152         for (i = 0; i < MaxWorkers; ++i) {
 153                 DoWorkerUnmounts(&WorkerAry[i]);
 154         }
 155 }
 156
 157 void
 158 DoBuild(pkg_t *pkgs)
 159 {
 160         pkg_t *build_list = NULL;
 161         pkg_t **build_tail = &build_list;
 162         pkg_t *scan;
 163         bulk_t *bulk;
 164         int haswork = 1;
 165         int first = 1;
 166         int newtemplate;
 167         time_t startTime;
 168         time_t t;
 169         int h, m, s;
 170
 171         /*
 172          * We use our bulk system to run hooks.  This will be for
 173          * Skipped and Ignored.  Success and Failure are handled by
 174          * WorkerProcess() which is a separately-exec'd dsynth.
 175          */
 176         if (UsingHooks)
 177                 initbulk(childHookRun, MaxBulk);
 178
 179         /*
 180          * Count up the packages, not counting dummies
 181          */
 182         for (scan = pkgs; scan; scan = scan->bnext) {
 183                 if ((scan->flags & PKGF_DUMMY) == 0)
 184                         ++BuildTotal;
 185         }
 186
 187         doHook(NULL, "hook_run_start", HookRunStart, 1);
 188
 189         /*
 190          * The pkg and pkg-static binaries are needed.  If already present
 191          * then assume that the template is also valid, otherwise build
 192          * both.
 193          */
 194         scan = GetPkgPkg(pkgs);
 195
 196         /*
 197          * Create our template.  The template will be missing pkg
 198          * and pkg-static.
 199          */
 200         if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0) {
 201                 /* force a fresh template */
 202                 newtemplate = DoCreateTemplate(1);
 203         } else {
 204                 newtemplate = DoCreateTemplate(0);
 205         }
 206
 207         /*
 208          * This will clear the screen and set-up our gui, so sleep
 209          * a little first in case the user wants to see what was
 210          * printed before.
 211          */
 212         sleep(2);
 213         pthread_mutex_lock(&WorkerMutex);
 214         startTime = time(NULL);
 215         RunStatsInit();
 216         RunStatsReset();
 217
 218         /*
 219          * Build pkg/pkg-static.
 220          */
 221         if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0) {
 222                 build_list = scan;
 223                 build_tail = &scan->build_next;
 224                 startbuild(&build_list, &build_tail);
 225                 while (RunningWorkers == 1)
 226                         waitbuild(1, 0);
 227
 228                 if (scan->flags & PKGF_NOBUILD)
 229                         dfatal("Unable to build 'pkg'");
 230                 if (scan->flags & PKGF_ERROR)
 231                         dfatal("Error building 'pkg'");
 232                 if ((scan->flags & PKGF_SUCCESS) == 0)
 233                         dfatal("Error building 'pkg'");
 234                 newtemplate = 1;
 235         }
 236
 237         /*
 238          * Install pkg/pkg-static into the template
 239          */
 240         if (newtemplate) {
 241                 char *buf;
 242                 int rc;
 243
 244                 asprintf(&buf,
 245                          "cd %s/Template; "
 246                          "tar --exclude '+*' --exclude '*/man/*' "
 247                          "-xvzpf %s/%s > /dev/null 2>&1",
 248                          BuildBase,
 249                          RepositoryPath,
 250                          scan->pkgfile);
 251                 rc = system(buf);
 252                 if (rc)
 253                         dfatal("Command failed: %s\n", buf);
 254                 freestrp(&buf);
 255         }
 256
 257         /*
 258          * Calculate depi_depth, the longest chain of dependencies
 259          * for who depends on me, weighted by powers of two.
 260          */
 261         for (scan = pkgs; scan; scan = scan->bnext) {
 262                 buildCalculateDepiDepth(scan);
 263         }
 264
 265         /*
 266          * Nominal bulk build sequence
 267          */
 268         while (haswork) {
 269                 haswork = 0;
 270                 fflush(stdout);
 271                 for (scan = pkgs; scan; scan = scan->bnext) {
 272                         ddprintf(0, "SCANLEAVES %08x %s\n",
 273                                  scan->flags, scan->portdir);
 274                         scan->flags |= PKGF_BUILDLOOP;
 275                         /*
 276                          * NOTE: We must still find dependencies if PACKAGED
 277                          *       to fill in the gaps, as some of them may
 278                          *       need to be rebuilt.
 279                          */
 280                         if (scan->flags & (PKGF_SUCCESS | PKGF_FAILURE |
 281                                            PKGF_ERROR)) {
 282 #if 0
 283                                 ddprintf(0, "%s: already built\n",
 284                                          scan->portdir);
 285 #endif
 286                         } else {
 287                                 int ap = 0;
 288                                 build_find_leaves(NULL, scan, &build_tail,
 289                                                   &ap, &haswork, 0, first, 0);
 290                                 ddprintf(0, "TOPLEVEL %s %08x\n",
 291                                          scan->portdir, ap);
 292                         }
 293                         scan->flags &= ~PKGF_BUILDLOOP;
 294                         build_clear_trav(scan);
 295                 }
 296                 first = 0;
 297                 fflush(stdout);
 298                 startbuild(&build_list, &build_tail);
 299
 300                 if (haswork == 0 && RunningWorkers) {
 301                         waitbuild(RunningWorkers, 1);
 302                         haswork = 1;
 303                 }
 304         }
 305         pthread_mutex_unlock(&WorkerMutex);
 306
 307         RunStatsUpdateTop(0);
 308         RunStatsUpdateLogs();
 309         RunStatsSync();
 310         RunStatsDone();
 311
 312         doHook(NULL, "hook_run_end", HookRunEnd, 1);
 313         if (UsingHooks) {
 314                 while ((bulk = getbulk()) != NULL)
 315                         freebulk(bulk);
 316                 donebulk();
 317         }
 318
 319         t = time(NULL) - startTime;
 320         h = t / 3600;
 321         m = t / 60 % 60;
 322         s = t % 60;
 323
 324         dlog(DLOG_ALL|DLOG_STDOUT,
 325                 "\n"
 326                 "Initial queue size: %d\n"
 327                 "    packages built: %d\n"
 328                 "           ignored: %d\n"
 329                 "           skipped: %d\n"
 330                 "            failed: %d\n"
 331                 "\n"
 332                 "Duration: %02d:%02d:%02d\n"
 333                 "\n",
 334                 BuildTotal,
 335                 BuildSuccessCount,
 336                 BuildIgnoreCount,
 337                 BuildSkipCount,
 338                 BuildFailCount,
 339                 h, m, s);
 340 }
 341
 342 /*
 343  * Traverse the packages (pkg) depends on recursively until we find
 344  * a leaf to build or report as unbuildable.  Calculates and assigns a
 345  * dependency count.  Returns all parallel-buildable packages.
 346  *
 347  * (pkg) itself is only added to the list if it is immediately buildable.
 348  */
 349 static
 350 int
 351 build_find_leaves(pkg_t *parent, pkg_t *pkg, pkg_t ***build_tailp,
 352                   int *app, int *hasworkp, int depth, int first,
 353                   int first_one_only)
 354 {
 355         pkglink_t *link;
 356         pkg_t *scan;
 357         int idep_count = 0;
 358         int apsub;
 359         int dfirst_one_only;
 360         int ndepth;
 361         char *buf;
 362
 363         ndepth = depth + 1;
 364
 365         /*
 366          * Already on build list, possibly in-progress, tell caller that
 367          * it is not ready.
 368          */
 369         ddprintf(ndepth, "sbuild_find_leaves %d %s %08x {\n",
 370                  depth, pkg->portdir, pkg->flags);
 371         if (pkg->flags & PKGF_BUILDLIST) {
 372                 ddprintf(ndepth, "} (already on build list)\n");
 373                 *app |= PKGF_NOTREADY;
 374                 return (pkg->idep_count);
 375         }
 376
 377         /*
 378          * Check dependencies
 379          */
 380         PKGLIST_FOREACH(link, &pkg->idepon_list) {
 381                 scan = link->pkg;
 382
 383                 if (scan == NULL) {
 384                         if (first_one_only)
 385                                 break;
 386                         continue;
 387                 }
 388                 ddprintf(ndepth, "check %s %08x\t", scan->portdir, scan->flags);
 389
 390                 /*
 391                  * If this dependency is to a DUMMY node it is a dependency
 392                  * only on the default flavor which is only the first node
 393                  * under this one, not all of them.
 394                  *
 395                  * DUMMY nodes can be marked SUCCESS so the build skips past
 396                  * them, but this doesn't mean that their sub-nodes succeeded.
 397                  * We have to check, so recurse even if it is marked
 398                  * successful.
 399                  *
 400                  * NOTE: The depth is not being for complex dependency type
 401                  *       tests like it is in childInstallPkgDeps_recurse(),
 402                  *       so we don't have to hicup it like we do in that
 403                  *       procedure.
 404                  */
 405                 dfirst_one_only = (scan->flags & PKGF_DUMMY) ? 1 : 0;
 406                 if (dfirst_one_only)
 407                         goto skip_to_flavor;
 408
 409                 /*
 410                  * When accounting for a successful build, just bump
 411                  * idep_count by one.  scan->idep_count will heavily
 412                  * overlap packages that we count down multiple branches.
 413                  *
 414                  * We must still recurse through PACKAGED packages as
 415                  * some of their dependencies might be missing.
 416                  */
 417                 if (scan->flags & PKGF_SUCCESS) {
 418                         ddprintf(0, "SUCCESS - OK\n");
 419                         ++idep_count;
 420                         if (first_one_only)
 421                                 break;
 422                         continue;
 423                 }
 424
 425                 /*
 426                  * ERROR includes FAILURE, which is set in numerous situations
 427                  * including when NOBUILD state is processed.  So check for
 428                  * NOBUILD state first.
 429                  *
 430                  * An ERROR in a sub-package causes a NOBUILD in packages
 431                  * that depend on it.
 432                  */
 433                 if (scan->flags & PKGF_NOBUILD) {
 434                         ddprintf(0, "NOBUILD - OK "
 435                                     "(propagate failure upward)\n");
 436                         *app |= PKGF_NOBUILD_S;
 437                         if (first_one_only)
 438                                 break;
 439                         continue;
 440                 }
 441                 if (scan->flags & PKGF_ERROR) {
 442                         ddprintf(0, "ERROR - OK (propagate failure upward)\n");
 443                         *app |= PKGF_NOBUILD_S;
 444                         if (first_one_only)
 445                                 break;
 446                         continue;
 447                 }
 448
 449                 /*
 450                  * If already on build-list this dependency is not ready.
 451                  */
 452                 if (scan->flags & PKGF_BUILDLIST) {
 453                         ddprintf(0, " [BUILDLIST]");
 454                         *app |= PKGF_NOTREADY;
 455                 }
 456
 457                 /*
 458                  * If not packaged this dependency is not ready for
 459                  * the caller.
 460                  */
 461                 if ((scan->flags & PKGF_PACKAGED) == 0) {
 462                         ddprintf(0, " [NOT_PACKAGED]");
 463                         *app |= PKGF_NOTREADY;
 464                 }
 465
 466 skip_to_flavor:
 467                 /*
 468                  * Reduce search complexity, if we have already processed
 469                  * scan in the traversal it will either already be on the
 470                  * build list or it will not be buildable.  Either way
 471                  * the parent is not buildable.
 472                  */
 473                 if (scan->flags & PKGF_BUILDTRAV) {
 474                         ddprintf(0, " [BUILDTRAV]\n");
 475                         *app |= PKGF_NOTREADY;
 476                         if (first_one_only)
 477                                 break;
 478                         continue;
 479                 }
 480
 481                 /*
 482                  * Assert on dependency loop
 483                  */
 484                 ++idep_count;
 485                 if (scan->flags & PKGF_BUILDLOOP) {
 486                         dfatal("pkg dependency loop %s -> %s",
 487                                 parent->portdir, scan->portdir);
 488                 }
 489
 490                 /*
 491                  * NOTE: For debug tabbing purposes we use (ndepth + 1)
 492                  *       here (i.e. depth + 2) in our iteration.
 493                  */
 494                 scan->flags |= PKGF_BUILDLOOP;
 495                 apsub = 0;
 496                 ddprintf(0, " SUBRECURSION {\n");
 497                 idep_count += build_find_leaves(pkg, scan, build_tailp,
 498                                                 &apsub, hasworkp,
 499                                                 ndepth + 1, first,
 500                                                 dfirst_one_only);
 501                 scan->flags &= ~PKGF_BUILDLOOP;
 502                 *app |= apsub;
 503                 if (apsub & PKGF_NOBUILD) {
 504                         ddprintf(ndepth, "} (sub-nobuild)\n");
 505                 } else if (apsub & PKGF_ERROR) {
 506                         ddprintf(ndepth, "} (sub-error)\n");
 507                 } else if (apsub & PKGF_NOTREADY) {
 508                         ddprintf(ndepth, "} (sub-notready)\n");
 509                 } else {
 510                         ddprintf(ndepth, "} (sub-ok)\n");
 511                 }
 512                 if (first_one_only)
 513                         break;
 514         }
 515         pkg->idep_count = idep_count;
 516         pkg->flags |= PKGF_BUILDTRAV;
 517
 518         /*
 519          * Incorporate scan results into pkg state.
 520          */
 521         if ((pkg->flags & PKGF_NOBUILD) == 0 && (*app & PKGF_NOBUILD)) {
 522                 *hasworkp = 1;
 523         } else if ((pkg->flags & PKGF_ERROR) == 0 && (*app & PKGF_ERROR)) {
 524                 *hasworkp = 1;
 525         }
 526         pkg->flags |= *app & ~PKGF_NOTREADY;
 527
 528         /*
 529          * Clear PACKAGED bit if sub-dependencies aren't clean
 530          */
 531         if ((pkg->flags & PKGF_PACKAGED) &&
 532             (pkg->flags & (PKGF_NOTREADY|PKGF_ERROR|PKGF_NOBUILD))) {
 533                 pkg->flags &= ~PKGF_PACKAGED;
 534                 ddassert(pkg->pkgfile);
 535                 asprintf(&buf, "%s/%s", RepositoryPath, pkg->pkgfile);
 536                 if (remove(buf) < 0) {
 537                         dlog(DLOG_ALL,
 538                              "[XXX] %s DELETE-PACKAGE %s (failed)\n",
 539                              pkg->portdir, buf);
 540                 } else {
 541                         dlog(DLOG_ALL,
 542                              "[XXX] %s DELETE-PACKAGE %s "
 543                              "(due to dependencies)\n",
 544                              pkg->portdir, buf);
 545                 }
 546                 freestrp(&buf);
 547                 *hasworkp = 1;
 548         }
 549
 550         /*
 551          * Set PKGF_NOBUILD_I if there is IGNORE data
 552          */
 553         if (pkg->ignore)
 554                 pkg->flags |= PKGF_NOBUILD_I;
 555
 556         /*
 557          * Handle propagated flags
 558          */
 559         if (pkg->flags & PKGF_ERROR) {
 560                 /*
 561                  * This can only happen if the ERROR has already been
 562                  * processed and accounted for.
 563                  */
 564                 ddprintf(depth, "} (ERROR - %s)\n", pkg->portdir);
 565         } else if (*app & PKGF_NOTREADY) {
 566                 /*
 567                  * We don't set PKGF_NOTREADY in the pkg, it is strictly
 568                  * a transient flag propagated via build_find_leaves().
 569                  *
 570                  * Just don't add the package to the list.
 571                  *
 572                  * NOTE: Even if NOBUILD is set (meaning we could list it
 573                  *       and let startbuild() finish it up as a skip, we
 574                  *       don't process it to the list because we want to
 575                  *       process all the dependencies, so someone doing a
 576                  *       manual build can get more complete information and
 577                  *       does not have to iterate each failed dependency one
 578                  *       at a time.
 579                  */
 580                 ;
 581         } else if (pkg->flags & PKGF_SUCCESS) {
 582                 ddprintf(depth, "} (SUCCESS - %s)\n", pkg->portdir);
 583         } else if (pkg->flags & PKGF_DUMMY) {
 584                 /*
 585                  * Just mark dummy packages as successful when all of their
 586                  * sub-depends (flavors) complete successfully.  Note that
 587                  * dummy packages are not counted in the total, so do not
 588                  * decrement BuildTotal.
 589                  */
 590                 if (pkg->flags & PKGF_NOBUILD) {
 591                         ddprintf(depth, "} (DUMMY/META - IGNORED)\n");
 592                 } else {
 593                         ddprintf(depth, "} (DUMMY/META - SUCCESS)\n");
 594                         pkg->flags |= PKGF_SUCCESS;
 595                         *hasworkp = 1;
 596                         if (first) {
 597                                 dlog(DLOG_ALL | DLOG_FILTER,
 598                                      "[XXX] %s META-ALREADY-BUILT\n",
 599                                      pkg->portdir);
 600                         } else {
 601                                 dlog(DLOG_SUCC, "[XXX] %s meta-node complete\n",
 602                                      pkg->portdir);
 603                                 RunStatsUpdateCompletion(NULL, DLOG_SUCC, pkg, "");
 604                         }
 605                 }
 606         } else if (pkg->flags & PKGF_PACKAGED) {
 607                 /*
 608                  * We can just mark the pkg successful.  If this is
 609                  * the first pass, we count this as an initial pruning
 610                  * pass and reduce BuildTotal.
 611                  */
 612                 ddprintf(depth, "} (PACKAGED - SUCCESS)\n");
 613                 pkg->flags |= PKGF_SUCCESS;
 614                 *hasworkp = 1;
 615                 if (first) {
 616                         dlog(DLOG_ALL | DLOG_FILTER,
 617                              "[XXX] %s ALREADY-BUILT\n",
 618                              pkg->portdir);
 619                         --BuildTotal;
 620                 }
 621         } else {
 622                 /*
 623                  * All dependencies are successful, queue new work
 624                  * and indicate not-ready to the parent (since our
 625                  * package has to be built).
 626                  *
 627                  * NOTE: The NOBUILD case propagates to here as well
 628                  *       and is ultimately handled by startbuild().
 629                  */
 630                 *hasworkp = 1;
 631                 if (pkg->flags & PKGF_NOBUILD_I)
 632                         ddprintf(depth, "} (ADDLIST(IGNORE/BROKEN) - %s)\n",
 633                                  pkg->portdir);
 634                 else if (pkg->flags & PKGF_NOBUILD)
 635                         ddprintf(depth, "} (ADDLIST(NOBUILD) - %s)\n",
 636                                  pkg->portdir);
 637                 else
 638                         ddprintf(depth, "} (ADDLIST - %s)\n", pkg->portdir);
 639                 pkg->flags |= PKGF_BUILDLIST;
 640                 **build_tailp = pkg;
 641                 *build_tailp = &pkg->build_next;
 642                 *app |= PKGF_NOTREADY;
 643         }
 644
 645         return idep_count;
 646 }
 647
 648 static
 649 void
 650 build_clear_trav(pkg_t *pkg)
 651 {
 652         pkglink_t *link;
 653         pkg_t *scan;
 654
 655         pkg->flags &= ~PKGF_BUILDTRAV;
 656         PKGLIST_FOREACH(link, &pkg->idepon_list) {
 657                 scan = link->pkg;
 658                 if (scan && (scan->flags & PKGF_BUILDTRAV))
 659                         build_clear_trav(scan);
 660         }
 661 }
 662
 663 /*
 664  * Calculate the longest chain of packages that depend on me.  The
 665  * long the chain, the more important my package is to build earlier
 666  * rather than later.
 667  */
 668 static int
 669 buildCalculateDepiDepth(pkg_t *pkg)
 670 {
 671         pkglink_t *link;
 672         pkg_t *scan;
 673         int best_depth = 0;
 674         int res;
 675
 676         if (pkg->depi_depth)
 677                 return(pkg->depi_depth + 1);
 678         pkg->flags |= PKGF_BUILDLOOP;
 679         PKGLIST_FOREACH(link, &pkg->deponi_list) {
 680                 scan = link->pkg;
 681                 if (scan && (scan->flags & PKGF_BUILDLOOP) == 0) {
 682                         res = buildCalculateDepiDepth(scan);
 683                         if (best_depth < res)
 684                                 best_depth = res;
 685                 }
 686         }
 687         pkg->flags &= ~PKGF_BUILDLOOP;
 688         pkg->depi_depth = best_depth;
 689
 690         return (best_depth + 1);
 691 }
 692
 693 /*
 694  * Take a list of pkg ready to go, sort it, and assign it to worker
 695  * slots.  This routine blocks in waitbuild() until it can dispose of
 696  * the entire list.
 697  *
 698  * WorkerMutex is held by the caller.
 699  */
 700 static
 701 void
 702 startbuild(pkg_t **build_listp, pkg_t ***build_tailp)
 703 {
 704         pkg_t *pkg;
 705         pkg_t **idep_ary;
 706         pkg_t **depi_ary;
 707         int count;
 708         int idep_index;
 709         int depi_index;
 710         int i;
 711         int n;
 712         worker_t *work;
 713         static int IterateWorker;
 714
 715         /*
 716          * Nothing to do
 717          */
 718         if (*build_listp == NULL)
 719                 return;
 720
 721         /*
 722          * Sort
 723          */
 724         count = 0;
 725         for (pkg = *build_listp; pkg; pkg = pkg->build_next)
 726                 ++count;
 727         idep_ary = calloc(count, sizeof(pkg_t *));
 728         depi_ary = calloc(count, sizeof(pkg_t *));
 729
 730         count = 0;
 731         for (pkg = *build_listp; pkg; pkg = pkg->build_next) {
 732                 idep_ary[count] = pkg;
 733                 depi_ary[count] = pkg;
 734                 ++count;
 735         }
 736
 737         /*
 738          * idep_ary - sorted by #of dependencies this pkg has.
 739          * depi_ary - sorted by #of other packages that depend on this pkg.
 740          */
 741         qsort(idep_ary, count, sizeof(pkg_t *), qsort_idep);
 742         qsort(depi_ary, count, sizeof(pkg_t *), qsort_depi);
 743
 744         idep_index = 0;
 745         depi_index = 0;
 746
 747         /*
 748          * Half the workers build based on the highest depi count,
 749          * the other half build based on the highest idep count.
 750          *
 751          * This is an attempt to get projects which many other projects
 752          * depend on built first, but to also try to build large projects
 753          * (which tend to have a lot of dependencies) earlier rather than
 754          * later so the end of the bulk run doesn't inefficiently build
 755          * the last few huge projects.
 756          *
 757          * Loop until we manage to assign slots to everyone.  We do not
 758          * wait for build completion.
 759          *
 760          * This is the point where we handle DUMMY packages (these are
 761          * dummy unflavored packages which 'cover' all the flavors for
 762          * a package).  These are not real packages are marked SUCCESS
 763          * at this time because their dependencies (the flavors) have all
 764          * been built.
 765          */
 766         while (idep_index != count || depi_index != count) {
 767                 pkg_t *pkgi;
 768                 pkg_t *ipkg;
 769
 770                 /*
 771                  * Find candidate to start sorted by depi or idep.
 772                  */
 773                 ipkg = NULL;
 774                 while (idep_index < count) {
 775                         ipkg = idep_ary[idep_index];
 776                         if ((ipkg->flags &
 777                              (PKGF_SUCCESS | PKGF_FAILURE |
 778                               PKGF_ERROR | PKGF_RUNNING)) == 0) {
 779                                 break;
 780                         }
 781                         ipkg = NULL;
 782                         ++idep_index;
 783                 }
 784
 785                 pkgi = NULL;
 786                 while (depi_index < count) {
 787                         pkgi = depi_ary[depi_index];
 788                         if ((pkgi->flags &
 789                              (PKGF_SUCCESS | PKGF_FAILURE |
 790                               PKGF_ERROR | PKGF_RUNNING)) == 0) {
 791                                 break;
 792                         }
 793                         pkgi = NULL;
 794                         ++depi_index;
 795                 }
 796
 797                 /*
 798                  * ipkg and pkgi must either both be NULL, or both
 799                  * be non-NULL.
 800                  */
 801                 if (ipkg == NULL && pkgi == NULL)
 802                         break;
 803                 ddassert(ipkg && pkgi);
 804
 805                 /*
 806                  * Handle the NOBUILD case right here, there's no point
 807                  * queueing it anywhere.
 808                  */
 809                 if (ipkg->flags & PKGF_NOBUILD) {
 810                         char *reason;
 811
 812                         ipkg->flags |= PKGF_FAILURE;
 813                         ipkg->flags &= ~PKGF_BUILDLIST;
 814
 815                         reason = buildskipreason(NULL, ipkg);
 816                         if (ipkg->flags & PKGF_NOBUILD_I) {
 817                                 ++BuildIgnoreCount;
 818                                 dlog(DLOG_IGN, "[XXX] %s ignored due to %s\n",
 819                                      ipkg->portdir, reason);
 820                                 RunStatsUpdateCompletion(NULL, DLOG_IGN,
 821                                                          ipkg, reason);
 822                                 doHook(ipkg, "hook_pkg_ignored",
 823                                        HookPkgIgnored, 0);
 824                         } else {
 825                                 ++BuildSkipCount;
 826                                 dlog(DLOG_SKIP, "[XXX] %s skipped due to %s\n",
 827                                      ipkg->portdir, reason);
 828                                 RunStatsUpdateCompletion(NULL, DLOG_SKIP,
 829                                                          ipkg, reason);
 830                                 doHook(ipkg, "hook_pkg_skipped",
 831                                        HookPkgSkipped, 0);
 832                         }
 833                         free(reason);
 834                         ++BuildCount;
 835                         continue;
 836                 }
 837                 if (pkgi->flags & PKGF_NOBUILD) {
 838                         char *reason;
 839
 840                         pkgi->flags |= PKGF_FAILURE;
 841                         pkgi->flags &= ~PKGF_BUILDLIST;
 842
 843                         reason = buildskipreason(NULL, pkgi);
 844                         if (pkgi->flags & PKGF_NOBUILD_I) {
 845                                 ++BuildIgnoreCount;
 846                                 dlog(DLOG_IGN, "[XXX] %s ignored due to %s\n",
 847                                      pkgi->portdir, reason);
 848                                 RunStatsUpdateCompletion(NULL, DLOG_IGN,
 849                                                          pkgi, reason);
 850                                 doHook(pkgi, "hook_pkg_ignored",
 851                                        HookPkgIgnored, 0);
 852                         } else {
 853                                 ++BuildSkipCount;
 854                                 dlog(DLOG_SKIP, "[XXX] %s skipped due to %s\n",
 855                                      pkgi->portdir, reason);
 856                                 RunStatsUpdateCompletion(NULL, DLOG_SKIP,
 857                                                          pkgi, reason);
 858                                 doHook(pkgi, "hook_pkg_skipped",
 859                                        HookPkgSkipped, 0);
 860                         }
 861                         free(reason);
 862                         ++BuildCount;
 863                         continue;
 864                 }
 865
 866                 /*
 867                  * Block while no slots are available.  waitbuild()
 868                  * will clean out any DONE states.
 869                  */
 870                 while (RunningWorkers >= DynamicMaxWorkers ||
 871                        RunningWorkers >= MaxWorkers - FailedWorkers) {
 872                         waitbuild(RunningWorkers, 1);
 873                 }
 874
 875                 /*
 876                  * Find an available worker slot, there should be at
 877                  * least one.
 878                  */
 879                 for (i = 0; i < MaxWorkers; ++i) {
 880                         n = IterateWorker % MaxWorkers;
 881                         work = &WorkerAry[n];
 882
 883                         if (work->state == WORKER_DONE ||
 884                             work->state == WORKER_FAILED) {
 885                                 workercomplete(work);
 886                         }
 887                         if (work->state == WORKER_NONE ||
 888                             work->state == WORKER_IDLE) {
 889                                 if (n <= MaxWorkers / 2) {
 890                                         startworker(pkgi, work);
 891                                 } else {
 892                                         startworker(ipkg, work);
 893                                 }
 894                                 /*RunStatsUpdate(work);*/
 895                                 break;
 896                         }
 897                         ++IterateWorker;
 898                 }
 899                 ddassert(i != MaxWorkers);
 900         }
 901         RunStatsSync();
 902
 903         /*
 904          * We disposed of the whole list
 905          */
 906         free(idep_ary);
 907         free(depi_ary);
 908         *build_listp = NULL;
 909         *build_tailp = build_listp;
 910 }
 911
 912 typedef const pkg_t *pkg_tt;
 913
 914 static int
 915 qsort_idep(const void *pkg1_arg, const void *pkg2_arg)
 916 {
 917         const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg;
 918         const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg;
 919
 920         return (pkg2->idep_count - pkg1->idep_count);
 921 }
 922
 923 static int
 924 qsort_depi(const void *pkg1_arg, const void *pkg2_arg)
 925 {
 926         const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg;
 927         const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg;
 928
 929         return ((pkg2->depi_count * pkg2->depi_depth) -
 930                 (pkg1->depi_count * pkg1->depi_depth));
 931 }
 932
 933 /*
 934  * Frontend starts a pkg up on a worker
 935  *
 936  * WorkerMutex must be held.
 937  */
 938 static void
 939 startworker(pkg_t *pkg, worker_t *work)
 940 {
 941         switch(work->state) {
 942         case WORKER_NONE:
 943                 pthread_create(&work->td, NULL, childBuilderThread, work);
 944                 work->state = WORKER_IDLE;
 945                 /* fall through */
 946         case WORKER_IDLE:
 947                 work->pkg_dep_size =
 948                 childInstallPkgDeps_recurse(NULL, &pkg->idepon_list, 0, 1, 0);
 949                 childInstallPkgDeps_recurse(NULL, &pkg->idepon_list, 1, 1, 0);
 950                 RunningPkgDepSize += work->pkg_dep_size;
 951
 952                 dlog(DLOG_ALL, "[%03d] START   %s "
 953                                "##idep=%02d depi=%02d/%02d dep=%-4.2fG\n",
 954                      work->index, pkg->portdir,
 955                      pkg->idep_count, pkg->depi_count, pkg->depi_depth,
 956                      (double)work->pkg_dep_size / (double)ONEGB);
 957
 958                 cleanworker(work);
 959                 pkg->flags |= PKGF_RUNNING;
 960                 work->pkg = pkg;
 961                 pthread_cond_signal(&work->cond);
 962                 ++RunningWorkers;
 963                 /*RunStatsUpdate(work);*/
 964                 break;
 965         case WORKER_PENDING:
 966         case WORKER_RUNNING:
 967         case WORKER_DONE:
 968         case WORKER_FAILED:
 969         case WORKER_FROZEN:
 970         case WORKER_EXITING:
 971         default:
 972                 dfatal("startworker: [%03d] Unexpected state %d for worker %d",
 973                        work->index, work->state, work->index);
 974                 break;
 975         }
 976 }
 977
 978 static void
 979 cleanworker(worker_t *work)
 980 {
 981         work->state = WORKER_PENDING;
 982         work->flags = 0;
 983         work->accum_error = 0;
 984         work->start_time = time(NULL);
 985 }
 986
 987 /*
 988  * Frontend finishes up a completed pkg on a worker.
 989  *
 990  * If the worker is in a FAILED state we clean the pkg out but (for now)
 991  * leave it in its failed state so we can debug.  At this point
 992  * workercomplete() will be called every once in a while on the state
 993  * and we have to deal with the NULL pkg.
 994  *
 995  * WorkerMutex must be held.
 996  */
 997 static void
 998 workercomplete(worker_t *work)
 999 {
1000         pkg_t *pkg;
1001         time_t t;
1002         int h;
1003         int m;
1004         int s;
1005
1006         /*
1007          * Steady state FAILED case.
1008          */
1009         if (work->state == WORKER_FAILED) {
1010                 if (work->pkg == NULL)
1011                         return;
1012         }
1013
1014         t = time(NULL) - work->start_time;
1015         h = t / 3600;
1016         m = t / 60 % 60;
1017         s = t % 60;
1018
1019         /*
1020          * Reduce total dep size
1021          */
1022         RunningPkgDepSize -= work->pkg_dep_size;
1023         RunningPkgDepSize -= work->memuse;
1024         work->pkg_dep_size = 0;
1025         work->memuse = 0;
1026
1027         /*
1028          * Process pkg out of the worker
1029          */
1030         pkg = work->pkg;
1031         if (pkg->flags & (PKGF_ERROR|PKGF_NOBUILD)) {
1032                 pkg->flags |= PKGF_FAILURE;
1033
1034                 /*
1035                  * This NOBUILD condition XXX can occur if the package is
1036                  * not allowed to be built.
1037                  */
1038                 if (pkg->flags & PKGF_NOBUILD) {
1039                         char *reason;
1040
1041                         reason = buildskipreason(NULL, pkg);
1042                         if (pkg->flags & PKGF_NOBUILD_I) {
1043                                 ++BuildIgnoreCount;
1044                                 dlog(DLOG_SKIP, "[%03d] IGNORD %s - %s\n",
1045                                      work->index, pkg->portdir, reason);
1046                                 RunStatsUpdateCompletion(work, DLOG_SKIP,
1047                                                          pkg, reason);
1048                                 doHook(pkg, "hook_pkg_ignored",
1049                                        HookPkgIgnored, 0);
1050                         } else {
1051                                 ++BuildSkipCount;
1052                                 dlog(DLOG_SKIP, "[%03d] SKIPPD %s - %s\n",
1053                                      work->index, pkg->portdir, reason);
1054                                 RunStatsUpdateCompletion(work, DLOG_SKIP,
1055                                                          pkg, reason);
1056                                 doHook(pkg, "hook_pkg_skipped",
1057                                        HookPkgSkipped, 0);
1058                         }
1059                         free(reason);
1060                 } else {
1061                         char skipbuf[16];
1062                         int scount;
1063
1064                         scount = buildskipcount_dueto(pkg, 1);
1065                         buildskipcount_dueto(pkg, 0);
1066                         if (scount)
1067                                 snprintf(skipbuf, sizeof(skipbuf), " (%d)", scount);
1068                         else
1069                                 skipbuf[0] = 0;
1070
1071                         ++BuildFailCount;
1072                         dlog(DLOG_FAIL | DLOG_RED,
1073                              "[%03d] FAILURE %s%s ##%16.16s %02d:%02d:%02d\n",
1074                              work->index, pkg->portdir, skipbuf,
1075                              getphasestr(work->phase),
1076                              h, m, s);
1077                         RunStatsUpdateCompletion(work, DLOG_FAIL, pkg, skipbuf);
1078                         doHook(pkg, "hook_pkg_failure", HookPkgFailure, 0);
1079                 }
1080         } else {
1081                 pkg->flags |= PKGF_SUCCESS;
1082                 ++BuildSuccessCount;
1083                 dlog(DLOG_SUCC | DLOG_GRN,
1084                      "[%03d] SUCCESS %s ##%02d:%02d:%02d\n",
1085                      work->index, pkg->portdir, h, m, s);
1086                 RunStatsUpdateCompletion(work, DLOG_SUCC, pkg, "");
1087                 doHook(pkg, "hook_pkg_success", HookPkgSuccess, 0);
1088         }
1089         ++BuildCount;
1090         pkg->flags &= ~PKGF_BUILDLIST;
1091         pkg->flags &= ~PKGF_RUNNING;
1092         work->pkg = NULL;
1093         --RunningWorkers;
1094
1095         if (work->state == WORKER_FAILED) {
1096                 dlog(DLOG_ALL, "[%03d] XXX/XXX WORKER IS IN A FAILED STATE\n",
1097                      work->index);
1098                 ++FailedWorkers;
1099         } else if (work->flags & WORKERF_FREEZE) {
1100                 dlog(DLOG_ALL, "[%03d] FROZEN(DEBUG) %s\n",
1101                      work->index, pkg->portdir);
1102                 work->state = WORKER_FROZEN;
1103         } else {
1104                 work->state = WORKER_IDLE;
1105         }
1106 }
1107
1108 /*
1109  * Wait for one or more workers to complete.
1110  *
1111  * WorkerMutex must be held.
1112  */
1113 static void
1114 waitbuild(int whilematch, int dynamicmax)
1115 {
1116         static time_t wblast_time;
1117         static time_t dmlast_time;
1118         struct timespec ts;
1119         worker_t *work;
1120         time_t t;
1121         int i;
1122
1123         if (whilematch == 0)
1124                 whilematch = 1;
1125
1126         while (RunningWorkers == whilematch) {
1127                 for (i = 0; i < MaxWorkers; ++i) {
1128                         work = &WorkerAry[i];
1129                         if (work->state == WORKER_DONE ||
1130                             work->state == WORKER_FAILED) {
1131                                 workercomplete(work);
1132                         } else {
1133                                 pthread_cond_signal(&work->cond);
1134                         }
1135                         RunStatsUpdate(work, NULL);
1136                 }
1137                 RunStatsUpdateTop(1);
1138                 RunStatsUpdateLogs();
1139                 RunStatsSync();
1140                 if (RunningWorkers == whilematch) {
1141                         clock_gettime(CLOCK_REALTIME, &ts);
1142                         ts.tv_sec += 1;
1143                         ts.tv_nsec = 0;
1144                         pthread_cond_timedwait(&WorkerCond, &WorkerMutex, &ts);
1145                 }
1146
1147                 /*
1148                  * Dynamically reduce MaxWorkers based on the load.  When
1149                  * the load exceeds 2 x ncpus we reduce the number of workers
1150                  * up to 75% of MaxWorkers @ (5 x ncpus) load.
1151                  *
1152                  * Dynamically reduce MaxWorkers based on swap use, starting
1153                  * at 10% swap and up to 75% of MaxWorkers at 40% swap.
1154                  *
1155                  * NOTE! Generally speaking this allows more workers to be
1156                  *       configured which helps in two ways.  First it allows
1157                  *       a higher build rate for smaller packages.  Second
1158                  *       it allows dsynth to ratchet-down the number of slots
1159                  *       when large packages are forcing the load up.
1160                  *
1161                  *       A high load doesn't hurt efficiency, but swap usage
1162                  *       due to loading the tmpfs in lots of worker slots up
1163                  *       with tons of pkg install's (pre-reqs for a build)
1164                  *       does.  Reducing the number of worker slots has a
1165                  *       huge beneficial effect on reducing swap use / paging.
1166                  */
1167                 t = time(NULL);
1168                 if (dynamicmax && (wblast_time == 0 ||
1169                                    (unsigned)(t - wblast_time) >= 5)) {
1170                         double min_load = 1.5 * NumCores;
1171                         double max_load = 5.0 * NumCores;
1172                         double min_swap = 0.10;
1173                         double max_swap = 0.40;
1174                         double dload[3];
1175                         double dswap;
1176                         int max1;
1177                         int max2;
1178                         int max3;
1179                         int max_sel;
1180                         int noswap;
1181
1182                         wblast_time = t;
1183
1184                         /*
1185                          * Cap based on load.  This is back-loaded.
1186                          */
1187                         getloadavg(dload, 3);
1188                         adjloadavg(dload);
1189                         if (dload[0] < min_load) {
1190                                 max1 = MaxWorkers;
1191                         } else if (dload[0] <= max_load) {
1192                                 max1 = MaxWorkers -
1193                                        MaxWorkers * 0.75 *
1194                                        (dload[0] - min_load) /
1195                                        (max_load - min_load);
1196                         } else {
1197                                 max1 = MaxWorkers * 25 / 100;
1198                         }
1199
1200                         /*
1201                          * Cap based on swap use.  This is back-loaded.
1202                          */
1203                         dswap = getswappct(&noswap);
1204                         if (dswap < min_swap) {
1205                                 max2 = MaxWorkers;
1206                         } else if (dswap <= max_swap) {
1207                                 max2 = MaxWorkers -
1208                                        MaxWorkers * 0.75 *
1209                                        (dswap - min_swap) /
1210                                        (max_swap - min_swap);
1211                         } else {
1212                                 max2 = MaxWorkers * 25 / 100;
1213                         }
1214
1215                         /*
1216                          * Cap based on aggregate pkg-dependency memory
1217                          * use installed in worker slots.  This is
1218                          * front-loaded.
1219                          *
1220                          * Since it can take a while for workers to retire
1221                          * (to reduce RunningPkgDepSize), just set our
1222                          * target 1 below the current run count to allow
1223                          * jobs to retire without being replaced with new
1224                          * jobs.
1225                          *
1226                          * In addition, in order to avoid a paging 'shock',
1227                          * We enforce a 30 second-per-increment slow-start
1228                          * once RunningPkgDepSize exceeds 1/2 the target.
1229                          */
1230                         if (RunningPkgDepSize > PkgDepMemoryTarget) {
1231                                 max3 = RunningWorkers - 1;
1232                         } else if (RunningPkgDepSize > PkgDepMemoryTarget / 2) {
1233                                 if (dmlast_time == 0 ||
1234                                     (unsigned)(t - dmlast_time) >= 30) {
1235                                         dmlast_time = t;
1236                                         max3 = RunningWorkers + 1;
1237                                 } else {
1238                                         max3 = RunningWorkers;
1239                                 }
1240                         } else {
1241                                 max3 = MaxWorkers;
1242                         }
1243
1244                         /*
1245                          * Priority reduction, convert to DynamicMaxWorkers
1246                          */
1247                         max_sel = max1;
1248                         if (max_sel > max2)
1249                                 max_sel = max2;
1250                         if (max_sel > max3)
1251                                 max_sel = max3;
1252
1253                         /*
1254                          * Restrict to allowed range, and also handle
1255                          * slow-start.
1256                          */
1257                         if (max_sel < 1)
1258                                 max_sel = 1;
1259                         if (max_sel > DynamicMaxWorkers + 1)
1260                                 max_sel = DynamicMaxWorkers + 1;
1261                         if (max_sel > MaxWorkers)
1262                                 max_sel = MaxWorkers;
1263
1264                         /*
1265                          * Stop waiting if DynamicMaxWorkers is going to
1266                          * increase.
1267                          */
1268                         if (DynamicMaxWorkers < max1)
1269                                 whilematch = -1;
1270
1271                         /*
1272                          * And adjust
1273                          */
1274                         if (DynamicMaxWorkers != max1) {
1275                                 dlog(DLOG_ALL | DLOG_FILTER,
1276                                      "[XXX] Load=%-6.2f(%2d) "
1277                                      "Swap=%-3.2f%%(%2d) "
1278                                      "Mem=%3.2fG(%2d) "
1279                                      "Adjust Workers %d->%d\n",
1280                                      dload[0], max1,
1281                                      dswap * 100.0, max2,
1282                                      RunningPkgDepSize / (double)ONEGB, max3,
1283                                      DynamicMaxWorkers, max_sel);
1284                                 DynamicMaxWorkers = max_sel;
1285                         }
1286                 }
1287         }
1288 }
1289
1290
1291 /*
1292  * Worker pthread (WorkerAry)
1293  *
1294  * This thread belongs to the dsynth master process and handled a worker slot.
1295  * (this_thread) -> WORKER fork/exec (WorkerPocess) -> (pty) -> sub-processes.
1296  */
1297 static void *
1298 childBuilderThread(void *arg)
1299 {
1300         char *envary[1] = { NULL };
1301         worker_t *work = arg;
1302         wmsg_t wmsg;
1303         pkg_t *pkg;
1304         pid_t pid;
1305         int status;
1306         int flags;
1307         volatile int dowait;
1308         char slotbuf[8];
1309         char fdbuf[8];
1310         char flagsbuf[16];
1311
1312         pthread_mutex_lock(&WorkerMutex);
1313         while (work->terminate == 0) {
1314                 dowait = 1;
1315
1316                 switch(work->state) {
1317                 case WORKER_IDLE:
1318                         break;
1319                 case WORKER_PENDING:
1320                         /*
1321                          * Fork the management process for the pkg operation
1322                          * on this worker slot.
1323                          *
1324                          * This process will set up the environment, do the
1325                          * mounts, will become the reaper, and will process
1326                          * pipe commands and handle chroot operations.
1327                          *
1328                          * NOTE: If SOCK_CLOEXEC is not supported WorkerMutex
1329                          *       is sufficient to interlock F_SETFD FD_CLOEXEC
1330                          *       operations.
1331                          */
1332                         ddassert(work->pkg);
1333                         if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC,
1334                                        PF_UNSPEC, work->fds)) {
1335                                 dfatal_errno("socketpair() during worker fork");
1336                         }
1337                         snprintf(slotbuf, sizeof(slotbuf), "%d", work->index);
1338                         snprintf(fdbuf, sizeof(fdbuf), "3");
1339
1340                         /*
1341                          * Pass global flags and add-in the DEBUGSTOP if
1342                          * the package is flagged for debugging.
1343                          */
1344                         flags = WorkerProcFlags;
1345                         if (work->pkg->flags & PKGF_DEBUGSTOP) {
1346                                 flags |= WORKER_PROC_DEBUGSTOP;
1347                         } else {
1348                                 flags &= ~WORKER_PROC_DEBUGSTOP;
1349                         }
1350                         snprintf(flagsbuf, sizeof(flagsbuf), "%d", flags);
1351
1352                         /*
1353                          * fds[0] - master
1354                          * fds[1] - slave
1355                          *
1356                          * We pass the salve descriptor in fd 3 and close all
1357                          * other descriptors for security.
1358                          */
1359                         pthread_mutex_unlock(&WorkerMutex);
1360                         pid = vfork();
1361                         if (pid == 0) {
1362                                 close(work->fds[0]);
1363                                 dup2(work->fds[1], 3);
1364                                 closefrom(4);
1365                                 fcntl(3, F_SETFD, 0);
1366                                 execle(DSynthExecPath, DSynthExecPath,
1367                                        "-p", Profile,
1368                                        "WORKER", slotbuf, fdbuf,
1369                                        work->pkg->portdir, work->pkg->pkgfile,
1370                                        flagsbuf,
1371                                        NULL, envary);
1372                                 write(2, "EXECLE FAILURE\n", 15);
1373                                 _exit(1);
1374                         }
1375                         pthread_mutex_lock(&WorkerMutex);
1376                         close(work->fds[1]);
1377                         work->phase = PHASE_PENDING;
1378                         work->lines = 0;
1379                         work->memuse = 0;
1380                         work->pid = pid;
1381                         work->state = WORKER_RUNNING;
1382                         /* fall through */
1383                 case WORKER_RUNNING:
1384                         /*
1385                          * Poll for status updates, if NULL is returned
1386                          * and status is non-zero, the communications link
1387                          * failed unexpectedly.
1388                          */
1389                         pkg = work->pkg;
1390                         pthread_mutex_unlock(&WorkerMutex);
1391                         status = ipcreadmsg(work->fds[0], &wmsg);
1392                         pthread_mutex_lock(&WorkerMutex);
1393
1394                         if (status == 0) {
1395                                 /*
1396                                  * Normal message, can include normal
1397                                  * termination which changes us over
1398                                  * to another state.
1399                                  */
1400                                 dowait = 0;
1401                                 switch(wmsg.cmd) {
1402                                 case WMSG_CMD_INSTALL_PKGS:
1403                                         wmsg.cmd = WMSG_RES_INSTALL_PKGS;
1404                                         wmsg.status = childInstallPkgDeps(work);
1405                                         pthread_mutex_unlock(&WorkerMutex);
1406                                         ipcwritemsg(work->fds[0], &wmsg);
1407                                         pthread_mutex_lock(&WorkerMutex);
1408                                         break;
1409                                 case WMSG_CMD_STATUS_UPDATE:
1410                                         work->phase = wmsg.phase;
1411                                         work->lines = wmsg.lines;
1412                                         if (work->memuse != wmsg.memuse) {
1413                                                 RunningPkgDepSize +=
1414                                                 wmsg.memuse - work->memuse;
1415                                                 work->memuse = wmsg.memuse;
1416                                         }
1417                                         break;
1418                                 case WMSG_CMD_SUCCESS:
1419                                         work->flags |= WORKERF_SUCCESS;
1420                                         break;
1421                                 case WMSG_CMD_FAILURE:
1422                                         work->flags |= WORKERF_FAILURE;
1423                                         break;
1424                                 case WMSG_CMD_FREEZEWORKER:
1425                                         work->flags |= WORKERF_FREEZE;
1426                                         break;
1427                                 default:
1428                                         break;
1429                                 }
1430                                 RunStatsUpdate(work, NULL);
1431                                 RunStatsSync();
1432                         } else {
1433                                 close(work->fds[0]);
1434                                 pthread_mutex_unlock(&WorkerMutex);
1435                                 while (waitpid(work->pid, &status, 0) < 0 &&
1436                                        errno == EINTR) {
1437                                         ;
1438                                 }
1439                                 pthread_mutex_lock(&WorkerMutex);
1440
1441                                 if (work->flags & WORKERF_SUCCESS) {
1442                                         pkg->flags |= PKGF_SUCCESS;
1443                                         work->state = WORKER_DONE;
1444                                 } else if (work->flags & WORKERF_FAILURE) {
1445                                         pkg->flags |= PKGF_FAILURE;
1446                                         work->state = WORKER_DONE;
1447                                 } else {
1448                                         pkg->flags |= PKGF_FAILURE;
1449                                         work->state = WORKER_FAILED;
1450                                 }
1451                                 work->flags |= WORKERF_STATUS_UPDATE;
1452                                 pthread_cond_signal(&WorkerCond);
1453                         }
1454                         break;
1455                 case WORKER_DONE:
1456                         /*
1457                          * pkg remains attached until frontend processes the
1458                          * completion.  The frontend will then set the state
1459                          * back to idle.
1460                          */
1461                         break;
1462                 case WORKER_FAILED:
1463                         /*
1464                          * A worker failure means that the worker did not
1465                          * send us a WMSG_CMD_SUCCESS or WMSG_CMD_FAILURE
1466                          * ipc before terminating.
1467                          *
1468                          * We just sit in this state until the front-end
1469                          * does something about it.
1470                          */
1471                         break;
1472                 case WORKER_FROZEN:
1473                         /*
1474                          * A worker getting frozen is debug-related.  We
1475                          * just sit in this state (likely forever).
1476                          */
1477                         break;
1478                 default:
1479                         dfatal("worker: [%03d] Unexpected state %d "
1480                                "for worker %d",
1481                                work->index, work->state, work->index);
1482                         /* NOT REACHED */
1483                         break;
1484                 }
1485
1486                 /*
1487                  * The dsynth frontend will poll us approximately once
1488                  * a second (its variable).
1489                  */
1490                 if (dowait)
1491                         pthread_cond_wait(&work->cond, &WorkerMutex);
1492         }
1493
1494         /*
1495          * Scrap the comm socket if running, this should cause the worker
1496          * process to kill its sub-programs and cleanup.
1497          */
1498         if (work->state == WORKER_RUNNING) {
1499                 pthread_mutex_unlock(&WorkerMutex);
1500                 close(work->fds[0]);
1501                 while (waitpid(work->pid, &status, 0) < 0 &&
1502                        errno == EINTR);
1503                 pthread_mutex_lock(&WorkerMutex);
1504         }
1505
1506         /*
1507          * Final handshake
1508          */
1509         work->state = WORKER_EXITING;
1510         pthread_cond_signal(&WorkerCond);
1511         pthread_mutex_unlock(&WorkerMutex);
1512
1513         return NULL;
1514 }
1515
1516 /*
1517  * Install all the binary packages (we have already built them) that
1518  * the current work package depends on, without duplicates, in a script
1519  * which will be run from within the specified work jail.
1520  *
1521  * Locked by WorkerMutex (global)
1522  */
1523 static int
1524 childInstallPkgDeps(worker_t *work)
1525 {
1526         char *buf;
1527         FILE *fp;
1528
1529         if (PKGLIST_EMPTY(&work->pkg->idepon_list))
1530                 return 0;
1531
1532         asprintf(&buf, "%s/tmp/dsynth_install_pkgs", work->basedir);
1533         fp = fopen(buf, "w");
1534         ddassert(fp != NULL);
1535         fprintf(fp, "#!/bin/sh\n");
1536         fprintf(fp, "#\n");
1537         fchmod(fileno(fp), 0755);
1538
1539         childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 0, 1, 0);
1540         childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 1, 1, 0);
1541         fprintf(fp, "\nexit 0\n");
1542         fclose(fp);
1543         freestrp(&buf);
1544
1545         return 1;
1546 }
1547
1548 /*
1549  * Recursive child install dependencies.
1550  *
1551  * first_one_only is only specified if the pkg the list comes from
1552  * is a generic unflavored package that has flavors, telling us to
1553  * dive the first flavor only.
1554  *
1555  * However, in nearly all cases this flag will now be zero because
1556  * this code now dives the first flavor when encountering a dummy node
1557  * and clears nfirst on success.  Hence if you are asking why 'nfirst'
1558  * is set to 1, and then zero, instead of just being removed entirely,
1559  * it is because there might still be an edge case here.
1560  */
1561 static size_t
1562 childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list, int undoit,
1563                             int depth, int first_one_only)
1564 {
1565         pkglink_t *link;
1566         pkg_t *pkg;
1567         size_t tot = 0;
1568         int ndepth;
1569         int nfirst;
1570
1571         PKGLIST_FOREACH(link, list) {
1572                 pkg = link->pkg;
1573
1574                 /*
1575                  * We don't want to mess up our depth test just below if
1576                  * a DUMMY node had to be inserted.  The nodes under the
1577                  * dummy node.
1578                  *
1579                  * The elements under a dummy node represent all the flabor,
1580                  * a dependency that directly references a dummy node only
1581                  * uses the first flavor (first_one_only / nfirst).
1582                  */
1583                 ndepth = (pkg->flags & PKGF_DUMMY) ? depth : depth + 1;
1584                 nfirst = (pkg->flags & PKGF_DUMMY) ? 1 : 0;
1585
1586                 /*
1587                  * We only need all packages for the top-level dependencies.
1588                  * The deeper ones only need DEP_TYPE_LIB and DEP_TYPE_RUN
1589                  * (types greater than DEP_TYPE_BUILD) since they are already
1590                  * built.
1591                  */
1592                 if (depth > 1 && link->dep_type <= DEP_TYPE_BUILD) {
1593                         if (first_one_only)
1594                                 break;
1595                         continue;
1596                 }
1597
1598                 /*
1599                  * If this is a dummy node with no package, the originator
1600                  * is requesting a flavored package.  We select the default
1601                  * flavor which we presume is the first one.
1602                  */
1603                 if (pkg->pkgfile == NULL && (pkg->flags & PKGF_DUMMY)) {
1604                         pkg_t *spkg = pkg->idepon_list.next->pkg;
1605
1606                         if (spkg) {
1607                                 if (fp) {
1608                                         fprintf(fp,
1609                                                 "echo 'UNFLAVORED %s -> use "
1610                                                 "%s'\n",
1611                                                 pkg->portdir,
1612                                                 spkg->portdir);
1613                                 }
1614                                 pkg = spkg;
1615                                 nfirst = 0;
1616                         } else {
1617                                 if (fp) {
1618                                         fprintf(fp,
1619                                                 "echo 'CANNOT FIND DEFAULT "
1620                                                 "FLAVOR FOR %s'\n",
1621                                                 pkg->portdir);
1622                                 }
1623                         }
1624                 }
1625
1626                 if (undoit) {
1627                         if (pkg->dsynth_install_flg == 1) {
1628                                 pkg->dsynth_install_flg = 0;
1629                                 tot += childInstallPkgDeps_recurse(fp,
1630                                                             &pkg->idepon_list,
1631                                                             undoit,
1632                                                             ndepth, nfirst);
1633                         }
1634                         if (first_one_only)
1635                                 break;
1636                         continue;
1637                 }
1638
1639                 if (pkg->dsynth_install_flg) {
1640                         if (DebugOpt >= 2 && pkg->pkgfile && fp) {
1641                                 fprintf(fp, "echo 'AlreadyHave %s'\n",
1642                                         pkg->pkgfile);
1643                         }
1644                         if (first_one_only)
1645                                 break;
1646                         continue;
1647                 }
1648
1649                 tot += childInstallPkgDeps_recurse(fp, &pkg->idepon_list,
1650                                                    undoit, ndepth, nfirst);
1651                 if (pkg->dsynth_install_flg) {
1652                         if (first_one_only)
1653                                 break;
1654                         continue;
1655                 }
1656                 pkg->dsynth_install_flg = 1;
1657
1658                 /*
1659                  * Generate package installation command
1660                  */
1661                 if (fp && pkg->pkgfile) {
1662                         fprintf(fp, "echo 'Installing /packages/All/%s'\n",
1663                                 pkg->pkgfile);
1664                         fprintf(fp, "pkg install -q -U -y /packages/All/%s "
1665                                 "|| exit 1\n",
1666                                 pkg->pkgfile);
1667                 } else if (fp) {
1668                         fprintf(fp, "echo 'CANNOT FIND PKG FOR %s'\n",
1669                                 pkg->portdir);
1670                 }
1671
1672                 if (pkg->pkgfile) {
1673                         struct stat st;
1674                         char *path;
1675                         char *ptr;
1676
1677                         asprintf(&path, "%s/%s", RepositoryPath, pkg->pkgfile);
1678                         ptr = strrchr(pkg->pkgfile, '.');
1679                         if (stat(path, &st) == 0) {
1680                                 if (strcmp(ptr, ".tar") == 0)
1681                                         tot += st.st_size;
1682                                 else if (strcmp(ptr, ".tgz") == 0)
1683                                         tot += st.st_size * 3;
1684                                 else if (strcmp(ptr, ".txz") == 0)
1685                                         tot += st.st_size * 5;
1686                                 else if (strcmp(ptr, ".tbz") == 0)
1687                                         tot += st.st_size * 3;
1688                                 else
1689                                         tot += st.st_size * 2;
1690                         }
1691                         free(path);
1692                 }
1693                 if (first_one_only)
1694                         break;
1695         }
1696         return tot;
1697 }
1698
1699 /*
1700  * Worker process interactions.
1701  *
1702  * The worker process is responsible for managing the build of a single
1703  * package.  It is exec'd by the master dsynth and only loads the
1704  * configuration.
1705  *
1706  * This process does not run in the chroot.  It will become the reaper for
1707  * all sub-processes and it will enter the chroot to execute various phases.
1708  * It catches SIGINTR, SIGHUP, and SIGPIPE and will iterate, terminate, and
1709  * reap all sub-process upon kill or exit.
1710  *
1711  * The command line forwarded to this function is:
1712  *
1713  *      WORKER slot# socketfd portdir/subdir
1714  *
1715  * TERM=dumb
1716  * USER=root
1717  * HOME=/root
1718  * LANG=C
1719  * SSL_NO_VERIFY_PEER=1
1720  * USE_PACKAGE_DEPENDS_ONLY=1
1721  * PORTSDIR=/xports
1722  * PORT_DBDIR=/options          For ports options
1723  * PACKAGE_BUILDING=yes         Don't build packages that aren't legally
1724  *                              buildable for a binary repo.
1725  * PKG_DBDIR=/var/db/pkg
1726  * PKG_CACHEDIR=/var/cache/pkg
1727  * PKG_CREATE_VERBOSE=yes       Ensure periodic output during packaging
1728  * (custom environment)
1729  * PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
1730  * UNAME_s=DragonFly            (example)
1731  * UNAME_v=DragonFly 5.7-SYNTH  (example)
1732  * UNAME_p=x86_64               (example)
1733  * UNAME_m=x86_64               (example)
1734  * UNAME_r=5.7-SYNTH            (example)
1735  * NO_DEPENDS=yes               (conditional based on phase)
1736  * DISTDIR=/distfiles
1737  * WRKDIRPREFIX=/construction
1738  * BATCH=yes
1739  * MAKE_JOBS_NUMBER=n
1740  *
1741  * SETUP:
1742  *      ldconfig -R
1743  *      /usr/local/sbin/pkg-static install /packages/All/<the pkg pkg>
1744  *      /usr/local/sbin/pkg-static install /packages/All/<pkg>
1745  *                      (for all dependencies)
1746  *
1747  * PHASES:              make -C path FLAVOR=flavor <phase>
1748  *      check-sanity
1749  *      pkg-depends
1750  *      fetch-depends
1751  *      fetch
1752  *      checksum
1753  *      extract-depends
1754  *      extract
1755  *      patch-depends
1756  *      patch
1757  *      build-depends
1758  *      lib-depends
1759  *      configure
1760  *      build
1761  *      run-depends
1762  *      stage
1763  *      test            (skipped)
1764  *      check-plist     ('dsynth test blahblah' or 'dsynth -D everything' only)
1765  *      package          e.g. /construction/lang/perl5.28/pkg/perl5-5.28.2.txz
1766  *      install-mtree   (skipped)
1767  *      install         (skipped)
1768  *      deinstall       (skipped)
1769  */
1770 void
1771 WorkerProcess(int ac, char **av)
1772 {
1773         wmsg_t wmsg;
1774         int fd;
1775         int slot;
1776         int tmpfd;
1777         int pkgpkg = 0;
1778         int status;
1779         int len;
1780         int do_install_phase;
1781         char *portdir;
1782         char *pkgfile;
1783         char *flavor;
1784         char *buf;
1785         worker_t *work;
1786         bulk_t *bulk;
1787         pkg_t pkg;
1788         buildenv_t *benv;
1789         FILE *fp;
1790
1791         /*
1792          * Parse arguments
1793          */
1794         if (ac != 6) {
1795                 dlog(DLOG_ALL, "WORKER PROCESS %d- bad arguments\n", getpid());
1796                 exit(1);
1797         }
1798         slot = strtol(av[1], NULL, 0);
1799         fd = strtol(av[2], NULL, 0);    /* master<->slave messaging */
1800         portdir = av[3];
1801         pkgfile = av[4];
1802         flavor = strchr(portdir, '@');
1803         if (flavor) {
1804                 *flavor++ = 0;
1805                 asprintf(&buf, "@%s", flavor);
1806                 WorkerFlavorPrt = buf;
1807                 buf = NULL;     /* safety */
1808         }
1809         WorkerProcFlags = strtol(av[5], NULL, 0);
1810
1811         bzero(&wmsg, sizeof(wmsg));
1812
1813         setproctitle("[%02d] WORKER STARTUP  %s%s",
1814                      slot, portdir, WorkerFlavorPrt);
1815
1816         if (strcmp(portdir, "ports-mgmt/pkg") == 0)
1817                 pkgpkg = 1;
1818
1819         signal(SIGTERM, phaseTerminateSignal);
1820         signal(SIGINT, phaseTerminateSignal);
1821         signal(SIGHUP, phaseTerminateSignal);
1822
1823         /*
1824          * Set up the environment
1825          */
1826         setenv("TERM", "dumb", 1);
1827         setenv("USER", "root", 1);
1828         setenv("HOME", "/root", 1);
1829         setenv("LANG", "C", 1);
1830         setenv("SSL_NO_VERIFY_PEER", "1", 1);
1831
1832         addbuildenv("USE_PACKAGE_DEPENDS_ONLY", "yes", BENV_MAKECONF);
1833         addbuildenv("PORTSDIR", "/xports", BENV_MAKECONF);
1834         addbuildenv("PORT_DBDIR", "/options", BENV_MAKECONF);
1835         addbuildenv("PKG_DBDIR", "/var/db/pkg", BENV_MAKECONF);
1836         addbuildenv("PKG_CACHEDIR", "/var/cache/pkg", BENV_MAKECONF);
1837         addbuildenv("PKG_SUFX", UsePkgSufx, BENV_MAKECONF);
1838         if (WorkerProcFlags & WORKER_PROC_DEVELOPER)
1839                 addbuildenv("DEVELOPER", "1", BENV_MAKECONF);
1840
1841         /*
1842          * CCache is a horrible unreliable hack but... leave the
1843          * mechanism in-place in case someone has a death wish.
1844          */
1845         if (UseCCache) {
1846                 addbuildenv("WITH_CCACHE_BUILD", "yes", BENV_MAKECONF);
1847                 addbuildenv("CCACHE_DIR", "/ccache", BENV_MAKECONF);
1848         }
1849
1850         addbuildenv("UID", "0", BENV_MAKECONF);
1851         addbuildenv("ARCH", ArchitectureName, BENV_MAKECONF);
1852
1853 #ifdef __DragonFly__
1854         addbuildenv("OPSYS", "DragonFly", BENV_MAKECONF);
1855         addbuildenv("DFLYVERSION", VersionFromParamHeader, BENV_MAKECONF);
1856         addbuildenv("OSVERSION", "9999999", BENV_MAKECONF);
1857 #else
1858 #error "Need OS-specific data to generate make.conf"
1859 #endif
1860
1861         addbuildenv("OSREL", ReleaseName, BENV_MAKECONF);
1862         addbuildenv("_OSRELEASE", VersionOnlyName, BENV_MAKECONF);
1863
1864         setenv("PATH",
1865                "/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin",
1866                1);
1867
1868         setenv("UNAME_s", OperatingSystemName, 1);
1869         setenv("UNAME_v", VersionName, 1);
1870         setenv("UNAME_p", ArchitectureName, 1);
1871         setenv("UNAME_m", MachineName, 1);
1872         setenv("UNAME_r", ReleaseName, 1);
1873
1874         addbuildenv("DISTDIR", "/distfiles", BENV_MAKECONF);
1875         addbuildenv("WRKDIRPREFIX", "/construction", BENV_MAKECONF);
1876         addbuildenv("BATCH", "yes", BENV_MAKECONF);
1877
1878         /*
1879          * Special consideration
1880          *
1881          * PACKAGE_BUILDING     - Disallow packaging ports which do not allow
1882          *                        for binary distribution.
1883          *
1884          * PKG_CREATE_VERBOSE   - Ensure periodic output during the packaging
1885          *                        process to avoid a watchdog timeout.
1886          *
1887          */
1888         addbuildenv("PACKAGE_BUILDING", "yes", BENV_MAKECONF);
1889         addbuildenv("PKG_CREATE_VERBOSE", "yes", BENV_MAKECONF);
1890         asprintf(&buf, "%d", MaxJobs);
1891         addbuildenv("MAKE_JOBS_NUMBER", buf, BENV_MAKECONF);
1892         freestrp(&buf);
1893
1894         if (flavor)
1895                 setenv("FLAVOR", flavor, 1);
1896
1897         /*
1898          * Become the reaper
1899          */
1900         if (procctl(P_PID, getpid(), PROC_REAP_ACQUIRE, NULL) < 0)
1901                 dfatal_errno("procctl() - Cannot become reaper");
1902
1903         /*
1904          * Initialize a worker structure
1905          */
1906         DoInitBuild(slot);
1907
1908         bzero(&pkg, sizeof(pkg));
1909         pkg.portdir = portdir;          /* sans flavor */
1910         pkg.pkgfile = pkgfile;
1911         if (strchr(portdir, '/'))
1912                 len = strchr(portdir, '/') - portdir;
1913         else
1914                 len = 0;
1915
1916         /*
1917          * Setup the logfile
1918          */
1919         asprintf(&pkg.logfile,
1920                  "%s/%*.*s___%s%s%s.log",
1921                  LogsPath, len, len, portdir,
1922                  ((portdir[len] == '/') ? portdir + len + 1 : portdir + len),
1923                  (flavor ? "@" : ""),
1924                  (flavor ? flavor : ""));
1925         tmpfd = open(pkg.logfile, O_RDWR|O_CREAT|O_TRUNC, 0666);
1926         if (tmpfd >= 0) {
1927                 if (DebugOpt >= 2) {
1928                         dlog(DLOG_ALL, "[%03d] %s LOGFILE %s\n",
1929                              slot, pkg.portdir, pkg.logfile);
1930                 }
1931                 close(tmpfd);
1932         } else {
1933                 dlog(DLOG_ALL, "[%03d] LOGFILE %s (create failed)\n",
1934                      slot, pkg.logfile);
1935         }
1936
1937         /*
1938          * Setup the work structure.  Because this is an exec'd sub-process,
1939          * there is only one work structure.
1940          */
1941         work = &WorkerAry[0];
1942         work->flavor = flavor;
1943         work->fds[0] = fd;
1944         work->pkg = &pkg;
1945         work->start_time = time(NULL);
1946
1947         /*
1948          * Do mounts
1949          */
1950         SigWork = work;
1951         setproctitle("[%02d] WORKER MOUNTS   %s%s",
1952                      slot, portdir, WorkerFlavorPrt);
1953         DoWorkerMounts(work);
1954
1955         /*
1956          * Generate an /etc/make.conf in the build base
1957          */
1958         asprintf(&buf, "%s/etc/make.conf", work->basedir);
1959         fp = fopen(buf, "w");
1960         dassert_errno(fp, "Unable to create %s\n", buf);
1961         for (benv = BuildEnv; benv; benv = benv->next) {
1962                 if (benv->type & BENV_PKGLIST)
1963                         continue;
1964                 if ((benv->type & BENV_CMDMASK) == BENV_MAKECONF) {
1965                         if (DebugOpt >= 2) {
1966                                 dlog(DLOG_ALL, "[%03d] ENV %s=%s\n",
1967                                      slot, benv->label, benv->data);
1968                         }
1969                         fprintf(fp, "%s=%s\n", benv->label, benv->data);
1970                 }
1971         }
1972         fclose(fp);
1973         freestrp(&buf);
1974
1975         /*
1976          * Set up our hooks
1977          */
1978         if (UsingHooks)
1979                 initbulk(childHookRun, MaxBulk);
1980
1981         /*
1982          * Start phases
1983          */
1984         wmsg.cmd = WMSG_CMD_INSTALL_PKGS;
1985         ipcwritemsg(fd, &wmsg);
1986         status = ipcreadmsg(fd, &wmsg);
1987         if (status < 0 || wmsg.cmd != WMSG_RES_INSTALL_PKGS)
1988                 dfatal("pkg installation handshake failed");
1989         do_install_phase = wmsg.status;
1990
1991         wmsg.cmd = WMSG_CMD_STATUS_UPDATE;
1992         wmsg.phase = PHASE_INSTALL_PKGS;
1993         wmsg.lines = 0;
1994
1995         status = ipcwritemsg(fd, &wmsg);
1996
1997         if (pkgpkg) {
1998                 dophase(work, &wmsg,
1999                         WDOG5, PHASE_PACKAGE, "package");
2000         } else {
2001                 if (do_install_phase) {
2002                         dophase(work, &wmsg,
2003                                 WDOG4, PHASE_INSTALL_PKGS, "setup");
2004                 }
2005                 dophase(work, &wmsg,
2006                         WDOG2, PHASE_CHECK_SANITY, "check-sanity");
2007                 dophase(work, &wmsg,
2008                         WDOG2, PHASE_PKG_DEPENDS, "pkg-depends");
2009                 dophase(work, &wmsg,
2010                         WDOG7, PHASE_FETCH_DEPENDS, "fetch-depends");
2011                 dophase(work, &wmsg,
2012                         WDOG7, PHASE_FETCH, "fetch");
2013                 dophase(work, &wmsg,
2014                         WDOG2, PHASE_CHECKSUM, "checksum");
2015                 dophase(work, &wmsg,
2016                         WDOG3, PHASE_EXTRACT_DEPENDS, "extract-depends");
2017                 dophase(work, &wmsg,
2018                         WDOG3, PHASE_EXTRACT, "extract");
2019                 dophase(work, &wmsg,
2020                         WDOG2, PHASE_PATCH_DEPENDS, "patch-depends");
2021                 dophase(work, &wmsg,
2022                         WDOG2, PHASE_PATCH, "patch");
2023                 dophase(work, &wmsg,
2024                         WDOG5, PHASE_BUILD_DEPENDS, "build-depends");
2025                 dophase(work, &wmsg,
2026                         WDOG5, PHASE_LIB_DEPENDS, "lib-depends");
2027                 dophase(work, &wmsg,
2028                         WDOG3, PHASE_CONFIGURE, "configure");
2029                 dophase(work, &wmsg,
2030                         WDOG9, PHASE_BUILD, "build");
2031                 dophase(work, &wmsg,
2032                         WDOG5, PHASE_RUN_DEPENDS, "run-depends");
2033                 dophase(work, &wmsg,
2034                         WDOG5, PHASE_STAGE, "stage");
2035 #if 0
2036                 dophase(work, &wmsg,
2037                         WDOG5, PHASE_TEST, "test");
2038 #endif
2039                 if (WorkerProcFlags & WORKER_PROC_CHECK_PLIST) {
2040                         dophase(work, &wmsg,
2041                                 WDOG1, PHASE_CHECK_PLIST, "check-plist");
2042                 }
2043                 dophase(work, &wmsg,
2044                         WDOG5, PHASE_PACKAGE, "package");
2045 #if 0
2046                 dophase(work, &wmsg,
2047                         WDOG5, PHASE_INSTALL_MTREE, "install-mtree");
2048                 dophase(work, &wmsg,
2049                         WDOG5, PHASE_INSTALL, "install");
2050                 dophase(work, &wmsg,
2051                         WDOG5, PHASE_DEINSTALL, "deinstall");
2052 #endif
2053         }
2054
2055         if (MasterPtyFd >= 0) {
2056                 close(MasterPtyFd);
2057                 MasterPtyFd = -1;
2058         }
2059
2060         setproctitle("[%02d] WORKER CLEANUP  %s%s",
2061                      slot, portdir, WorkerFlavorPrt);
2062
2063         /*
2064          * Copy the package to the repo.
2065          */
2066         if (work->accum_error == 0) {
2067                 char *b1;
2068                 char *b2;
2069
2070                 asprintf(&b1, "%s/construction/%s/pkg/%s",
2071                          work->basedir, pkg.portdir, pkg.pkgfile);
2072                 asprintf(&b2, "%s/%s", RepositoryPath, pkg.pkgfile);
2073                 if (copyfile(b1, b2)) {
2074                         ++work->accum_error;
2075                         dlog(DLOG_ALL, "[%03d] %s Unable to copy %s to %s\n",
2076                              work->index, pkg.portdir, b1, b2);
2077                 }
2078                 free(b1);
2079                 free(b2);
2080         }
2081
2082         /*
2083          * Unmount, unless we are in DebugStopMode.
2084          */
2085         if ((WorkerProcFlags & WORKER_PROC_DEBUGSTOP) == 0)
2086                 DoWorkerUnmounts(work);
2087
2088         /*
2089          * Send completion status to master dsynth worker thread.
2090          */
2091         if (work->accum_error) {
2092                 wmsg.cmd = WMSG_CMD_FAILURE;
2093         } else {
2094                 wmsg.cmd = WMSG_CMD_SUCCESS;
2095         }
2096         ipcwritemsg(fd, &wmsg);
2097         if (WorkerProcFlags & WORKER_PROC_DEBUGSTOP) {
2098                 wmsg.cmd = WMSG_CMD_FREEZEWORKER;
2099                 ipcwritemsg(fd, &wmsg);
2100         }
2101         if (UsingHooks) {
2102                 while ((bulk = getbulk()) != NULL)
2103                         freebulk(bulk);
2104                 donebulk();
2105         }
2106 }
2107
2108 static void
2109 dophase(worker_t *work, wmsg_t *wmsg, int wdog, int phaseid, const char *phase)
2110 {
2111         pkg_t *pkg = work->pkg;
2112         char buf[1024];
2113         pid_t pid;
2114         int status;
2115         int ms;
2116         pid_t wpid;
2117         int wpid_reaped;
2118         int fdlog;
2119         time_t start_time;
2120         time_t last_time;
2121         time_t next_time;
2122         time_t wdog_time;
2123         FILE *fp;
2124
2125         if (work->accum_error)
2126                 return;
2127         setproctitle("[%02d] WORKER %-8.8s %s%s",
2128                      work->index, phase, pkg->portdir, WorkerFlavorPrt);
2129         wmsg->phase = phaseid;
2130         if (ipcwritemsg(work->fds[0], wmsg) < 0) {
2131                 dlog(DLOG_ALL, "[%03d] %s Lost Communication with dsynth, "
2132                      "aborting worker\n",
2133                      work->index, pkg->portdir);
2134                 ++work->accum_error;
2135                 return;
2136         }
2137
2138         /*
2139          * Execute the port make command in chroot on a pty.
2140          */
2141         fflush(stdout);
2142         fflush(stderr);
2143         if (MasterPtyFd >= 0) {
2144                 int slavefd;
2145
2146                 /*
2147                  * NOTE: We can't open the slave in the child because the
2148                  *       master may race a disconnection test.  If we open
2149                  *       it in the parent our close() will flush any pending
2150                  *       output not read by the master (which is the same
2151                  *       parent process) and deadlock.
2152                  *
2153                  *       Solve this by hand-shaking the slave tty to give
2154                  *       the master time to close its slavefd (after this
2155                  *       section).
2156                  *
2157                  *       Leave the tty defaults intact, which also likely
2158                  *       means it will be in line-buffered mode, so handshake
2159                  *       with a full line.
2160                  *
2161                  * TODO: Our handshake probably echos back to the master pty
2162                  *       due to tty echo, and ends up in the log, so just
2163                  *       pass through a newline.
2164                  */
2165                 slavefd = open(ptsname(MasterPtyFd), O_RDWR);
2166                 dassert_errno(slavefd >= 0, "Cannot open slave pty");
2167
2168                 /*
2169                  * Now do the fork.
2170                  */
2171                 pid = fork();
2172                 if (pid == 0) {
2173                         login_tty(slavefd);
2174                         /* login_tty() closes slavefd */
2175                 } else {
2176                         close(slavefd);
2177                 }
2178         } else {
2179                 /*
2180                  * Initial MasterPtyFd for the slot, just use forkpty().
2181                  */
2182                 pid = forkpty(&MasterPtyFd, NULL, NULL, NULL);
2183         }
2184
2185         /*
2186          * The slave must make sure the master has time to close slavefd
2187          * in the re-use case before going its merry way.  The master needs
2188          * to set terminal modes and the window as well.
2189          */
2190         if (pid == 0) {
2191                 /*
2192                  * Slave waits for handshake
2193                  */
2194                 char ttybuf[2];
2195
2196                 read(0, ttybuf, 1);
2197         } else {
2198                 /*
2199                  * We are going through a pty, so set the tty modes to
2200                  * Set tty modes so we do not get ^M's in the log files.
2201                  *
2202                  * This isn't fatal if it doesn't work.  Remember that
2203                  * our output goes through the pty to the management
2204                  * process which will log it.
2205                  */
2206                 struct termios tio;
2207                 struct winsize win;
2208
2209                 if (tcgetattr(MasterPtyFd, &tio) == 0) {
2210                         tio.c_oflag |= OPOST | ONOCR;
2211                         tio.c_oflag &= ~(OCRNL | ONLCR);
2212                         tio.c_iflag |= ICRNL;
2213                         tio.c_iflag &= ~(INLCR | IGNCR);
2214                         if (tcsetattr(MasterPtyFd, TCSANOW, &tio)) {
2215                                 printf("tcsetattr failed: %s\n",
2216                                        strerror(errno));
2217                         }
2218
2219                         /*
2220                          * Give the tty a non-zero columns field.
2221                          * This fixes at least one port (textproc/po4a)
2222                          */
2223                         if (ioctl(MasterPtyFd, TIOCGWINSZ, &win) == 0) {
2224                                 win.ws_col = 80;
2225                                 ioctl(MasterPtyFd, TIOCSWINSZ, &win);
2226                         } else {
2227                                 printf("TIOCGWINSZ failed: %s\n",
2228                                        strerror(errno));
2229                         }
2230
2231                 } else {
2232                         printf("tcgetattr failed: %s\n", strerror(errno));
2233                 }
2234
2235                 /*
2236                  * Master issues handshake
2237                  */
2238                 write(MasterPtyFd, "\n", 1);
2239         }
2240
2241         if (pid == 0) {
2242                 /*
2243                  * Additional phase-specific environment variables
2244                  *
2245                  * - Do not try to process missing depends outside of the
2246                  *   depends phases.  Also relies on USE_PACKAGE_DEPENDS_ONLY
2247                  *   in the make.conf.
2248                  */
2249                 switch(phaseid) {
2250                 case PHASE_CHECK_SANITY:
2251                 case PHASE_FETCH:
2252                 case PHASE_CHECKSUM:
2253                 case PHASE_EXTRACT:
2254                 case PHASE_PATCH:
2255                 case PHASE_CONFIGURE:
2256                 case PHASE_STAGE:
2257                 case PHASE_TEST:
2258                 case PHASE_CHECK_PLIST:
2259                 case PHASE_INSTALL_MTREE:
2260                 case PHASE_INSTALL:
2261                 case PHASE_DEINSTALL:
2262                         break;
2263                 case PHASE_PKG_DEPENDS:
2264                 case PHASE_FETCH_DEPENDS:
2265                 case PHASE_EXTRACT_DEPENDS:
2266                 case PHASE_PATCH_DEPENDS:
2267                 case PHASE_BUILD_DEPENDS:
2268                 case PHASE_LIB_DEPENDS:
2269                 case PHASE_RUN_DEPENDS:
2270                         break;
2271                 default:
2272                         setenv("NO_DEPENDS", "1", 1);
2273                         break;
2274                 }
2275
2276                 /*
2277                  * Clean-up, chdir, and chroot.
2278                  */
2279                 closefrom(3);
2280                 if (chdir(work->basedir) < 0)
2281                         dfatal_errno("chdir in phase initialization");
2282                 if (chroot(work->basedir) < 0)
2283                         dfatal_errno("chroot in phase initialization");
2284
2285                 /*
2286                  * We have a choice here on how to handle stdin (fd 0).
2287                  * We can leave it connected to the pty in which case
2288                  * the build will just block if it tries to ask a
2289                  * question (and the watchdog will kill it, eventually),
2290                  * or we can try to EOF the pty, or we can attach /dev/null
2291                  * to descriptor 0.
2292                  */
2293                 if (NullStdinOpt) {
2294                         int fd;
2295
2296                         fd = open("/dev/null", O_RDWR);
2297                         dassert_errno(fd >= 0, "cannot open /dev/null");
2298                         if (fd != 0) {
2299                                 dup2(fd, 0);
2300                                 close(fd);
2301                         }
2302                 }
2303
2304                 /*
2305                  * Execute the appropriate command.
2306                  */
2307                 switch(phaseid) {
2308                 case PHASE_INSTALL_PKGS:
2309                         snprintf(buf, sizeof(buf), "/tmp/dsynth_install_pkgs");
2310                         execl(buf, buf, NULL);
2311                         break;
2312                 default:
2313                         snprintf(buf, sizeof(buf), "/xports/%s", pkg->portdir);
2314                         execl(MAKE_BINARY, MAKE_BINARY, "-C", buf, phase, NULL);
2315                         break;
2316                 }
2317                 _exit(1);
2318         }
2319         fcntl(MasterPtyFd, F_SETFL, O_NONBLOCK);
2320
2321         if (pid < 0) {
2322                 dlog(DLOG_ALL, "[%03d] %s Fork Failed: %s\n",
2323                      work->index, pkg->logfile, strerror(errno));
2324                 ++work->accum_error;
2325                 return;
2326         }
2327
2328         SigPid = pid;
2329
2330         fdlog = open(pkg->logfile, O_RDWR|O_CREAT|O_APPEND, 0644);
2331         if (fdlog < 0) {
2332                 dlog(DLOG_ALL, "[%03d] %s Cannot open logfile '%s': %s\n",
2333                      work->index, pkg->portdir,
2334                      pkg->logfile, strerror(errno));
2335         }
2336
2337         snprintf(buf, sizeof(buf),
2338                  "----------------------------------------"
2339                  "---------------------------------------\n"
2340                  "-- Phase: %s\n"
2341                  "----------------------------------------"
2342                  "---------------------------------------\n",
2343                  phase);
2344         write(fdlog, buf, strlen(buf));
2345
2346         start_time = time(NULL);
2347         last_time = start_time;
2348         wdog_time = start_time;
2349         wpid_reaped = 0;
2350
2351         status = 0;
2352         for (;;) {
2353                 ms = mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time);
2354                 if (ms == MPTY_FAILED) {
2355                         dlog(DLOG_ALL,
2356                              "[%03d] %s lost pty in phase %s, terminating\n",
2357                              work->index, pkg->portdir, phase);
2358                         break;
2359                 }
2360                 if (ms == MPTY_EOF)
2361                         break;
2362
2363                 /*
2364                  * Generally speaking update status once a second.
2365                  * This also allows us to detect if the management
2366                  * dsynth process has gone away.
2367                  */
2368                 next_time = time(NULL);
2369                 if (next_time != last_time) {
2370                         double dload[3];
2371                         double dv;
2372                         int wdog_scaled;
2373
2374                         /*
2375                          * Send status update to the worker management thread
2376                          * in the master dsynth process.  Remember, *WE* are
2377                          * the worker management process sub-fork.
2378                          */
2379                         if (ipcwritemsg(work->fds[0], wmsg) < 0)
2380                                 break;
2381                         last_time = next_time;
2382
2383                         /*
2384                          * Watchdog scaling
2385                          */
2386                         getloadavg(dload, 3);
2387                         adjloadavg(dload);
2388                         dv = dload[2] / NumCores;
2389                         if (dv < (double)NumCores) {
2390                                 wdog_scaled = wdog;
2391                         } else {
2392                                 if (dv > 4.0 * NumCores)
2393                                         dv = 4.0 * NumCores;
2394                                 wdog_scaled = wdog * dv / NumCores;
2395                         }
2396
2397                         /*
2398                          * Watchdog
2399                          */
2400                         if (next_time - wdog_time >= wdog_scaled * 60) {
2401                                 snprintf(buf, sizeof(buf),
2402                                          "\n--------\n"
2403                                          "WATCHDOG TIMEOUT FOR %s in %s "
2404                                          "after %d minutes\n"
2405                                          "Killing pid %d\n"
2406                                          "--------\n",
2407                                          pkg->portdir, phase, wdog_scaled, pid);
2408                                 if (fdlog >= 0)
2409                                         write(fdlog, buf, strlen(buf));
2410                                 dlog(DLOG_ALL,
2411                                      "[%03d] %s WATCHDOG TIMEOUT in %s "
2412                                      "after %d minutes (%d min scaled)\n",
2413                                      work->index, pkg->portdir, phase,
2414                                      wdog, wdog_scaled);
2415                                 kill(pid, SIGKILL);
2416                                 ++work->accum_error;
2417                                 break;
2418                         }
2419                 }
2420
2421                 /*
2422                  * Check process exit.  Normally the pty will EOF
2423                  * but if background processes remain we need to
2424                  * check here to see if our primary exec is done,
2425                  * so we can break out and reap those processes.
2426                  *
2427                  * Generally reap any other processes we have inherited
2428                  * while we are here.
2429                  */
2430                 do {
2431                         wpid = wait3(&status, WNOHANG, NULL);
2432                 } while (wpid > 0 && wpid != pid);
2433                 if (wpid == pid && WIFEXITED(status)) {
2434                         wpid_reaped = 1;
2435                         break;
2436                 }
2437         }
2438
2439         next_time = time(NULL);
2440
2441         setproctitle("[%02d] WORKER EXITREAP %s%s",
2442                      work->index, pkg->portdir, WorkerFlavorPrt);
2443
2444         /*
2445          * We usually get here due to a mpty EOF, but not always as there
2446          * could be persistent processes still holding the slave.  Finish
2447          * up getting the exit status for the main process we are waiting
2448          * on and clean out any data left on the MasterPtyFd (as it could
2449          * be blocking the exit).
2450          */
2451         while (wpid_reaped == 0) {
2452                 (void)mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time);
2453                 wpid = waitpid(pid, &status, WNOHANG);
2454                 if (wpid == pid && WIFEXITED(status)) {
2455                         wpid_reaped = 1;
2456                         break;
2457                 }
2458                 if (wpid < 0 && errno != EINTR) {
2459                         break;
2460                 }
2461
2462                 /*
2463                  * Safety.  The normal phase waits until the fork/exec'd
2464                  * pid finishes, causing a pty EOF on exit (the slave
2465                  * descriptor is closed by the kernel on exit so the
2466                  * process should already have exited).
2467                  *
2468                  * However, it is also possible to get here if the pty fails
2469                  * for some reason.  In this case, make sure that the process
2470                  * is killed.
2471                  */
2472                 kill(pid, SIGKILL);
2473         }
2474
2475         /*
2476          * Clean out anything left on the pty but don't wait around
2477          * because there could be background processes preventing the
2478          * slave side from closing.
2479          */
2480         while (mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time) == MPTY_DATA)
2481                 ;
2482
2483         /*
2484          * Report on the exit condition.  If the pid was somehow lost
2485          * (probably due to someone gdb'ing the process), assume an error.
2486          */
2487         if (wpid_reaped) {
2488                 if (WEXITSTATUS(status)) {
2489                         dlog(DLOG_ALL | DLOG_FILTER,
2490                              "[%03d] %s Build phase '%s' failed exit %d\n",
2491                              work->index, pkg->portdir, phase,
2492                              WEXITSTATUS(status));
2493                         ++work->accum_error;
2494                 }
2495         } else {
2496                 dlog(DLOG_ALL, "[%03d] %s Build phase '%s' failed - lost pid\n",
2497                      work->index, pkg->portdir, phase);
2498                 ++work->accum_error;
2499         }
2500
2501         /*
2502          * Kill any processes still running (sometimes processes end up in
2503          * the background during a dports build), and clean up any other
2504          * children that we have inherited.
2505          */
2506         phaseReapAll();
2507
2508         /*
2509          * After the extraction phase add the space used by /construction
2510          * to the memory use.  This helps us reduce the amount of paging
2511          * we do due to extremely large package extractions (languages,
2512          * chromium, etc).
2513          *
2514          * (dsynth already estimated the space used by the package deps
2515          * up front, but this will help us further).
2516          */
2517         if (work->accum_error == 0 && phaseid == PHASE_EXTRACT) {
2518                 struct statfs sfs;
2519                 char *b1;
2520
2521                 asprintf(&b1, "%s/construction", work->basedir);
2522                 if (statfs(b1, &sfs) == 0) {
2523                         wmsg->memuse = (sfs.f_blocks - sfs.f_bfree) *
2524                                        sfs.f_bsize;
2525                         ipcwritemsg(work->fds[0], wmsg);
2526                 }
2527         }
2528
2529         /*
2530          * Update log
2531          */
2532         if (fdlog >= 0) {
2533                 struct stat st;
2534                 int h;
2535                 int m;
2536                 int s;
2537
2538                 last_time = next_time - start_time;
2539                 s = last_time % 60;
2540                 m = last_time / 60 % 60;
2541                 h = last_time / 3600;
2542
2543                 fp = fdopen(fdlog, "a");
2544                 if (fp == NULL) {
2545                         dlog(DLOG_ALL, "[%03d] %s Cannot fdopen fdlog: %s %d\n",
2546                              work->index, pkg->portdir,
2547                              strerror(errno), fstat(fdlog, &st));
2548                         close(fdlog);
2549                         goto skip;
2550                 }
2551
2552                 fprintf(fp, "\n");
2553                 if (work->accum_error) {
2554                         fprintf(fp, "FAILED %02d:%02d:%02d\n", h, m, s);
2555                 } else {
2556                         if (phaseid == PHASE_EXTRACT && wmsg->memuse) {
2557                                 fprintf(fp, "Extracted Memory Use: %6.2fM\n",
2558                                         wmsg->memuse / (1024.0 * 1024.0));
2559                         }
2560                         fprintf(fp, "SUCCEEDED %02d:%02d:%02d\n", h, m, s);
2561                 }
2562                 last_time = next_time - work->start_time;
2563                 s = last_time % 60;
2564                 m = last_time / 60 % 60;
2565                 h = last_time / 3600;
2566                 if (phaseid == PHASE_PACKAGE) {
2567                         fprintf(fp, "TOTAL TIME %02d:%02d:%02d\n", h, m, s);
2568                 }
2569                 fprintf(fp, "\n");
2570                 fclose(fp);
2571 skip:
2572                 ;
2573         }
2574
2575 }
2576
2577 static void
2578 phaseReapAll(void)
2579 {
2580         struct reaper_status rs;
2581         int status;
2582
2583         while (procctl(P_PID, getpid(), PROC_REAP_STATUS, &rs) == 0) {
2584                 if ((rs.flags & PROC_REAP_ACQUIRE) == 0)
2585                         break;
2586                 if (rs.pid_head < 0)
2587                         break;
2588                 if (kill(rs.pid_head, SIGKILL) == 0) {
2589                         while (waitpid(rs.pid_head, &status, 0) < 0)
2590                                 ;
2591                 }
2592         }
2593         while (wait3(&status, 0, NULL) > 0)
2594                 ;
2595 }
2596
2597 static void
2598 phaseTerminateSignal(int sig __unused)
2599 {
2600         if (CopyFileFd >= 0)
2601                 close(CopyFileFd);
2602         if (MasterPtyFd >= 0)
2603                 close(MasterPtyFd);
2604         if (SigPid > 1)
2605                 kill(SigPid, SIGKILL);
2606         phaseReapAll();
2607         if (SigWork)
2608                 DoWorkerUnmounts(SigWork);
2609         exit(1);
2610 }
2611
2612 static
2613 char *
2614 buildskipreason(pkglink_t *parent, pkg_t *pkg)
2615 {
2616         pkglink_t *link;
2617         pkg_t *scan;
2618         char *reason = NULL;
2619         char *ptr;
2620         size_t tot;
2621         size_t len;
2622         pkglink_t stack;
2623
2624         if ((pkg->flags & PKGF_NOBUILD_I) && pkg->ignore)
2625                 asprintf(&reason, "%s ", pkg->ignore);
2626
2627         tot = 0;
2628         PKGLIST_FOREACH(link, &pkg->idepon_list) {
2629 #if 0
2630                 if (link->dep_type > DEP_TYPE_BUILD)
2631                         continue;
2632 #endif
2633                 scan = link->pkg;
2634                 if (scan == NULL)
2635                         continue;
2636                 if ((scan->flags & (PKGF_ERROR | PKGF_NOBUILD)) == 0)
2637                         continue;
2638                 if (scan->flags & PKGF_NOBUILD) {
2639                         stack.pkg = scan;
2640                         stack.next = parent;
2641                         ptr = buildskipreason(&stack, scan);
2642                         len = strlen(scan->portdir) + strlen(ptr) + 8;
2643                         reason = realloc(reason, tot + len);
2644                         snprintf(reason + tot, len, "%s->%s",
2645                                  scan->portdir, ptr);
2646                         free(ptr);
2647                 } else {
2648                         len = strlen(scan->portdir) + 8;
2649                         reason = realloc(reason, tot + len);
2650                         snprintf(reason + tot, len, "%s", scan->portdir);
2651                 }
2652
2653                 /*
2654                  * Don't try to print the entire graph
2655                  */
2656                 if (parent)
2657                         break;
2658                 tot += strlen(reason + tot);
2659                 reason[tot++] = ' ';
2660                 reason[tot] = 0;
2661         }
2662         return (reason);
2663 }
2664
2665 /*
2666  * Count number of packages that would be skipped due to the
2667  * specified package having failed.
2668  *
2669  * Call with mode 1 to count, and mode 0 to clear the
2670  * cumulative rscan flag (used to de-duplicate the count).
2671  *
2672  * Must be serialized.
2673  */
2674 static int
2675 buildskipcount_dueto(pkg_t *pkg, int mode)
2676 {
2677         pkglink_t *link;
2678         pkg_t *scan;
2679         int total;
2680
2681         total = 0;
2682         PKGLIST_FOREACH(link, &pkg->deponi_list) {
2683                 scan = link->pkg;
2684                 if (scan == NULL || scan->rscan == mode)
2685                         continue;
2686                 scan->rscan = mode;
2687                 ++total;
2688                 total += buildskipcount_dueto(scan, mode);
2689         }
2690         return total;
2691 }
2692
2693 /*
2694  * The master ptyfd is in non-blocking mode.  Drain up to 1024 bytes
2695  * and update wmsg->lines and *wdog_timep as appropriate.
2696  *
2697  * This function will poll, stalling up to 1 second.
2698  */
2699 static int
2700 mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg, time_t *wdog_timep)
2701 {
2702         struct pollfd pfd;
2703         char buf[1024];
2704         ssize_t r;
2705
2706         pfd.fd = ptyfd;
2707         pfd.events = POLLIN;
2708         pfd.revents = 0;
2709
2710         poll(&pfd, 1, 1000);
2711         if (pfd.revents) {
2712                 r = read(ptyfd, buf, sizeof(buf));
2713                 if (r > 0) {
2714                         *wdog_timep = time(NULL);
2715                         if (r > 0 && fdlog >= 0)
2716                                 write(fdlog, buf, r);
2717                         while (--r >= 0) {
2718                                 if (buf[r] == '\n')
2719                                         ++wmsg->lines;
2720                         }
2721                         return MPTY_DATA;
2722                 } else if (r < 0) {
2723                         if (errno != EINTR && errno != EAGAIN)
2724                                 return MPTY_FAILED;
2725                         return MPTY_AGAIN;
2726                 } else if (r == 0) {
2727                         return MPTY_EOF;
2728                 }
2729         }
2730         return MPTY_AGAIN;
2731 }
2732
2733 /*
2734  * Copy a (package) file from (src) to (dst), use an intermediate file and
2735  * rename to ensure that interruption does not leave us with a corrupt
2736  * package file.
2737  *
2738  * This is called by the WORKER process.
2739  *
2740  * (dsynth management thread -> WORKER process -> sub-processes)
2741  */
2742 #define COPYBLKSIZE     32768
2743
2744 int
2745 copyfile(char *src, char *dst)
2746 {
2747         char *tmp;
2748         char *buf;
2749         int fd1;
2750         int fd2;
2751         int error = 0;
2752         int mask;
2753         ssize_t r;
2754
2755         asprintf(&tmp, "%s.new", dst);
2756         buf = malloc(COPYBLKSIZE);
2757
2758         mask = sigsetmask(sigmask(SIGTERM)|sigmask(SIGINT)|sigmask(SIGHUP));
2759         fd1 = open(src, O_RDONLY|O_CLOEXEC);
2760         fd2 = open(tmp, O_RDWR|O_CREAT|O_TRUNC|O_CLOEXEC, 0644);
2761         CopyFileFd = fd1;
2762         sigsetmask(mask);
2763         while ((r = read(fd1, buf, COPYBLKSIZE)) > 0) {
2764                 if (write(fd2, buf, r) != r)
2765                         error = 1;
2766         }
2767         if (r < 0)
2768                 error = 1;
2769         mask = sigsetmask(sigmask(SIGTERM)|sigmask(SIGINT)|sigmask(SIGHUP));
2770         CopyFileFd = -1;
2771         close(fd1);
2772         close(fd2);
2773         sigsetmask(mask);
2774         if (error) {
2775                 remove(tmp);
2776         } else {
2777                 if (rename(tmp, dst)) {
2778                         error = 1;
2779                         remove(tmp);
2780                 }
2781         }
2782
2783         freestrp(&buf);
2784         freestrp(&tmp);
2785
2786         return error;
2787 }
2788
2789 /*
2790  * doHook()
2791  *
2792  * primary process (threaded) - run_start, run_end, pkg_ignored, pkg_skipped
2793  * worker process  (threaded) - pkg_sucess, pkg_failure
2794  *
2795  * If waitfor is non-zero this hook will be serialized.
2796  */
2797 static void
2798 doHook(pkg_t *pkg, const char *id, const char *path, int waitfor)
2799 {
2800         if (path == NULL)
2801                 return;
2802         while (waitfor && getbulk() != NULL)
2803                 ;
2804         if (pkg)
2805                 queuebulk(pkg->portdir, id, path, pkg->pkgfile);
2806         else
2807                 queuebulk(NULL, id, path, NULL);
2808         while (waitfor && getbulk() != NULL)
2809                 ;
2810 }
2811
2812 /*
2813  * Execute hook (backend)
2814  *
2815  * s1 - portdir
2816  * s2 - id
2817  * s3 - script path
2818  * s4 - pkgfile         (if applicable)
2819  */
2820 static void
2821 childHookRun(bulk_t *bulk)
2822 {
2823         const char *cav[MAXCAC];
2824         buildenv_t benv[MAXCAC];
2825         char buf1[128];
2826         char buf2[128];
2827         char buf3[128];
2828         char buf4[128];
2829         FILE *fp;
2830         char *ptr;
2831         size_t len;
2832         pid_t pid;
2833         int cac;
2834         int bi;
2835
2836         cac = 0;
2837         bi = 0;
2838         bzero(benv, sizeof(benv));
2839
2840         cav[cac++] = bulk->s3;
2841
2842         benv[bi].label = "PROFILE";
2843         benv[bi].data = Profile;
2844         ++bi;
2845
2846         benv[bi].label = "DIR_PACKAGES";
2847         benv[bi].data = PackagesPath;
2848         ++bi;
2849
2850         benv[bi].label = "DIR_REPOSITORY";
2851         benv[bi].data = RepositoryPath;
2852         ++bi;
2853
2854         benv[bi].label = "DIR_PORTS";
2855         benv[bi].data = DPortsPath;
2856         ++bi;
2857
2858         benv[bi].label = "DIR_OPTIONS";
2859         benv[bi].data = OptionsPath;
2860         ++bi;
2861
2862         benv[bi].label = "DIR_DISTFILES";
2863         benv[bi].data = DistFilesPath;
2864         ++bi;
2865
2866         benv[bi].label = "DIR_LOGS";
2867         benv[bi].data = LogsPath;
2868         ++bi;
2869
2870         benv[bi].label = "DIR_BUILDBASE";
2871         benv[bi].data = BuildBase;
2872         ++bi;
2873
2874         if (strcmp(bulk->s2, "hook_run_start") == 0) {
2875                 snprintf(buf1, sizeof(buf1), "%d", BuildTotal);
2876                 benv[bi].label = "PORTS_QUEUED";
2877                 benv[bi].data = buf1;
2878                 ++bi;
2879         } else if (strcmp(bulk->s2, "hook_run_end") == 0) {
2880                 snprintf(buf1, sizeof(buf1), "%d", BuildSuccessCount);
2881                 benv[bi].label = "PORTS_BUILT";
2882                 benv[bi].data = buf1;
2883                 ++bi;
2884                 snprintf(buf2, sizeof(buf2), "%d", BuildFailCount);
2885                 benv[bi].label = "PORTS_FAILED";
2886                 benv[bi].data = buf2;
2887                 ++bi;
2888                 snprintf(buf3, sizeof(buf3), "%d", BuildIgnoreCount);
2889                 benv[bi].label = "PORTS_IGNORED";
2890                 benv[bi].data = buf3;
2891                 ++bi;
2892                 snprintf(buf4, sizeof(buf4), "%d", BuildSkipCount);
2893                 benv[bi].label = "PORTS_SKIPPED";
2894                 benv[bi].data = buf4;
2895                 ++bi;
2896         } else {
2897                 /*
2898                  * success, failure, ignored, skipped
2899                  */
2900                 benv[bi].label = "RESULT";
2901                 if (strcmp(bulk->s2, "hook_pkg_success") == 0) {
2902                         benv[bi].data = "success";
2903                 } else if (strcmp(bulk->s2, "hook_pkg_failure") == 0) {
2904                         benv[bi].data = "failure";
2905                 } else if (strcmp(bulk->s2, "hook_pkg_ignored") == 0) {
2906                         benv[bi].data = "ignored";
2907                 } else if (strcmp(bulk->s2, "hook_pkg_skipped") == 0) {
2908                         benv[bi].data = "skipped";
2909                 } else {
2910                         dfatal("Unknown hook id: %s", bulk->s2);
2911                         /* NOT REACHED */
2912                 }
2913                 ++bi;
2914
2915                 /*
2916                  * For compatibility with synth:
2917                  *
2918                  * ORIGIN does not include any @flavor, thus it is suitable
2919                  * for finding the actual port directory/subdirectory.
2920                  *
2921                  * FLAVOR is set to ORIGIN if there is no flavor, otherwise
2922                  * it is set to only the flavor sans the '@'.
2923                  */
2924                 if ((ptr = strchr(bulk->s1, '@')) != NULL) {
2925                         snprintf(buf1, sizeof(buf1), "%*.*s",
2926                                  (int)(ptr - bulk->s1),
2927                                  (int)(ptr - bulk->s1),
2928                                  bulk->s1);
2929                         benv[bi].label = "ORIGIN";
2930                         benv[bi].data = buf1;
2931                         ++bi;
2932                         benv[bi].label = "FLAVOR";
2933                         benv[bi].data = ptr + 1;
2934                         ++bi;
2935                 } else {
2936                         benv[bi].label = "ORIGIN";
2937                         benv[bi].data = bulk->s1;
2938                         ++bi;
2939                         benv[bi].label = "FLAVOR";
2940                         benv[bi].data = bulk->s1;
2941                         ++bi;
2942                 }
2943                 benv[bi].label = "PKGNAME";
2944                 benv[bi].data = bulk->s4;
2945                 ++bi;
2946         }
2947
2948         benv[bi].label = NULL;
2949         benv[bi].data = NULL;
2950
2951         fp = dexec_open(cav, cac, &pid, benv, 0, 0);
2952         while ((ptr = fgetln(fp, &len)) != NULL)
2953                 ;
2954
2955         if (dexec_close(fp, pid)) {
2956                 dlog(DLOG_ALL,
2957                      "[XXX] %s SCRIPT %s (%s)\n",
2958                      bulk->s1, bulk->s2, bulk->s3);
2959         }
2960 }
2961
2962 /*
2963  * Adjusts dload[0] by adding in t_pw (processes waiting on page-fault).
2964  * We don't want load reductions due to e.g. thrashing to cause dsynth
2965  * to increase the dynamic limit because it thinks the load is low.
2966  *
2967  * This has a desirable property.  If the system pager cannot keep up
2968  * with process demand t_pw will spike while loadavg will only drop
2969  * slowly, resulting in a high adjusted load calculation that causes
2970  * dsynth to quickly clamp-down the limit.  If the condition alleviates,
2971  * the limit will then rise slowly again, possibly even before existing
2972  * jobs are retired to meet the clamp-down from the original spike.
2973  */
2974 static void
2975 adjloadavg(double *dload)
2976 {
2977 #if defined(__DragonFly__)
2978         struct vmtotal total;
2979         size_t size;
2980
2981         size = sizeof(total);
2982         if (sysctlbyname("vm.vmtotal", &total, &size, NULL, 0) == 0) {
2983                 dload[0] += (double)total.t_pw;
2984         }
2985 #else
2986         dload[0] += 0.0;        /* just avoid compiler 'unused' warnings */
2987 #endif
2988 }