usr.bin/dsynth/build.c

   1 /*
   2  * Copyright (c) 2019 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * This code uses concepts and configuration based on 'synth', by
   8  * John R. Marino <draco@marino.st>, which was written in ada.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  *
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in
  18  *    the documentation and/or other materials provided with the
  19  *    distribution.
  20  * 3. Neither the name of The DragonFly Project nor the names of its
  21  *    contributors may be used to endorse or promote products derived
  22  *    from this software without specific, prior written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  25  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  26  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  27  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  28  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  29  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  30  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  31  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  32  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  33  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  34  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  35  * SUCH DAMAGE.
  36  */
  37 #include "dsynth.h"
  38
  39 worker_t WorkerAry[MAXWORKERS];
  40 int BuildInitialized;
  41 int RunningWorkers;
  42 int DynamicMaxWorkers;
  43 int FailedWorkers;
  44 long RunningPkgDepSize;
  45 pthread_mutex_t WorkerMutex;
  46 pthread_cond_t WorkerCond;
  47
  48 static int build_find_leaves(pkg_t *parent, pkg_t *pkg,
  49                         pkg_t ***build_tailp, int *app, int *hasworkp,
  50                         int depth, int first, int first_one_only);
  51 static int buildCalculateDepiDepth(pkg_t *pkg);
  52 static void build_clear_trav(pkg_t *pkg);
  53 static void startbuild(pkg_t **build_listp, pkg_t ***build_tailp);
  54 static int qsort_depi(const void *pkg1, const void *pkg2);
  55 static int qsort_idep(const void *pkg1, const void *pkg2);
  56 static void startworker(pkg_t *pkg, worker_t *work);
  57 static void cleanworker(worker_t *work);
  58 static void waitbuild(int whilematch, int dynamicmax);
  59 static void workercomplete(worker_t *work);
  60 static void *childBuilderThread(void *arg);
  61 static int childInstallPkgDeps(worker_t *work);
  62 static size_t childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list,
  63                         int undoit, int depth, int first_one_only);
  64 static void dophase(worker_t *work, wmsg_t *wmsg,
  65                         int wdog, int phaseid, const char *phase);
  66 static void phaseReapAll(void);
  67 static void phaseTerminateSignal(int sig);
  68 static char *buildskipreason(pkglink_t *parent, pkg_t *pkg);
  69 static int mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg,
  70                         time_t *wdog_timep);
  71 static int copyfile(char *src, char *dst);
  72 static void doHook(pkg_t *pkg, const char *id, const char *path, int waitfor);
  73 static void childHookRun(bulk_t *bulk);
  74
  75 static worker_t *SigWork;
  76 static int MasterPtyFd = -1;
  77 static int CopyFileFd = -1;
  78 static pid_t SigPid;
  79 static const char *WorkerFlavorPrt = "";        /* "" or "@flavor" */
  80
  81 #define MPTY_FAILED     -2
  82 #define MPTY_AGAIN      -1
  83 #define MPTY_EOF        0
  84 #define MPTY_DATA       1
  85
  86 int BuildTotal;
  87 int BuildCount;
  88 int BuildSkipCount;
  89 int BuildIgnoreCount;
  90 int BuildFailCount;
  91 int BuildSuccessCount;
  92
  93 /*
  94  * Initialize the WorkerAry[]
  95  */
  96 void
  97 DoInitBuild(int slot_override)
  98 {
  99         worker_t *work;
 100         struct stat st;
 101         int i;
 102
 103         ddassert(slot_override < 0 || MaxWorkers == 1);
 104
 105         bzero(WorkerAry, MaxWorkers * sizeof(worker_t));
 106         pthread_mutex_init(&WorkerMutex, NULL);
 107
 108         for (i = 0; i < MaxWorkers; ++i) {
 109                 work = &WorkerAry[i];
 110                 work->index = (slot_override >= 0) ? slot_override : i;
 111                 work->state = WORKER_NONE;
 112                 asprintf(&work->basedir, "%s/SL%02d", BuildBase, work->index);
 113                 pthread_cond_init(&work->cond, NULL);
 114         }
 115         BuildCount = 0;
 116
 117         /*
 118          * Create required sub-directories. The base directories must already
 119          * exist as a dsynth configuration safety.
 120          */
 121         if (stat(RepositoryPath, &st) < 0) {
 122                 if (mkdir(RepositoryPath, 0755) < 0)
 123                         dfatal("Cannot mkdir %s\n", RepositoryPath);
 124         }
 125
 126         BuildInitialized = 1;
 127
 128         /*
 129          * slow-start (increases at a rate of 1 per 5 seconds)
 130          */
 131         if (SlowStartOpt > MaxWorkers)
 132                 DynamicMaxWorkers = MaxWorkers;
 133         else if (SlowStartOpt > 0)
 134                 DynamicMaxWorkers = SlowStartOpt;
 135         else
 136                 DynamicMaxWorkers = MaxWorkers;
 137 }
 138
 139 /*
 140  * Called by the frontend to clean-up any hanging mounts.
 141  */
 142 void
 143 DoCleanBuild(int resetlogs)
 144 {
 145         int i;
 146
 147         ddassert(BuildInitialized);
 148
 149         if (resetlogs)
 150                 dlogreset();
 151         for (i = 0; i < MaxWorkers; ++i) {
 152                 DoWorkerUnmounts(&WorkerAry[i]);
 153         }
 154 }
 155
 156 void
 157 DoBuild(pkg_t *pkgs)
 158 {
 159         pkg_t *build_list = NULL;
 160         pkg_t **build_tail = &build_list;
 161         pkg_t *scan;
 162         bulk_t *bulk;
 163         int haswork = 1;
 164         int first = 1;
 165         int newtemplate;
 166         time_t startTime;
 167         time_t t;
 168         int h, m, s;
 169
 170         /*
 171          * We use our bulk system to run hooks.  This will be for
 172          * Skipped and Ignored.  Success and Failure are handled by
 173          * WorkerProcess() which is a separately-exec'd dsynth.
 174          */
 175         if (UsingHooks)
 176                 initbulk(childHookRun, MaxBulk);
 177
 178         /*
 179          * Count up the packages, not counting dummies
 180          */
 181         for (scan = pkgs; scan; scan = scan->bnext) {
 182                 if ((scan->flags & PKGF_DUMMY) == 0)
 183                         ++BuildTotal;
 184         }
 185
 186         doHook(NULL, "hook_run_start", HookRunStart, 1);
 187
 188         /*
 189          * The pkg and pkg-static binaries are needed.  If already present
 190          * then assume that the template is also valid, otherwise build
 191          * both.
 192          */
 193         scan = GetPkgPkg(pkgs);
 194
 195         /*
 196          * Create our template.  The template will be missing pkg
 197          * and pkg-static.
 198          */
 199         if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0) {
 200                 /* force a fresh template */
 201                 newtemplate = DoCreateTemplate(1);
 202         } else {
 203                 newtemplate = DoCreateTemplate(0);
 204         }
 205
 206         /*
 207          * This will clear the screen and set-up our gui, so sleep
 208          * a little first in case the user wants to see what was
 209          * printed before.
 210          */
 211         sleep(2);
 212         pthread_mutex_lock(&WorkerMutex);
 213         startTime = time(NULL);
 214         RunStatsInit();
 215         RunStatsReset();
 216
 217         /*
 218          * Build pkg/pkg-static.
 219          */
 220         if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0) {
 221                 build_list = scan;
 222                 build_tail = &scan->build_next;
 223                 startbuild(&build_list, &build_tail);
 224                 while (RunningWorkers == 1)
 225                         waitbuild(1, 0);
 226
 227                 if (scan->flags & PKGF_NOBUILD)
 228                         dfatal("Unable to build 'pkg'");
 229                 if (scan->flags & PKGF_ERROR)
 230                         dfatal("Error building 'pkg'");
 231                 if ((scan->flags & PKGF_SUCCESS) == 0)
 232                         dfatal("Error building 'pkg'");
 233                 newtemplate = 1;
 234         }
 235
 236         /*
 237          * Install pkg/pkg-static into the template
 238          */
 239         if (newtemplate) {
 240                 char *buf;
 241                 int rc;
 242
 243                 asprintf(&buf,
 244                          "cd %s/Template; "
 245                          "tar --exclude '+*' --exclude '*/man/*' "
 246                          "-xvzpf %s/%s > /dev/null 2>&1",
 247                          BuildBase,
 248                          RepositoryPath,
 249                          scan->pkgfile);
 250                 rc = system(buf);
 251                 if (rc)
 252                         dfatal("Command failed: %s\n", buf);
 253                 freestrp(&buf);
 254         }
 255
 256         /*
 257          * Calculate depi_depth, the longest chain of dependencies
 258          * for who depends on me, weighted by powers of two.
 259          */
 260         for (scan = pkgs; scan; scan = scan->bnext) {
 261                 buildCalculateDepiDepth(scan);
 262         }
 263
 264         /*
 265          * Nominal bulk build sequence
 266          */
 267         while (haswork) {
 268                 haswork = 0;
 269                 fflush(stdout);
 270                 for (scan = pkgs; scan; scan = scan->bnext) {
 271                         ddprintf(0, "SCANLEAVES %08x %s\n",
 272                                  scan->flags, scan->portdir);
 273                         scan->flags |= PKGF_BUILDLOOP;
 274                         /*
 275                          * NOTE: We must still find dependencies if PACKAGED
 276                          *       to fill in the gaps, as some of them may
 277                          *       need to be rebuilt.
 278                          */
 279                         if (scan->flags & (PKGF_SUCCESS | PKGF_FAILURE |
 280                                            PKGF_ERROR)) {
 281 #if 0
 282                                 ddprintf(0, "%s: already built\n",
 283                                          scan->portdir);
 284 #endif
 285                         } else {
 286                                 int ap = 0;
 287                                 build_find_leaves(NULL, scan, &build_tail,
 288                                                   &ap, &haswork, 0, first, 0);
 289                                 ddprintf(0, "TOPLEVEL %s %08x\n",
 290                                          scan->portdir, ap);
 291                         }
 292                         scan->flags &= ~PKGF_BUILDLOOP;
 293                         build_clear_trav(scan);
 294                 }
 295                 first = 0;
 296                 fflush(stdout);
 297                 startbuild(&build_list, &build_tail);
 298
 299                 if (haswork == 0 && RunningWorkers) {
 300                         waitbuild(RunningWorkers, 1);
 301                         haswork = 1;
 302                 }
 303         }
 304         pthread_mutex_unlock(&WorkerMutex);
 305
 306         RunStatsUpdateTop();
 307         RunStatsUpdateLogs();
 308         RunStatsSync();
 309         RunStatsDone();
 310
 311         doHook(NULL, "hook_run_end", HookRunEnd, 1);
 312         if (UsingHooks) {
 313                 while ((bulk = getbulk()) != NULL)
 314                         freebulk(bulk);
 315                 donebulk();
 316         }
 317
 318         t = time(NULL) - startTime;
 319         h = t / 3600;
 320         m = t / 60 % 60;
 321         s = t % 60;
 322
 323         dlog(DLOG_ALL|DLOG_STDOUT,
 324                 "\n"
 325                 "Initial queue size: %d\n"
 326                 "    packages built: %d\n"
 327                 "           ignored: %d\n"
 328                 "           skipped: %d\n"
 329                 "            failed: %d\n"
 330                 "\n"
 331                 "Duration: %02d:%02d:%02d\n"
 332                 "\n",
 333                 BuildTotal,
 334                 BuildSuccessCount,
 335                 BuildIgnoreCount,
 336                 BuildSkipCount,
 337                 BuildFailCount,
 338                 h, m, s);
 339 }
 340
 341 /*
 342  * Traverse the packages (pkg) depends on recursively until we find
 343  * a leaf to build or report as unbuildable.  Calculates and assigns a
 344  * dependency count.  Returns all parallel-buildable packages.
 345  *
 346  * (pkg) itself is only added to the list if it is immediately buildable.
 347  */
 348 static
 349 int
 350 build_find_leaves(pkg_t *parent, pkg_t *pkg, pkg_t ***build_tailp,
 351                   int *app, int *hasworkp, int depth, int first,
 352                   int first_one_only)
 353 {
 354         pkglink_t *link;
 355         pkg_t *scan;
 356         int idep_count = 0;
 357         int apsub;
 358         int dfirst_one_only;
 359         int ndepth;
 360         char *buf;
 361
 362         ndepth = depth + 1;
 363
 364         /*
 365          * Already on build list, possibly in-progress, tell caller that
 366          * it is not ready.
 367          */
 368         ddprintf(ndepth, "sbuild_find_leaves %d %s %08x {\n",
 369                  depth, pkg->portdir, pkg->flags);
 370         if (pkg->flags & PKGF_BUILDLIST) {
 371                 ddprintf(ndepth, "} (already on build list)\n");
 372                 *app |= PKGF_NOTREADY;
 373                 return (pkg->idep_count);
 374         }
 375
 376         /*
 377          * Check dependencies
 378          */
 379         PKGLIST_FOREACH(link, &pkg->idepon_list) {
 380                 scan = link->pkg;
 381
 382                 if (scan == NULL) {
 383                         if (first_one_only)
 384                                 break;
 385                         continue;
 386                 }
 387                 ddprintf(ndepth, "check %s %08x\t", scan->portdir, scan->flags);
 388
 389                 /*
 390                  * If this dependency is to a DUMMY node it is a dependency
 391                  * only on the default flavor which is only the first node
 392                  * under this one, not all of them.
 393                  *
 394                  * NOTE: The depth is not being for complex dependency type
 395                  *       tests like it is in childInstallPkgDeps_recurse(),
 396                  *       so we don't have to hicup it like we do in that
 397                  *       procedure.
 398                  */
 399                 dfirst_one_only = (scan->flags & PKGF_DUMMY) ? 1 : 0;
 400
 401                 /*
 402                  * When accounting for a successful build, just bump
 403                  * idep_count by one.  scan->idep_count will heavily
 404                  * overlap packages that we count down multiple branches.
 405                  *
 406                  * We must still recurse through PACKAGED packages as
 407                  * some of their dependencies might be missing.
 408                  */
 409                 if (scan->flags & PKGF_SUCCESS) {
 410                         ddprintf(0, "SUCCESS - OK\n");
 411                         ++idep_count;
 412                         if (first_one_only)
 413                                 break;
 414                         continue;
 415                 }
 416
 417                 /*
 418                  * ERROR includes FAILURE, which is set in numerous situations
 419                  * including when NOBUILD state is processed.  So check for
 420                  * NOBUILD state first.
 421                  *
 422                  * An ERROR in a sub-package causes a NOBUILD in packages
 423                  * that depend on it.
 424                  */
 425                 if (scan->flags & PKGF_NOBUILD) {
 426                         ddprintf(0, "NOBUILD - OK "
 427                                     "(propogate failure upward)\n");
 428                         *app |= PKGF_NOBUILD_S;
 429                         if (first_one_only)
 430                                 break;
 431                         continue;
 432                 }
 433                 if (scan->flags & PKGF_ERROR) {
 434                         ddprintf(0, "ERROR - OK (propogate failure upward)\n");
 435                         *app |= PKGF_NOBUILD_S;
 436                         if (first_one_only)
 437                                 break;
 438                         continue;
 439                 }
 440
 441                 /*
 442                  * If already on build-list this dependency is not ready.
 443                  */
 444                 if (scan->flags & PKGF_BUILDLIST) {
 445                         ddprintf(0, " [BUILDLIST]");
 446                         *app |= PKGF_NOTREADY;
 447                 }
 448
 449                 /*
 450                  * If not packaged this dependency is not ready for
 451                  * the caller.
 452                  */
 453                 if ((scan->flags & PKGF_PACKAGED) == 0) {
 454                         ddprintf(0, " [NOT_PACKAGED]");
 455                         *app |= PKGF_NOTREADY;
 456                 }
 457
 458                 /*
 459                  * Reduce search complexity, if we have already processed
 460                  * scan in the traversal it will either already be on the
 461                  * build list or it will not be buildable.  Either way
 462                  * the parent is not buildable.
 463                  */
 464                 if (scan->flags & PKGF_BUILDTRAV) {
 465                         ddprintf(0, " [BUILDTRAV]\n");
 466                         *app |= PKGF_NOTREADY;
 467                         if (first_one_only)
 468                                 break;
 469                         continue;
 470                 }
 471
 472                 /*
 473                  * Assert on dependency loop
 474                  */
 475                 ++idep_count;
 476                 if (scan->flags & PKGF_BUILDLOOP) {
 477                         dfatal("pkg dependency loop %s -> %s",
 478                                 parent->portdir, scan->portdir);
 479                 }
 480
 481                 /*
 482                  * NOTE: For debug tabbing purposes we use (ndepth + 1)
 483                  *       here (i.e. depth + 2) in our iteration.
 484                  */
 485                 scan->flags |= PKGF_BUILDLOOP;
 486                 apsub = 0;
 487                 ddprintf(0, " SUBRECURSION {\n");
 488                 idep_count += build_find_leaves(pkg, scan, build_tailp,
 489                                                 &apsub, hasworkp,
 490                                                 ndepth + 1, first,
 491                                                 dfirst_one_only);
 492                 scan->flags &= ~PKGF_BUILDLOOP;
 493                 *app |= apsub;
 494                 if (apsub & PKGF_NOBUILD) {
 495                         ddprintf(ndepth, "} (sub-nobuild)\n");
 496                 } else if (apsub & PKGF_ERROR) {
 497                         ddprintf(ndepth, "} (sub-error)\n");
 498                 } else if (apsub & PKGF_NOTREADY) {
 499                         ddprintf(ndepth, "} (sub-notready)\n");
 500                 } else {
 501                         ddprintf(ndepth, "} (sub-ok)\n");
 502                 }
 503                 if (first_one_only)
 504                         break;
 505         }
 506         pkg->idep_count = idep_count;
 507         pkg->flags |= PKGF_BUILDTRAV;
 508
 509         /*
 510          * Incorporate scan results into pkg state.
 511          */
 512         if ((pkg->flags & PKGF_NOBUILD) == 0 && (*app & PKGF_NOBUILD)) {
 513                 *hasworkp = 1;
 514         } else if ((pkg->flags & PKGF_ERROR) == 0 && (*app & PKGF_ERROR)) {
 515                 *hasworkp = 1;
 516         }
 517         pkg->flags |= *app & ~PKGF_NOTREADY;
 518
 519         /*
 520          * Clear PACKAGED bit if sub-dependencies aren't clean
 521          */
 522         if ((pkg->flags & PKGF_PACKAGED) &&
 523             (pkg->flags & (PKGF_NOTREADY|PKGF_ERROR|PKGF_NOBUILD))) {
 524                 pkg->flags &= ~PKGF_PACKAGED;
 525                 ddassert(pkg->pkgfile);
 526                 asprintf(&buf, "%s/%s", RepositoryPath, pkg->pkgfile);
 527                 if (remove(buf) < 0) {
 528                         dlog(DLOG_ALL,
 529                              "[XXX] %s DELETE-PACKAGE %s (failed)\n",
 530                              pkg->portdir, buf);
 531                 } else {
 532                         dlog(DLOG_ALL,
 533                              "[XXX] %s DELETE-PACKAGE %s "
 534                              "(due to dependencies)\n",
 535                              pkg->portdir, buf);
 536                 }
 537                 freestrp(&buf);
 538                 *hasworkp = 1;
 539         }
 540
 541         /*
 542          * Set PKGF_NOBUILD_I if there is IGNORE data
 543          */
 544         if (pkg->ignore)
 545                 pkg->flags |= PKGF_NOBUILD_I;
 546
 547         /*
 548          * Handle propagated flags
 549          */
 550         if (pkg->flags & PKGF_ERROR) {
 551                 /*
 552                  * This can only happen if the ERROR has already been
 553                  * processed and accounted for.
 554                  */
 555                 ddprintf(depth, "} (ERROR - %s)\n", pkg->portdir);
 556         } else if (*app & PKGF_NOTREADY) {
 557                 /*
 558                  * We don't set PKGF_NOTREADY in the pkg, it is strictly
 559                  * a transient flag propagated via build_find_leaves().
 560                  *
 561                  * Just don't add the package to the list.
 562                  *
 563                  * NOTE: Even if NOBUILD is set (meaning we could list it
 564                  *       and let startbuild() finish it up as a skip, we
 565                  *       don't process it to the list because we want to
 566                  *       process all the dependencies, so someone doing a
 567                  *       manual build can get more complete information and
 568                  *       does not have to iterate each failed dependency one
 569                  *       at a time.
 570                  */
 571                 ;
 572         } else if (pkg->flags & PKGF_SUCCESS) {
 573                 ddprintf(depth, "} (SUCCESS - %s)\n", pkg->portdir);
 574         } else if (pkg->flags & PKGF_DUMMY) {
 575                 /*
 576                  * Just mark dummy packages as successful when all of their
 577                  * sub-depends (flavors) complete successfully.  Note that
 578                  * dummy packages are not counted in the total, so do not
 579                  * decrement BuildTotal.
 580                  */
 581                 ddprintf(depth, "} (DUMMY/META - SUCCESS)\n");
 582                 pkg->flags |= PKGF_SUCCESS;
 583                 *hasworkp = 1;
 584                 if (first) {
 585                         dlog(DLOG_ALL | DLOG_FILTER,
 586                              "[XXX] %s META-ALREADY-BUILT\n",
 587                              pkg->portdir);
 588                 } else {
 589                         dlog(DLOG_SUCC, "[XXX] %s meta-node complete\n",
 590                              pkg->portdir);
 591                 }
 592         } else if (pkg->flags & PKGF_PACKAGED) {
 593                 /*
 594                  * We can just mark the pkg successful.  If this is
 595                  * the first pass, we count this as an initial pruning
 596                  * pass and reduce BuildTotal.
 597                  */
 598                 ddprintf(depth, "} (PACKAGED - SUCCESS)\n");
 599                 pkg->flags |= PKGF_SUCCESS;
 600                 *hasworkp = 1;
 601                 if (first) {
 602                         dlog(DLOG_ALL | DLOG_FILTER,
 603                              "[XXX] %s ALREADY-BUILT\n",
 604                              pkg->portdir);
 605                         --BuildTotal;
 606                 }
 607         } else {
 608                 /*
 609                  * All dependencies are successful, queue new work
 610                  * and indicate not-ready to the parent (since our
 611                  * package has to be built).
 612                  *
 613                  * NOTE: The NOBUILD case propagates to here as well
 614                  *       and is ultimately handled by startbuild().
 615                  */
 616                 *hasworkp = 1;
 617                 if (pkg->flags & PKGF_NOBUILD_I)
 618                         ddprintf(depth, "} (ADDLIST(IGNORE/BROKEN) - %s)\n",
 619                                  pkg->portdir);
 620                 else if (pkg->flags & PKGF_NOBUILD)
 621                         ddprintf(depth, "} (ADDLIST(NOBUILD) - %s)\n",
 622                                  pkg->portdir);
 623                 else
 624                         ddprintf(depth, "} (ADDLIST - %s)\n", pkg->portdir);
 625                 pkg->flags |= PKGF_BUILDLIST;
 626                 **build_tailp = pkg;
 627                 *build_tailp = &pkg->build_next;
 628                 *app |= PKGF_NOTREADY;
 629         }
 630
 631         return idep_count;
 632 }
 633
 634 static
 635 void
 636 build_clear_trav(pkg_t *pkg)
 637 {
 638         pkglink_t *link;
 639         pkg_t *scan;
 640
 641         pkg->flags &= ~PKGF_BUILDTRAV;
 642         PKGLIST_FOREACH(link, &pkg->idepon_list) {
 643                 scan = link->pkg;
 644                 if (scan && (scan->flags & PKGF_BUILDTRAV))
 645                         build_clear_trav(scan);
 646         }
 647 }
 648
 649 /*
 650  * Calculate the longest chain of packages that depend on me.  The
 651  * long the chain, the more important my package is to build earlier
 652  * rather than later.
 653  */
 654 static int
 655 buildCalculateDepiDepth(pkg_t *pkg)
 656 {
 657         pkglink_t *link;
 658         pkg_t *scan;
 659         int best_depth = 0;
 660         int res;
 661
 662         if (pkg->depi_depth)
 663                 return(pkg->depi_depth + 1);
 664         pkg->flags |= PKGF_BUILDLOOP;
 665         PKGLIST_FOREACH(link, &pkg->deponi_list) {
 666                 scan = link->pkg;
 667                 if (scan && (scan->flags & PKGF_BUILDLOOP) == 0) {
 668                         res = buildCalculateDepiDepth(scan);
 669                         if (best_depth < res)
 670                                 best_depth = res;
 671                 }
 672         }
 673         pkg->flags &= ~PKGF_BUILDLOOP;
 674         pkg->depi_depth = best_depth;
 675
 676         return (best_depth + 1);
 677 }
 678
 679 /*
 680  * Take a list of pkg ready to go, sort it, and assign it to worker
 681  * slots.  This routine blocks in waitbuild() until it can dispose of
 682  * the entire list.
 683  *
 684  * WorkerMutex is held by the caller.
 685  */
 686 static
 687 void
 688 startbuild(pkg_t **build_listp, pkg_t ***build_tailp)
 689 {
 690         pkg_t *pkg;
 691         pkg_t **idep_ary;
 692         pkg_t **depi_ary;
 693         int count;
 694         int idep_index;
 695         int depi_index;
 696         int i;
 697         int n;
 698         worker_t *work;
 699         static int IterateWorker;
 700
 701         /*
 702          * Nothing to do
 703          */
 704         if (*build_listp == NULL)
 705                 return;
 706
 707         /*
 708          * Sort
 709          */
 710         count = 0;
 711         for (pkg = *build_listp; pkg; pkg = pkg->build_next)
 712                 ++count;
 713         idep_ary = calloc(count, sizeof(pkg_t *));
 714         depi_ary = calloc(count, sizeof(pkg_t *));
 715
 716         count = 0;
 717         for (pkg = *build_listp; pkg; pkg = pkg->build_next) {
 718                 idep_ary[count] = pkg;
 719                 depi_ary[count] = pkg;
 720                 ++count;
 721         }
 722
 723         /*
 724          * idep_ary - sorted by #of dependencies this pkg has.
 725          * depi_ary - sorted by #of other packages that depend on this pkg.
 726          */
 727         qsort(idep_ary, count, sizeof(pkg_t *), qsort_idep);
 728         qsort(depi_ary, count, sizeof(pkg_t *), qsort_depi);
 729
 730         idep_index = 0;
 731         depi_index = 0;
 732
 733         /*
 734          * Half the workers build based on the highest depi count,
 735          * the other half build based on the highest idep count.
 736          *
 737          * This is an attempt to get projects which many other projects
 738          * depend on built first, but to also try to build large projects
 739          * (which tend to have a lot of dependencies) earlier rather than
 740          * later so the end of the bulk run doesn't inefficiently build
 741          * the last few huge projects.
 742          *
 743          * Loop until we manage to assign slots to everyone.  We do not
 744          * wait for build completion.
 745          *
 746          * This is the point where we handle DUMMY packages (these are
 747          * dummy unflavored packages which 'cover' all the flavors for
 748          * a package).  These are not real packages are marked SUCCESS
 749          * at this time because their dependencies (the flavors) have all
 750          * been built.
 751          */
 752         while (idep_index != count || depi_index != count) {
 753                 pkg_t *pkgi;
 754                 pkg_t *ipkg;
 755
 756                 /*
 757                  * Find candidate to start sorted by depi or idep.
 758                  */
 759                 ipkg = NULL;
 760                 while (idep_index < count) {
 761                         ipkg = idep_ary[idep_index];
 762                         if ((ipkg->flags &
 763                              (PKGF_SUCCESS | PKGF_FAILURE |
 764                               PKGF_ERROR | PKGF_RUNNING)) == 0) {
 765                                 break;
 766                         }
 767                         ipkg = NULL;
 768                         ++idep_index;
 769                 }
 770
 771                 pkgi = NULL;
 772                 while (depi_index < count) {
 773                         pkgi = depi_ary[depi_index];
 774                         if ((pkgi->flags &
 775                              (PKGF_SUCCESS | PKGF_FAILURE |
 776                               PKGF_ERROR | PKGF_RUNNING)) == 0) {
 777                                 break;
 778                         }
 779                         pkgi = NULL;
 780                         ++depi_index;
 781                 }
 782
 783                 /*
 784                  * ipkg and pkgi must either both be NULL, or both
 785                  * be non-NULL.
 786                  */
 787                 if (ipkg == NULL && pkgi == NULL)
 788                         break;
 789                 ddassert(ipkg && pkgi);
 790
 791                 /*
 792                  * Handle the NOBUILD case right here, there's no point
 793                  * queueing it anywhere.
 794                  */
 795                 if (ipkg->flags & PKGF_NOBUILD) {
 796                         char *reason;
 797
 798                         ipkg->flags |= PKGF_FAILURE;
 799                         ipkg->flags &= ~PKGF_BUILDLIST;
 800
 801                         reason = buildskipreason(NULL, ipkg);
 802                         if (ipkg->flags & PKGF_NOBUILD_I) {
 803                                 ++BuildIgnoreCount;
 804                                 dlog(DLOG_IGN, "[XXX] %s ignored due to %s\n",
 805                                      ipkg->portdir, reason);
 806                                 doHook(ipkg, "hook_pkg_ignored",
 807                                        HookPkgIgnored, 0);
 808                         } else {
 809                                 ++BuildSkipCount;
 810                                 dlog(DLOG_SKIP, "[XXX] %s skipped due to %s\n",
 811                                      ipkg->portdir, reason);
 812                                 doHook(ipkg, "hook_pkg_skipped",
 813                                        HookPkgSkipped, 0);
 814                         }
 815                         free(reason);
 816                         ++BuildCount;
 817                         continue;
 818                 }
 819                 if (pkgi->flags & PKGF_NOBUILD) {
 820                         char *reason;
 821
 822                         pkgi->flags |= PKGF_FAILURE;
 823                         pkgi->flags &= ~PKGF_BUILDLIST;
 824
 825                         reason = buildskipreason(NULL, pkgi);
 826                         if (pkgi->flags & PKGF_NOBUILD_I) {
 827                                 ++BuildIgnoreCount;
 828                                 dlog(DLOG_IGN, "[XXX] %s ignored due to %s\n",
 829                                      pkgi->portdir, reason);
 830                                 doHook(pkgi, "hook_pkg_ignored",
 831                                        HookPkgIgnored, 0);
 832                         } else {
 833                                 ++BuildSkipCount;
 834                                 dlog(DLOG_SKIP, "[XXX] %s skipped due to %s\n",
 835                                      pkgi->portdir, reason);
 836                                 doHook(pkgi, "hook_pkg_skipped",
 837                                        HookPkgSkipped, 0);
 838                         }
 839                         free(reason);
 840                         ++BuildCount;
 841                         continue;
 842                 }
 843
 844                 /*
 845                  * Block while no slots are available.  waitbuild()
 846                  * will clean out any DONE states.
 847                  */
 848                 while (RunningWorkers >= DynamicMaxWorkers ||
 849                        RunningWorkers >= MaxWorkers - FailedWorkers) {
 850                         waitbuild(RunningWorkers, 1);
 851                 }
 852
 853                 /*
 854                  * Find an available worker slot, there should be at
 855                  * least one.
 856                  */
 857                 for (i = 0; i < MaxWorkers; ++i) {
 858                         n = IterateWorker % MaxWorkers;
 859                         work = &WorkerAry[n];
 860
 861                         if (work->state == WORKER_DONE ||
 862                             work->state == WORKER_FAILED) {
 863                                 workercomplete(work);
 864                         }
 865                         if (work->state == WORKER_NONE ||
 866                             work->state == WORKER_IDLE) {
 867                                 if (n <= MaxWorkers / 2) {
 868                                         startworker(pkgi, work);
 869                                 } else {
 870                                         startworker(ipkg, work);
 871                                 }
 872                                 /*RunStatsUpdate(work);*/
 873                                 break;
 874                         }
 875                         ++IterateWorker;
 876                 }
 877                 ddassert(i != MaxWorkers);
 878         }
 879         RunStatsSync();
 880
 881         /*
 882          * We disposed of the whole list
 883          */
 884         free(idep_ary);
 885         free(depi_ary);
 886         *build_listp = NULL;
 887         *build_tailp = build_listp;
 888 }
 889
 890 typedef const pkg_t *pkg_tt;
 891
 892 static int
 893 qsort_idep(const void *pkg1_arg, const void *pkg2_arg)
 894 {
 895         const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg;
 896         const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg;
 897
 898         return (pkg2->idep_count - pkg1->idep_count);
 899 }
 900
 901 static int
 902 qsort_depi(const void *pkg1_arg, const void *pkg2_arg)
 903 {
 904         const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg;
 905         const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg;
 906
 907         return ((pkg2->depi_count * pkg2->depi_depth) -
 908                 (pkg1->depi_count * pkg1->depi_depth));
 909 }
 910
 911 /*
 912  * Frontend starts a pkg up on a worker
 913  *
 914  * WorkerMutex must be held.
 915  */
 916 static void
 917 startworker(pkg_t *pkg, worker_t *work)
 918 {
 919         switch(work->state) {
 920         case WORKER_NONE:
 921                 pthread_create(&work->td, NULL, childBuilderThread, work);
 922                 work->state = WORKER_IDLE;
 923                 /* fall through */
 924         case WORKER_IDLE:
 925                 work->pkg_dep_size =
 926                 childInstallPkgDeps_recurse(NULL, &pkg->idepon_list, 0, 1, 0);
 927                 childInstallPkgDeps_recurse(NULL, &pkg->idepon_list, 1, 1, 0);
 928                 RunningPkgDepSize += work->pkg_dep_size;
 929
 930                 dlog(DLOG_ALL, "[%03d] START   %s "
 931                                "##idep=%02d depi=%02d/%02d dep=%-4.2fG\n",
 932                      work->index, pkg->portdir,
 933                      pkg->idep_count, pkg->depi_count, pkg->depi_depth,
 934                      (double)work->pkg_dep_size / (double)ONEGB);
 935
 936                 cleanworker(work);
 937                 pkg->flags |= PKGF_RUNNING;
 938                 work->pkg = pkg;
 939                 pthread_cond_signal(&work->cond);
 940                 ++RunningWorkers;
 941                 /*RunStatsUpdate(work);*/
 942                 break;
 943         case WORKER_PENDING:
 944         case WORKER_RUNNING:
 945         case WORKER_DONE:
 946         case WORKER_FAILED:
 947         case WORKER_FROZEN:
 948         case WORKER_EXITING:
 949         default:
 950                 dfatal("startworker: [%03d] Unexpected state %d for worker %d",
 951                        work->index, work->state, work->index);
 952                 break;
 953         }
 954 }
 955
 956 static void
 957 cleanworker(worker_t *work)
 958 {
 959         work->state = WORKER_PENDING;
 960         work->flags = 0;
 961         work->accum_error = 0;
 962         work->start_time = time(NULL);
 963 }
 964
 965 /*
 966  * Frontend finishes up a completed pkg on a worker.
 967  *
 968  * If the worker is in a FAILED state we clean the pkg out but (for now)
 969  * leave it in its failed state so we can debug.  At this point
 970  * workercomplete() will be called every once in a while on the state
 971  * and we have to deal with the NULL pkg.
 972  *
 973  * WorkerMutex must be held.
 974  */
 975 static void
 976 workercomplete(worker_t *work)
 977 {
 978         pkg_t *pkg;
 979         time_t t;
 980         int h;
 981         int m;
 982         int s;
 983
 984         /*
 985          * Steady state FAILED case.
 986          */
 987         if (work->state == WORKER_FAILED) {
 988                 if (work->pkg == NULL)
 989                         return;
 990         }
 991
 992         t = time(NULL) - work->start_time;
 993         h = t / 3600;
 994         m = t / 60 % 60;
 995         s = t % 60;
 996
 997         /*
 998          * Reduce total dep size
 999          */
1000         RunningPkgDepSize -= work->pkg_dep_size;
1001         RunningPkgDepSize -= work->memuse;
1002         work->pkg_dep_size = 0;
1003         work->memuse = 0;
1004
1005         /*
1006          * Process pkg out of the worker
1007          */
1008         pkg = work->pkg;
1009         if (pkg->flags & (PKGF_ERROR|PKGF_NOBUILD)) {
1010                 pkg->flags |= PKGF_FAILURE;
1011
1012                 /*
1013                  * This NOBUILD condition XXX can occur if the package is
1014                  * not allowed to be built.
1015                  */
1016                 if (pkg->flags & PKGF_NOBUILD) {
1017                         char *reason;
1018
1019                         reason = buildskipreason(NULL, pkg);
1020                         if (pkg->flags & PKGF_NOBUILD_I) {
1021                                 ++BuildIgnoreCount;
1022                                 dlog(DLOG_SKIP, "[%03d] IGNORD %s - %s\n",
1023                                      work->index, pkg->portdir, reason);
1024                                 doHook(pkg, "hook_pkg_ignored",
1025                                        HookPkgIgnored, 0);
1026                         } else {
1027                                 ++BuildSkipCount;
1028                                 dlog(DLOG_SKIP, "[%03d] SKIPPD %s - %s\n",
1029                                      work->index, pkg->portdir, reason);
1030                                 doHook(pkg, "hook_pkg_skipped",
1031                                        HookPkgSkipped, 0);
1032                         }
1033                         free(reason);
1034                 } else {
1035                         ++BuildFailCount;
1036                         dlog(DLOG_FAIL | DLOG_RED,
1037                              "[%03d] FAILURE %s ##%16.16s %02d:%02d:%02d\n",
1038                              work->index, pkg->portdir,
1039                              getphasestr(work->phase),
1040                              h, m, s);
1041                         doHook(pkg, "hook_pkg_failure", HookPkgFailure, 0);
1042                 }
1043         } else {
1044                 pkg->flags |= PKGF_SUCCESS;
1045                 ++BuildSuccessCount;
1046                 dlog(DLOG_SUCC | DLOG_GRN,
1047                      "[%03d] SUCCESS %s ##%02d:%02d:%02d\n",
1048                      work->index, pkg->portdir, h, m, s);
1049                 doHook(pkg, "hook_pkg_success", HookPkgSuccess, 0);
1050         }
1051         ++BuildCount;
1052         pkg->flags &= ~PKGF_BUILDLIST;
1053         pkg->flags &= ~PKGF_RUNNING;
1054         work->pkg = NULL;
1055         --RunningWorkers;
1056
1057         if (work->state == WORKER_FAILED) {
1058                 dlog(DLOG_ALL, "[%03d] XXX/XXX WORKER IS IN A FAILED STATE\n",
1059                      work->index);
1060                 ++FailedWorkers;
1061         } else if (work->flags & WORKERF_FREEZE) {
1062                 dlog(DLOG_ALL, "[%03d] FROZEN(DEBUG) %s\n",
1063                      work->index, pkg->portdir);
1064                 work->state = WORKER_FROZEN;
1065         } else {
1066                 work->state = WORKER_IDLE;
1067         }
1068 }
1069
1070 /*
1071  * Wait for one or more workers to complete.
1072  *
1073  * WorkerMutex must be held.
1074  */
1075 static void
1076 waitbuild(int whilematch, int dynamicmax)
1077 {
1078         static time_t wblast_time;
1079         static time_t dmlast_time;
1080         struct timespec ts;
1081         worker_t *work;
1082         time_t t;
1083         int i;
1084
1085         if (whilematch == 0)
1086                 whilematch = 1;
1087
1088         while (RunningWorkers == whilematch) {
1089                 for (i = 0; i < MaxWorkers; ++i) {
1090                         work = &WorkerAry[i];
1091                         if (work->state == WORKER_DONE ||
1092                             work->state == WORKER_FAILED) {
1093                                 workercomplete(work);
1094                         } else {
1095                                 pthread_cond_signal(&work->cond);
1096                         }
1097                         RunStatsUpdate(work, NULL);
1098                 }
1099                 RunStatsUpdateTop();
1100                 RunStatsUpdateLogs();
1101                 RunStatsSync();
1102                 if (RunningWorkers == whilematch) {
1103                         clock_gettime(CLOCK_REALTIME, &ts);
1104                         ts.tv_sec += 1;
1105                         ts.tv_nsec = 0;
1106                         pthread_cond_timedwait(&WorkerCond, &WorkerMutex, &ts);
1107                 }
1108
1109                 /*
1110                  * Dynamically reduce MaxWorkers based on the load.  When
1111                  * the load exceeds 2 x ncpus we reduce the number of workers
1112                  * up to 75% of MaxWorkers @ (5 x ncpus) load.
1113                  *
1114                  * Dynamically reduce MaxWorkers based on swap use, starting
1115                  * at 10% swap and up to 75% of MaxWorkers at 40% swap.
1116                  *
1117                  * NOTE! Generally speaking this allows more workers to be
1118                  *       configured which helps in two ways.  First it allows
1119                  *       a higher build rate for smaller packages.  Second
1120                  *       it allows dsynth to ratchet-down the number of slots
1121                  *       when large packages are forcing the load up.
1122                  *
1123                  *       A high load doesn't hurt efficiency, but swap usage
1124                  *       due to loading the tmpfs in lots of worker slots up
1125                  *       with tons of pkg install's (pre-reqs for a build)
1126                  *       does.  Reducing the number of worker slots has a
1127                  *       huge beneficial effect on reducing swap use / paging.
1128                  */
1129                 t = time(NULL);
1130                 if (dynamicmax && (wblast_time == 0 ||
1131                                    (unsigned)(t - wblast_time) >= 5)) {
1132                         double min_load = 1.5 * NumCores;
1133                         double max_load = 5.0 * NumCores;
1134                         double min_swap = 0.10;
1135                         double max_swap = 0.40;
1136                         double dload[3];
1137                         double dswap;
1138                         int max1;
1139                         int max2;
1140                         int max3;
1141                         int max_sel;
1142                         int noswap;
1143
1144                         wblast_time = t;
1145
1146                         /*
1147                          * Cap based on load.  This is back-loaded.
1148                          */
1149                         getloadavg(dload, 3);
1150                         if (dload[0] < min_load) {
1151                                 max1 = MaxWorkers;
1152                         } else if (dload[0] <= max_load) {
1153                                 max1 = MaxWorkers -
1154                                        MaxWorkers * 0.75 *
1155                                        (dload[0] - min_load) /
1156                                        (max_load - min_load);
1157                         } else {
1158                                 max1 = MaxWorkers * 25 / 100;
1159                         }
1160
1161                         /*
1162                          * Cap based on swap use.  This is back-loaded.
1163                          */
1164                         dswap = getswappct(&noswap);
1165                         if (dswap < min_swap) {
1166                                 max2 = MaxWorkers;
1167                         } else if (dswap <= max_swap) {
1168                                 max2 = MaxWorkers -
1169                                        MaxWorkers * 0.75 *
1170                                        (dswap - min_swap) /
1171                                        (max_swap - min_swap);
1172                         } else {
1173                                 max2 = MaxWorkers * 25 / 100;
1174                         }
1175
1176                         /*
1177                          * Cap based on aggregate pkg-dependency memory
1178                          * use installed in worker slots.  This is
1179                          * front-loaded.
1180                          *
1181                          * Since it can take a while for workers to retire
1182                          * (to reduce RunningPkgDepSize), just set our
1183                          * target 1 below the current run count to allow
1184                          * jobs to retire without being replaced with new
1185                          * jobs.
1186                          *
1187                          * In addition, in order to avoid a paging 'shock',
1188                          * We enforce a 30 second-per-increment slow-start
1189                          * once RunningPkgDepSize exceeds 1/2 the target.
1190                          */
1191                         if (RunningPkgDepSize > PkgDepMemoryTarget) {
1192                                 max3 = RunningWorkers - 1;
1193                         } else if (RunningPkgDepSize > PkgDepMemoryTarget / 2) {
1194                                 if (dmlast_time == 0 ||
1195                                     (unsigned)(t - dmlast_time) >= 30) {
1196                                         dmlast_time = t;
1197                                         max3 = RunningWorkers + 1;
1198                                 } else {
1199                                         max3 = RunningWorkers;
1200                                 }
1201                         } else {
1202                                 max3 = MaxWorkers;
1203                         }
1204
1205                         /*
1206                          * Priority reduction, convert to DynamicMaxWorkers
1207                          */
1208                         max_sel = max1;
1209                         if (max_sel > max2)
1210                                 max_sel = max2;
1211                         if (max_sel > max3)
1212                                 max_sel = max3;
1213
1214                         /*
1215                          * Restrict to allowed range, and also handle
1216                          * slow-start.
1217                          */
1218                         if (max_sel < 1)
1219                                 max_sel = 1;
1220                         if (max_sel > DynamicMaxWorkers + 1)
1221                                 max_sel = DynamicMaxWorkers + 1;
1222                         if (max_sel > MaxWorkers)
1223                                 max_sel = MaxWorkers;
1224
1225                         /*
1226                          * Stop waiting if DynamicMaxWorkers is going to
1227                          * increase.
1228                          */
1229                         if (DynamicMaxWorkers < max1)
1230                                 whilematch = -1;
1231
1232                         /*
1233                          * And adjust
1234                          */
1235                         if (DynamicMaxWorkers != max1) {
1236                                 dlog(DLOG_ALL | DLOG_FILTER,
1237                                      "[XXX] Load=%-6.2f(%2d) "
1238                                      "Swap=%-3.2f%%(%2d) "
1239                                      "Mem=%3.2fG(%2d) "
1240                                      "Adjust Workers %d->%d\n",
1241                                      dload[0], max1,
1242                                      dswap * 100.0, max2,
1243                                      RunningPkgDepSize / (double)ONEGB, max3,
1244                                      DynamicMaxWorkers, max_sel);
1245                                 DynamicMaxWorkers = max_sel;
1246                         }
1247                 }
1248         }
1249 }
1250
1251
1252 /*
1253  * Worker pthread (WorkerAry)
1254  *
1255  * This thread belongs to the dsynth master process and handled a worker slot.
1256  * (this_thread) -> WORKER fork/exec (WorkerPocess) -> (pty) -> sub-processes.
1257  */
1258 static void *
1259 childBuilderThread(void *arg)
1260 {
1261         char *envary[1] = { NULL };
1262         worker_t *work = arg;
1263         wmsg_t wmsg;
1264         pkg_t *pkg;
1265         pid_t pid;
1266         int status;
1267         volatile int dowait;
1268         char slotbuf[8];
1269         char fdbuf[8];
1270         char flagsbuf[16];
1271
1272         pthread_mutex_lock(&WorkerMutex);
1273         while (work->terminate == 0) {
1274                 dowait = 1;
1275
1276                 switch(work->state) {
1277                 case WORKER_IDLE:
1278                         break;
1279                 case WORKER_PENDING:
1280                         /*
1281                          * Fork the management process for the pkg operation
1282                          * on this worker slot.
1283                          *
1284                          * This process will set up the environment, do the
1285                          * mounts, will become the reaper, and will process
1286                          * pipe commands and handle chroot operations.
1287                          *
1288                          * NOTE: If SOCK_CLOEXEC is not supported WorkerMutex
1289                          *       is sufficient to interlock F_SETFD FD_CLOEXEC
1290                          *       operations.
1291                          */
1292                         ddassert(work->pkg);
1293                         if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC,
1294                                        PF_UNSPEC, work->fds)) {
1295                                 dfatal_errno("socketpair() during worker fork");
1296                         }
1297                         snprintf(slotbuf, sizeof(slotbuf),
1298                                  "%d", work->index);
1299                         snprintf(fdbuf, sizeof(fdbuf),
1300                                  "3");
1301                         snprintf(flagsbuf, sizeof(flagsbuf),
1302                                  "%d", WorkerProcFlags);
1303
1304                         /*
1305                          * fds[0] - master
1306                          * fds[1] - slave
1307                          *
1308                          * We pass the salve descriptor in fd 3 and close all
1309                          * other descriptors for security.
1310                          */
1311                         pthread_mutex_unlock(&WorkerMutex);
1312                         pid = vfork();
1313                         if (pid == 0) {
1314                                 close(work->fds[0]);
1315                                 dup2(work->fds[1], 3);
1316                                 closefrom(4);
1317                                 fcntl(3, F_SETFD, 0);
1318                                 execle(DSynthExecPath, DSynthExecPath,
1319                                        "WORKER", slotbuf, fdbuf,
1320                                        work->pkg->portdir, work->pkg->pkgfile,
1321                                        flagsbuf,
1322                                        NULL, envary);
1323                                 write(2, "EXECLE FAILURE\n", 15);
1324                                 _exit(1);
1325                         }
1326                         pthread_mutex_lock(&WorkerMutex);
1327                         close(work->fds[1]);
1328                         work->phase = PHASE_PENDING;
1329                         work->lines = 0;
1330                         work->memuse = 0;
1331                         work->pid = pid;
1332                         work->state = WORKER_RUNNING;
1333                         /* fall through */
1334                 case WORKER_RUNNING:
1335                         /*
1336                          * Poll for status updates, if NULL is returned
1337                          * and status is non-zero, the communications link
1338                          * failed unexpectedly.
1339                          */
1340                         pkg = work->pkg;
1341                         pthread_mutex_unlock(&WorkerMutex);
1342                         status = ipcreadmsg(work->fds[0], &wmsg);
1343                         pthread_mutex_lock(&WorkerMutex);
1344
1345                         if (status == 0) {
1346                                 /*
1347                                  * Normal message, can include normal
1348                                  * termination which changes us over
1349                                  * to another state.
1350                                  */
1351                                 dowait = 0;
1352                                 switch(wmsg.cmd) {
1353                                 case WMSG_CMD_INSTALL_PKGS:
1354                                         wmsg.cmd = WMSG_RES_INSTALL_PKGS;
1355                                         wmsg.status = childInstallPkgDeps(work);
1356                                         pthread_mutex_unlock(&WorkerMutex);
1357                                         ipcwritemsg(work->fds[0], &wmsg);
1358                                         pthread_mutex_lock(&WorkerMutex);
1359                                         break;
1360                                 case WMSG_CMD_STATUS_UPDATE:
1361                                         work->phase = wmsg.phase;
1362                                         work->lines = wmsg.lines;
1363                                         if (work->memuse != wmsg.memuse) {
1364                                                 RunningPkgDepSize +=
1365                                                 wmsg.memuse - work->memuse;
1366                                                 work->memuse = wmsg.memuse;
1367                                         }
1368                                         break;
1369                                 case WMSG_CMD_SUCCESS:
1370                                         work->flags |= WORKERF_SUCCESS;
1371                                         break;
1372                                 case WMSG_CMD_FAILURE:
1373                                         work->flags |= WORKERF_FAILURE;
1374                                         break;
1375                                 case WMSG_CMD_FREEZEWORKER:
1376                                         work->flags |= WORKERF_FREEZE;
1377                                         break;
1378                                 default:
1379                                         break;
1380                                 }
1381                                 RunStatsUpdate(work, NULL);
1382                                 RunStatsSync();
1383                         } else {
1384                                 close(work->fds[0]);
1385                                 pthread_mutex_unlock(&WorkerMutex);
1386                                 while (waitpid(work->pid, &status, 0) < 0 &&
1387                                        errno == EINTR) {
1388                                         ;
1389                                 }
1390                                 pthread_mutex_lock(&WorkerMutex);
1391
1392                                 if (work->flags & WORKERF_SUCCESS) {
1393                                         pkg->flags |= PKGF_SUCCESS;
1394                                         work->state = WORKER_DONE;
1395                                 } else if (work->flags & WORKERF_FAILURE) {
1396                                         pkg->flags |= PKGF_FAILURE;
1397                                         work->state = WORKER_DONE;
1398                                 } else {
1399                                         pkg->flags |= PKGF_FAILURE;
1400                                         work->state = WORKER_FAILED;
1401                                 }
1402                                 work->flags |= WORKERF_STATUS_UPDATE;
1403                                 pthread_cond_signal(&WorkerCond);
1404                         }
1405                         break;
1406                 case WORKER_DONE:
1407                         /*
1408                          * pkg remains attached until frontend processes the
1409                          * completion.  The frontend will then set the state
1410                          * back to idle.
1411                          */
1412                         break;
1413                 case WORKER_FAILED:
1414                         /*
1415                          * A worker failure means that the worker did not
1416                          * send us a WMSG_CMD_SUCCESS or WMSG_CMD_FAILURE
1417                          * ipc before terminating.
1418                          *
1419                          * We just sit in this state until the front-end
1420                          * does something about it.
1421                          */
1422                         break;
1423                 default:
1424                         dfatal("worker: [%03d] Unexpected state %d for worker %d",
1425                                work->index, work->state, work->index);
1426                         /* NOT REACHED */
1427                         break;
1428                 }
1429
1430                 /*
1431                  * The dsynth frontend will poll us approximately once
1432                  * a second (its variable).
1433                  */
1434                 if (dowait)
1435                         pthread_cond_wait(&work->cond, &WorkerMutex);
1436         }
1437
1438         /*
1439          * Scrap the comm socket if running, this should cause the worker
1440          * process to kill its sub-programs and cleanup.
1441          */
1442         if (work->state == WORKER_RUNNING) {
1443                 pthread_mutex_unlock(&WorkerMutex);
1444                 close(work->fds[0]);
1445                 while (waitpid(work->pid, &status, 0) < 0 &&
1446                        errno == EINTR);
1447                 pthread_mutex_lock(&WorkerMutex);
1448         }
1449
1450         /*
1451          * Final handshake
1452          */
1453         work->state = WORKER_EXITING;
1454         pthread_cond_signal(&WorkerCond);
1455         pthread_mutex_unlock(&WorkerMutex);
1456
1457         return NULL;
1458 }
1459
1460 /*
1461  * Install all the binary packages (we have already built them) that
1462  * the current work package depends on, without duplicates, in a script
1463  * which will be run from within the specified work jail.
1464  *
1465  * Locked by WorkerMutex (global)
1466  */
1467 static int
1468 childInstallPkgDeps(worker_t *work)
1469 {
1470         char *buf;
1471         FILE *fp;
1472
1473         if (PKGLIST_EMPTY(&work->pkg->idepon_list))
1474                 return 0;
1475
1476         asprintf(&buf, "%s/tmp/dsynth_install_pkgs", work->basedir);
1477         fp = fopen(buf, "w");
1478         ddassert(fp != NULL);
1479         fprintf(fp, "#!/bin/sh\n");
1480         fprintf(fp, "#\n");
1481         fchmod(fileno(fp), 0755);
1482
1483         childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 0, 1, 0);
1484         childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 1, 1, 0);
1485         fprintf(fp, "\nexit 0\n");
1486         fclose(fp);
1487         freestrp(&buf);
1488
1489         return 1;
1490 }
1491
1492 static size_t
1493 childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list, int undoit,
1494                             int depth, int first_one_only)
1495 {
1496         pkglink_t *link;
1497         pkg_t *pkg;
1498         size_t tot = 0;
1499         int ndepth;
1500         int nfirst;
1501
1502         PKGLIST_FOREACH(link, list) {
1503                 pkg = link->pkg;
1504
1505                 /*
1506                  * We don't want to mess up our depth test just below if
1507                  * a DUMMY node had to be inserted.  The nodes under the
1508                  * dummy node.
1509                  *
1510                  * The elements under a dummy node represent all the flabor,
1511                  * a dependency that directly references a dummy node only
1512                  * uses the first flavor (first_one_only / nfirst).
1513                  */
1514                 ndepth = (pkg->flags & PKGF_DUMMY) ? depth : depth + 1;
1515                 nfirst = (pkg->flags & PKGF_DUMMY) ? 1 : 0;
1516
1517                 /*
1518                  * We only need all packages for the top-level dependencies.
1519                  * The deeper ones only need DEP_TYPE_LIB and DEP_TYPE_RUN
1520                  * (types greater than DEP_TYPE_BUILD) since they are already
1521                  * built.
1522                  */
1523                 if (depth > 1 && link->dep_type <= DEP_TYPE_BUILD) {
1524                         if (first_one_only)
1525                                 break;
1526                         continue;
1527                 }
1528
1529                 if (undoit) {
1530                         if (pkg->dsynth_install_flg == 1) {
1531                                 pkg->dsynth_install_flg = 0;
1532                                 tot += childInstallPkgDeps_recurse(fp,
1533                                                             &pkg->idepon_list,
1534                                                             undoit,
1535                                                             ndepth, nfirst);
1536                         }
1537                         if (first_one_only)
1538                                 break;
1539                         continue;
1540                 }
1541                 if (pkg->dsynth_install_flg) {
1542                         if (DebugOpt >= 2 && pkg->pkgfile && fp) {
1543                                 fprintf(fp, "echo 'AlreadyHave %s'\n",
1544                                         pkg->pkgfile);
1545                         }
1546                         if (first_one_only)
1547                                 break;
1548                         continue;
1549                 }
1550
1551                 tot += childInstallPkgDeps_recurse(fp, &pkg->idepon_list,
1552                                                    undoit, ndepth, nfirst);
1553                 if (pkg->dsynth_install_flg) {
1554                         if (first_one_only)
1555                                 break;
1556                         continue;
1557                 }
1558                 pkg->dsynth_install_flg = 1;
1559
1560                 /*
1561                  * If this is a dummy node with no package, the originator
1562                  * is requesting a flavored package.  We select the default
1563                  * flavor which we presume is the first one.
1564                  */
1565                 if (pkg->pkgfile == NULL && (pkg->flags & PKGF_DUMMY)) {
1566                         pkg_t *spkg = pkg->idepon_list.next->pkg;
1567
1568                         if (spkg) {
1569                                 pkg = spkg;
1570                                 if (fp) {
1571                                         fprintf(fp,
1572                                                 "echo 'DUMMY use %s (%p)'\n",
1573                                                 pkg->portdir, pkg->pkgfile);
1574                                 }
1575                         } else {
1576                                 if (fp) {
1577                                         fprintf(fp,
1578                                                 "echo 'CANNOT FIND DEFAULT "
1579                                                 "FLAVOR FOR %s'\n",
1580                                                 pkg->portdir);
1581                                 }
1582                         }
1583                 }
1584
1585                 /*
1586                  * Generate package installation command
1587                  */
1588                 if (fp && pkg->pkgfile) {
1589                         fprintf(fp, "echo 'Installing /packages/All/%s'\n",
1590                                 pkg->pkgfile);
1591                         fprintf(fp, "pkg install -q -y /packages/All/%s "
1592                                 "|| exit 1\n",
1593                                 pkg->pkgfile);
1594                 } else if (fp) {
1595                         fprintf(fp, "echo 'CANNOT FIND PKG FOR %s'\n",
1596                                 pkg->portdir);
1597                 }
1598
1599                 if (pkg->pkgfile) {
1600                         struct stat st;
1601                         char *path;
1602                         char *ptr;
1603
1604                         asprintf(&path, "%s/%s", RepositoryPath, pkg->pkgfile);
1605                         ptr = strrchr(pkg->pkgfile, '.');
1606                         if (stat(path, &st) == 0) {
1607                                 if (strcmp(ptr, ".tar") == 0)
1608                                         tot += st.st_size;
1609                                 else if (strcmp(ptr, ".tgz") == 0)
1610                                         tot += st.st_size * 3;
1611                                 else if (strcmp(ptr, ".txz") == 0)
1612                                         tot += st.st_size * 5;
1613                                 else if (strcmp(ptr, ".tbz") == 0)
1614                                         tot += st.st_size * 3;
1615                                 else
1616                                         tot += st.st_size * 2;
1617                         }
1618                         free(path);
1619                 }
1620                 if (first_one_only)
1621                         break;
1622         }
1623         return tot;
1624 }
1625
1626 /*
1627  * Worker process interactions.
1628  *
1629  * The worker process is responsible for managing the build of a single
1630  * package.  It is exec'd by the master dsynth and only loads the
1631  * configuration.
1632  *
1633  * This process does not run in the chroot.  It will become the reaper for
1634  * all sub-processes and it will enter the chroot to execute various phases.
1635  * It catches SIGINTR, SIGHUP, and SIGPIPE and will iterate, terminate, and
1636  * reap all sub-process upon kill or exit.
1637  *
1638  * The command line forwarded to this function is:
1639  *
1640  *      WORKER slot# socketfd portdir/subdir
1641  *
1642  * TERM=dumb
1643  * USER=root
1644  * HOME=/root
1645  * LANG=C
1646  * SSL_NO_VERIFY_PEER=1
1647  * USE_PACKAGE_DEPENDS_ONLY=1   (exec_phase_depends)
1648  * PORTSDIR=/xports
1649  * PORT_DBDIR=/options          For ports options
1650  * PACKAGE_BUILDING=yes         Don't build packages that aren't legally
1651  *                              buildable for a binary repo.
1652  * PKG_DBDIR=/var/db/pkg
1653  * PKG_CACHEDIR=/var/cache/pkg
1654  * PKG_CREATE_VERBOSE=yes       Ensure periodic output during packaging
1655  * (custom environment)
1656  * PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
1657  * UNAME_s=DragonFly            (example)
1658  * UNAME_v=DragonFly 5.7-SYNTH  (example)
1659  * UNAME_p=x86_64               (example)
1660  * UNAME_m=x86_64               (example)
1661  * UNAME_r=5.7-SYNTH            (example)
1662  * NO_DEPENDS=yes               (exec_phase)
1663  * DISTDIR=/distfiles
1664  * WRKDIRPREFIX=/construction
1665  * BATCH=yes
1666  * MAKE_JOBS_NUMBER=n
1667  *
1668  * SETUP:
1669  *      ldconfig -R
1670  *      /usr/local/sbin/pkg-static install /packages/All/<the pkg pkg>
1671  *      /usr/local/sbin/pkg-static install /packages/All/<pkg>
1672  *                      (for all dependencies)
1673  *
1674  * PHASES:              make -C path FLAVOR=flavor <phase>
1675  *      check-sanity
1676  *      pkg-depends
1677  *      fetch-depends
1678  *      fetch
1679  *      checksum
1680  *      extract-depends
1681  *      extract
1682  *      patch-depends
1683  *      patch
1684  *      build-depends
1685  *      lib-depends
1686  *      configure
1687  *      build
1688  *      run-depends
1689  *      stage
1690  *      test            (skipped)
1691  *      check-plist
1692  *      package          e.g. /construction/lang/perl5.28/pkg/perl5-5.28.2.txz
1693  *      install-mtree   (skipped)
1694  *      install         (skipped)
1695  *      deinstall       (skipped)
1696  */
1697 void
1698 WorkerProcess(int ac, char **av)
1699 {
1700         wmsg_t wmsg;
1701         int fd;
1702         int slot;
1703         int tmpfd;
1704         int pkgpkg = 0;
1705         int status;
1706         int len;
1707         int do_install_phase;
1708         char *portdir;
1709         char *pkgfile;
1710         char *flavor;
1711         char *buf;
1712         worker_t *work;
1713         bulk_t *bulk;
1714         pkg_t pkg;
1715         buildenv_t *benv;
1716         FILE *fp;
1717
1718         /*
1719          * Parse arguments
1720          */
1721         if (ac != 6) {
1722                 dlog(DLOG_ALL, "WORKER PROCESS %d- bad arguments\n", getpid());
1723                 exit(1);
1724         }
1725         slot = strtol(av[1], NULL, 0);
1726         fd = strtol(av[2], NULL, 0);    /* master<->slave messaging */
1727         portdir = av[3];
1728         pkgfile = av[4];
1729         flavor = strchr(portdir, '@');
1730         if (flavor) {
1731                 *flavor++ = 0;
1732                 asprintf(&buf, "@%s", flavor);
1733                 WorkerFlavorPrt = buf;
1734                 buf = NULL;     /* safety */
1735         }
1736         WorkerProcFlags = strtol(av[5], NULL, 0);
1737
1738         bzero(&wmsg, sizeof(wmsg));
1739
1740         setproctitle("[%02d] WORKER STARTUP  %s%s",
1741                      slot, portdir, WorkerFlavorPrt);
1742
1743         if (strcmp(portdir, "ports-mgmt/pkg") == 0)
1744                 pkgpkg = 1;
1745
1746         signal(SIGTERM, phaseTerminateSignal);
1747         signal(SIGINT, phaseTerminateSignal);
1748         signal(SIGHUP, phaseTerminateSignal);
1749
1750         /*
1751          * Set up the environment
1752          */
1753         setenv("TERM", "dumb", 1);
1754         setenv("USER", "root", 1);
1755         setenv("HOME", "/root", 1);
1756         setenv("LANG", "C", 1);
1757         setenv("SSL_NO_VERIFY_PEER", "1", 1);
1758
1759         addbuildenv("USE_PACKAGE_DEPENDS_ONLY", "yes", BENV_MAKECONF);
1760         addbuildenv("PORTSDIR", "/xports", BENV_MAKECONF);
1761         addbuildenv("PORT_DBDIR", "/options", BENV_MAKECONF);
1762         addbuildenv("PKG_DBDIR", "/var/db/pkg", BENV_MAKECONF);
1763         addbuildenv("PKG_CACHEDIR", "/var/cache/pkg", BENV_MAKECONF);
1764         addbuildenv("PKG_SUFX", UsePkgSufx, BENV_MAKECONF);
1765         if (WorkerProcFlags & WORKER_PROC_DEVELOPER)
1766                 addbuildenv("DEVELOPER", "1", BENV_MAKECONF);
1767
1768         /*
1769          * CCache is a horrible unreliable hack but... leave the
1770          * mechanism in-place in case someone has a death wish.
1771          */
1772         if (UseCCache) {
1773                 addbuildenv("WITH_CCACHE_BUILD", "yes", BENV_MAKECONF);
1774                 addbuildenv("CCACHE_DIR", "/ccache", BENV_MAKECONF);
1775         }
1776
1777         addbuildenv("UID", "0", BENV_MAKECONF);
1778         addbuildenv("ARCH", ArchitectureName, BENV_MAKECONF);
1779
1780 #ifdef __DragonFly__
1781         addbuildenv("OPSYS", "DragonFly", BENV_MAKECONF);
1782         addbuildenv("DFLYVERSION", VersionFromParamHeader, BENV_MAKECONF);
1783         addbuildenv("OSVERSION", "9999999", BENV_MAKECONF);
1784 #else
1785 #error "Need OS-specific data to generate make.conf"
1786 #endif
1787
1788         addbuildenv("OSREL", ReleaseName, BENV_MAKECONF);
1789         addbuildenv("_OSRELEASE", VersionOnlyName, BENV_MAKECONF);
1790
1791         setenv("PATH",
1792                "/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin",
1793                1);
1794
1795         setenv("UNAME_s", OperatingSystemName, 1);
1796         setenv("UNAME_v", VersionName, 1);
1797         setenv("UNAME_p", ArchitectureName, 1);
1798         setenv("UNAME_m", MachineName, 1);
1799         setenv("UNAME_r", ReleaseName, 1);
1800
1801         addbuildenv("NO_DEPENDS", "yes", BENV_MAKECONF);
1802         addbuildenv("DISTDIR", "/distfiles", BENV_MAKECONF);
1803         addbuildenv("WRKDIRPREFIX", "/construction", BENV_MAKECONF);
1804         addbuildenv("BATCH", "yes", BENV_MAKECONF);
1805
1806         /*
1807          * Special consideration
1808          *
1809          * PACKAGE_BUILDING     - Disallow packaging ports which do not allow
1810          *                        for binary distribution.
1811          *
1812          * PKG_CREATE_VERBOSE   - Ensure periodic output during the packaging
1813          *                        process to avoid a watchdog timeout.
1814          *
1815          */
1816         addbuildenv("PACKAGE_BUILDING", "yes", BENV_MAKECONF);
1817         addbuildenv("PKG_CREATE_VERBOSE", "yes", BENV_MAKECONF);
1818         asprintf(&buf, "%d", MaxJobs);
1819         addbuildenv("MAKE_JOBS_NUMBER", buf, BENV_MAKECONF);
1820         freestrp(&buf);
1821
1822         if (flavor)
1823                 setenv("FLAVOR", flavor, 1);
1824
1825         /*
1826          * Become the reaper
1827          */
1828         if (procctl(P_PID, getpid(), PROC_REAP_ACQUIRE, NULL) < 0)
1829                 dfatal_errno("procctl() - Cannot become reaper");
1830
1831         /*
1832          * Initialize a worker structure
1833          */
1834         DoInitBuild(slot);
1835
1836         bzero(&pkg, sizeof(pkg));
1837         pkg.portdir = portdir;          /* sans flavor */
1838         pkg.pkgfile = pkgfile;
1839         if (strchr(portdir, '/'))
1840                 len = strchr(portdir, '/') - portdir;
1841         else
1842                 len = 0;
1843
1844         /*
1845          * Setup the logfile
1846          */
1847         asprintf(&pkg.logfile,
1848                  "%s/%*.*s___%s%s%s.log",
1849                  LogsPath, len, len, portdir,
1850                  ((portdir[len] == '/') ? portdir + len + 1 : portdir + len),
1851                  (flavor ? "@" : ""),
1852                  (flavor ? flavor : ""));
1853         tmpfd = open(pkg.logfile, O_RDWR|O_CREAT|O_TRUNC, 0666);
1854         if (tmpfd >= 0) {
1855                 if (DebugOpt >= 2) {
1856                         dlog(DLOG_ALL, "[%03d] %s LOGFILE %s\n",
1857                              slot, pkg.portdir, pkg.logfile);
1858                 }
1859                 close(tmpfd);
1860         } else {
1861                 dlog(DLOG_ALL, "[%03d] LOGFILE %s (create failed)\n",
1862                      slot, pkg.logfile);
1863         }
1864
1865         /*
1866          * Setup the work structure.  Because this is an exec'd sub-process,
1867          * there is only one work structure.
1868          */
1869         work = &WorkerAry[0];
1870         work->flavor = flavor;
1871         work->fds[0] = fd;
1872         work->pkg = &pkg;
1873         work->start_time = time(NULL);
1874
1875         /*
1876          * Do mounts
1877          */
1878         SigWork = work;
1879         setproctitle("[%02d] WORKER MOUNTS   %s%s",
1880                      slot, portdir, WorkerFlavorPrt);
1881         DoWorkerMounts(work);
1882
1883         /*
1884          * Generate an /etc/make.conf in the build base
1885          */
1886         asprintf(&buf, "%s/etc/make.conf", work->basedir);
1887         fp = fopen(buf, "w");
1888         dassert_errno(fp, "Unable to create %s\n", buf);
1889         for (benv = BuildEnv; benv; benv = benv->next) {
1890                 if (benv->type & BENV_PKGLIST)
1891                         continue;
1892                 if ((benv->type & BENV_CMDMASK) == BENV_MAKECONF) {
1893                         if (DebugOpt >= 2) {
1894                                 dlog(DLOG_ALL, "[%03d] ENV %s=%s\n",
1895                                      slot, benv->label, benv->data);
1896                         }
1897                         fprintf(fp, "%s=%s\n", benv->label, benv->data);
1898                 }
1899         }
1900         fclose(fp);
1901         freestrp(&buf);
1902
1903         /*
1904          * Set up our hooks
1905          */
1906         if (UsingHooks)
1907                 initbulk(childHookRun, MaxBulk);
1908
1909         /*
1910          * Start phases
1911          */
1912         wmsg.cmd = WMSG_CMD_INSTALL_PKGS;
1913         ipcwritemsg(fd, &wmsg);
1914         status = ipcreadmsg(fd, &wmsg);
1915         if (status < 0 || wmsg.cmd != WMSG_RES_INSTALL_PKGS)
1916                 dfatal("pkg installation handshake failed");
1917         do_install_phase = wmsg.status;
1918
1919         wmsg.cmd = WMSG_CMD_STATUS_UPDATE;
1920         wmsg.phase = PHASE_INSTALL_PKGS;
1921         wmsg.lines = 0;
1922
1923         status = ipcwritemsg(fd, &wmsg);
1924
1925         if (pkgpkg) {
1926                 dophase(work, &wmsg,
1927                         WDOG5, PHASE_PACKAGE, "package");
1928         } else {
1929                 if (do_install_phase) {
1930                         dophase(work, &wmsg,
1931                                 WDOG4, PHASE_INSTALL_PKGS, "setup");
1932                 }
1933                 dophase(work, &wmsg,
1934                         WDOG2, PHASE_CHECK_SANITY, "check-sanity");
1935                 dophase(work, &wmsg,
1936                         WDOG2, PHASE_PKG_DEPENDS, "pkg-depends");
1937                 dophase(work, &wmsg,
1938                         WDOG7, PHASE_FETCH_DEPENDS, "fetch-depends");
1939                 dophase(work, &wmsg,
1940                         WDOG7, PHASE_FETCH, "fetch");
1941                 dophase(work, &wmsg,
1942                         WDOG2, PHASE_CHECKSUM, "checksum");
1943                 dophase(work, &wmsg,
1944                         WDOG3, PHASE_EXTRACT_DEPENDS, "extract-depends");
1945                 dophase(work, &wmsg,
1946                         WDOG3, PHASE_EXTRACT, "extract");
1947                 dophase(work, &wmsg,
1948                         WDOG2, PHASE_PATCH_DEPENDS, "patch-depends");
1949                 dophase(work, &wmsg,
1950                         WDOG2, PHASE_PATCH, "patch");
1951                 dophase(work, &wmsg,
1952                         WDOG5, PHASE_BUILD_DEPENDS, "build-depends");
1953                 dophase(work, &wmsg,
1954                         WDOG5, PHASE_LIB_DEPENDS, "lib-depends");
1955                 dophase(work, &wmsg,
1956                         WDOG3, PHASE_CONFIGURE, "configure");
1957                 dophase(work, &wmsg,
1958                         WDOG9, PHASE_BUILD, "build");
1959                 dophase(work, &wmsg,
1960                         WDOG5, PHASE_RUN_DEPENDS, "run-depends");
1961                 dophase(work, &wmsg,
1962                         WDOG5, PHASE_STAGE, "stage");
1963 #if 0
1964                 dophase(work, &wmsg,
1965                         WDOG5, PHASE_TEST, "test");
1966 #endif
1967                 dophase(work, &wmsg,
1968                         WDOG1, PHASE_CHECK_PLIST, "check-plist");
1969                 dophase(work, &wmsg,
1970                         WDOG5, PHASE_PACKAGE, "package");
1971 #if 0
1972                 dophase(work, &wmsg,
1973                         WDOG5, PHASE_INSTALL_MTREE, "install-mtree");
1974                 dophase(work, &wmsg,
1975                         WDOG5, PHASE_INSTALL, "install");
1976                 dophase(work, &wmsg,
1977                         WDOG5, PHASE_DEINSTALL, "deinstall");
1978 #endif
1979         }
1980
1981         if (MasterPtyFd >= 0) {
1982                 close(MasterPtyFd);
1983                 MasterPtyFd = -1;
1984         }
1985
1986         setproctitle("[%02d] WORKER CLEANUP  %s%s",
1987                      slot, portdir, WorkerFlavorPrt);
1988
1989         /*
1990          * Copy the package to the repo.
1991          */
1992         if (work->accum_error == 0) {
1993                 char *b1;
1994                 char *b2;
1995
1996                 asprintf(&b1, "%s/construction/%s/pkg/%s",
1997                          work->basedir, pkg.portdir, pkg.pkgfile);
1998                 asprintf(&b2, "%s/%s", RepositoryPath, pkg.pkgfile);
1999                 if (copyfile(b1, b2)) {
2000                         ++work->accum_error;
2001                         dlog(DLOG_ALL, "[%03d] %s Unable to copy %s to %s\n",
2002                              work->index, pkg.portdir, b1, b2);
2003                 }
2004                 free(b1);
2005                 free(b2);
2006         }
2007
2008         /*
2009          * Unmount, unless we are in DebugStopMode.
2010          */
2011         if ((WorkerProcFlags & WORKER_PROC_DEBUGSTOP) == 0)
2012                 DoWorkerUnmounts(work);
2013
2014         /*
2015          * Send completion status to master dsynth worker thread.
2016          */
2017         if (work->accum_error) {
2018                 wmsg.cmd = WMSG_CMD_FAILURE;
2019         } else {
2020                 wmsg.cmd = WMSG_CMD_SUCCESS;
2021         }
2022         ipcwritemsg(fd, &wmsg);
2023         if (WorkerProcFlags & WORKER_PROC_DEBUGSTOP) {
2024                 wmsg.cmd = WMSG_CMD_FREEZEWORKER;
2025                 ipcwritemsg(fd, &wmsg);
2026         }
2027         if (UsingHooks) {
2028                 while ((bulk = getbulk()) != NULL)
2029                         freebulk(bulk);
2030                 donebulk();
2031         }
2032 }
2033
2034 static void
2035 dophase(worker_t *work, wmsg_t *wmsg, int wdog, int phaseid, const char *phase)
2036 {
2037         pkg_t *pkg = work->pkg;
2038         char buf[1024];
2039         pid_t pid;
2040         int status;
2041         int ms;
2042         pid_t wpid;
2043         int wpid_reaped;
2044         int fdlog;
2045         time_t start_time;
2046         time_t last_time;
2047         time_t next_time;
2048         time_t wdog_time;
2049         FILE *fp;
2050
2051         if (work->accum_error)
2052                 return;
2053         setproctitle("[%02d] WORKER %-8.8s %s%s",
2054                      work->index, phase, pkg->portdir, WorkerFlavorPrt);
2055         wmsg->phase = phaseid;
2056         if (ipcwritemsg(work->fds[0], wmsg) < 0) {
2057                 dlog(DLOG_ALL, "[%03d] %s Lost Communication with dsynth, "
2058                      "aborting worker\n",
2059                      work->index, pkg->portdir);
2060                 ++work->accum_error;
2061                 return;
2062         }
2063
2064         /*
2065          * Execute the port make command in chroot on a pty.
2066          */
2067         fflush(stdout);
2068         fflush(stderr);
2069         if (MasterPtyFd >= 0) {
2070                 int slavefd;
2071                 char ttybuf[2];
2072
2073                 /*
2074                  * NOTE: We can't open the slave in the child because the
2075                  *       master may race a disconnection test.  If we open
2076                  *       it in the parent our close() will flush any pending
2077                  *       output not read by the master (which is the same
2078                  *       parent process) and deadlock.
2079                  *
2080                  *       Solve this by hand-shaking the slave tty to give
2081                  *       the master time to close its slavefd.
2082                  *
2083                  *       Leave the tty defaults intact, which also likely
2084                  *       means it will be in line-buffered mode, so handshake
2085                  *       with a full line.
2086                  *
2087                  * TODO: Our handshake probably echos back to the master pty
2088                  *       due to tty echo, and ends up in the log, fix me.
2089                  */
2090                 slavefd = open(ptsname(MasterPtyFd), O_RDWR);
2091                 dassert_errno(slavefd >= 0, "Cannot open slave pty");
2092                 pid = fork();
2093                 if (pid == 0) {
2094                         login_tty(slavefd);
2095                         /* login_tty() closes slavefd */
2096                         read(0, ttybuf, 2);
2097                 } else {
2098                         close(slavefd);
2099                         write(MasterPtyFd, "x\n", 2);
2100                 }
2101         } else {
2102                 pid = forkpty(&MasterPtyFd, NULL, NULL, NULL);
2103         }
2104
2105         if (pid == 0) {
2106                 struct termios tio;
2107
2108                 /*
2109                  * We are going through a pty, so set the tty modes to
2110                  * Set tty modes so we do not get ^M's in the log files.
2111                  *
2112                  * This isn't fatal if it doesn't work.  Remember that
2113                  * our output goes through the pty to the management
2114                  * process which will log it.
2115                  */
2116                 if (tcgetattr(1, &tio) == 0) {
2117                         tio.c_oflag |= OPOST | ONOCR;
2118                         tio.c_oflag &= ~(OCRNL | ONLCR);
2119                         tio.c_iflag |= ICRNL;
2120                         tio.c_iflag &= ~(INLCR|IGNCR);
2121                         if (tcsetattr(1, TCSANOW, &tio)) {
2122                                 printf("tcsetattr failed: %s\n",
2123                                        strerror(errno));
2124                         }
2125                 } else {
2126                         printf("tcgetattr failed: %s\n", strerror(errno));
2127                 }
2128
2129                 /*
2130                  * Clean-up, chdir, and chroot.
2131                  */
2132                 closefrom(3);
2133                 if (chdir(work->basedir) < 0)
2134                         dfatal_errno("chdir in phase initialization");
2135                 if (chroot(work->basedir) < 0)
2136                         dfatal_errno("chroot in phase initialization");
2137
2138                 /*
2139                  * We have a choice here on how to handle stdin (fd 0).
2140                  * We can leave it connected to the pty in which case
2141                  * the build will just block if it tries to ask a
2142                  * question (and the watchdog will kill it, eventually),
2143                  * or we can try to EOF the pty, or we can attach /dev/null
2144                  * to descriptor 0.
2145                  */
2146                 if (NullStdinOpt) {
2147                         int fd;
2148
2149                         fd = open("/dev/null", O_RDWR);
2150                         dassert_errno(fd >= 0, "cannot open /dev/null");
2151                         if (fd != 0) {
2152                                 dup2(fd, 0);
2153                                 close(fd);
2154                         }
2155                 }
2156
2157                 /*
2158                  * Execute the appropriate command.
2159                  */
2160                 switch(phaseid) {
2161                 case PHASE_INSTALL_PKGS:
2162                         snprintf(buf, sizeof(buf), "/tmp/dsynth_install_pkgs");
2163                         execl(buf, buf, NULL);
2164                         break;
2165                 default:
2166                         snprintf(buf, sizeof(buf), "/xports/%s", pkg->portdir);
2167                         execl(MAKE_BINARY, MAKE_BINARY, "-C", buf, phase, NULL);
2168                         break;
2169                 }
2170                 _exit(1);
2171         }
2172         fcntl(MasterPtyFd, F_SETFL, O_NONBLOCK);
2173
2174         if (pid < 0) {
2175                 dlog(DLOG_ALL, "[%03d] %s Fork Failed: %s\n",
2176                      work->index, pkg->logfile, strerror(errno));
2177                 ++work->accum_error;
2178                 return;
2179         }
2180
2181         SigPid = pid;
2182
2183         fdlog = open(pkg->logfile, O_RDWR|O_CREAT|O_APPEND, 0644);
2184         if (fdlog < 0) {
2185                 dlog(DLOG_ALL, "[%03d] %s Cannot open logfile '%s': %s\n",
2186                      work->index, pkg->portdir,
2187                      pkg->logfile, strerror(errno));
2188         }
2189
2190         snprintf(buf, sizeof(buf),
2191                  "----------------------------------------"
2192                  "---------------------------------------\n"
2193                  "-- Phase: %s\n"
2194                  "----------------------------------------"
2195                  "---------------------------------------\n",
2196                  phase);
2197         write(fdlog, buf, strlen(buf));
2198
2199         start_time = time(NULL);
2200         last_time = start_time;
2201         wdog_time = start_time;
2202         wpid_reaped = 0;
2203
2204         status = 0;
2205         for (;;) {
2206                 ms = mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time);
2207                 if (ms == MPTY_FAILED) {
2208                         dlog(DLOG_ALL,
2209                              "[%03d] %s lost pty in phase %s, terminating\n",
2210                              work->index, pkg->portdir, phase);
2211                         break;
2212                 }
2213                 if (ms == MPTY_EOF)
2214                         break;
2215
2216                 /*
2217                  * Generally speaking update status once a second.
2218                  * This also allows us to detect if the management
2219                  * dsynth process has gone away.
2220                  */
2221                 next_time = time(NULL);
2222                 if (next_time != last_time) {
2223                         double dload[3];
2224                         double dv;
2225                         int wdog_scaled;
2226
2227                         /*
2228                          * Send status update to the worker management thread
2229                          * in the master dsynth process.  Remember, *WE* are
2230                          * the worker management process sub-fork.
2231                          */
2232                         if (ipcwritemsg(work->fds[0], wmsg) < 0)
2233                                 break;
2234                         last_time = next_time;
2235
2236                         /*
2237                          * Watchdog scaling
2238                          */
2239                         getloadavg(dload, 3);
2240                         dv = dload[2] / NumCores;
2241                         if (dv < (double)NumCores) {
2242                                 wdog_scaled = wdog;
2243                         } else {
2244                                 if (dv > 4.0 * NumCores)
2245                                         dv = 4.0 * NumCores;
2246                                 wdog_scaled = wdog * dv / NumCores;
2247                         }
2248
2249                         /*
2250                          * Watchdog
2251                          */
2252                         if (next_time - wdog_time >= wdog_scaled * 60) {
2253                                 snprintf(buf, sizeof(buf),
2254                                          "\n--------\n"
2255                                          "WATCHDOG TIMEOUT FOR %s in %s "
2256                                          "after %d minutes\n"
2257                                          "Killing pid %d\n"
2258                                          "--------\n",
2259                                          pkg->portdir, phase, wdog_scaled, pid);
2260                                 if (fdlog >= 0)
2261                                         write(fdlog, buf, strlen(buf));
2262                                 dlog(DLOG_ALL,
2263                                      "[%03d] %s WATCHDOG TIMEOUT in %s "
2264                                      "after %d minutes (%d min scaled)\n",
2265                                      work->index, pkg->portdir, phase,
2266                                      wdog, wdog_scaled);
2267                                 kill(pid, SIGKILL);
2268                                 ++work->accum_error;
2269                                 break;
2270                         }
2271                 }
2272
2273                 /*
2274                  * Check process exit.  Normally the pty will EOF
2275                  * but if background processes remain we need to
2276                  * check here to see if our primary exec is done,
2277                  * so we can break out and reap those processes.
2278                  *
2279                  * Generally reap any other processes we have inherited
2280                  * while we are here.
2281                  */
2282                 do {
2283                         wpid = wait3(&status, WNOHANG, NULL);
2284                 } while (wpid > 0 && wpid != pid);
2285                 if (wpid == pid && WIFEXITED(status)) {
2286                         wpid_reaped = 1;
2287                         break;
2288                 }
2289         }
2290
2291         next_time = time(NULL);
2292
2293         setproctitle("[%02d] WORKER EXITREAP %s%s",
2294                      work->index, pkg->portdir, WorkerFlavorPrt);
2295
2296         /*
2297          * We usually get here due to a mpty EOF, but not always as there
2298          * could be persistent processes still holding the slave.  Finish
2299          * up getting the exit status for the main process we are waiting
2300          * on and clean out any data left on the MasterPtyFd (as it could
2301          * be blocking the exit).
2302          */
2303         while (wpid_reaped == 0) {
2304                 (void)mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time);
2305                 wpid = waitpid(pid, &status, WNOHANG);
2306                 if (wpid == pid && WIFEXITED(status)) {
2307                         wpid_reaped = 1;
2308                         break;
2309                 }
2310                 if (wpid < 0 && errno != EINTR) {
2311                         break;
2312                 }
2313
2314                 /*
2315                  * Safety.  The normal phase waits until the fork/exec'd
2316                  * pid finishes, causing a pty EOF on exit (the slave
2317                  * descriptor is closed by the kernel on exit so the
2318                  * process should already have exited).
2319                  *
2320                  * However, it is also possible to get here if the pty fails
2321                  * for some reason.  In this case, make sure that the process
2322                  * is killed.
2323                  */
2324                 kill(pid, SIGKILL);
2325         }
2326
2327         /*
2328          * Clean out anything left on the pty but don't wait around
2329          * because there could be background processes preventing the
2330          * slave side from closing.
2331          */
2332         while (mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time) == MPTY_DATA)
2333                 ;
2334
2335         /*
2336          * Report on the exit condition.  If the pid was somehow lost
2337          * (probably due to someone gdb'ing the process), assume an error.
2338          */
2339         if (wpid_reaped) {
2340                 if (WEXITSTATUS(status)) {
2341                         dlog(DLOG_ALL | DLOG_FILTER,
2342                              "[%03d] %s Build phase '%s' failed exit %d\n",
2343                              work->index, pkg->portdir, phase,
2344                              WEXITSTATUS(status));
2345                         ++work->accum_error;
2346                 }
2347         } else {
2348                 dlog(DLOG_ALL, "[%03d] %s Build phase '%s' failed - lost pid\n",
2349                      work->index, pkg->portdir, phase);
2350                 ++work->accum_error;
2351         }
2352
2353         /*
2354          * Kill any processes still running (sometimes processes end up in
2355          * the background during a dports build), and clean up any other
2356          * children that we have inherited.
2357          */
2358         phaseReapAll();
2359
2360         /*
2361          * After the extraction phase add the space used by /construction
2362          * to the memory use.  This helps us reduce the amount of paging
2363          * we do due to extremely large package extractions (languages,
2364          * chromium, etc).
2365          *
2366          * (dsynth already estimated the space used by the package deps
2367          * up front, but this will help us further).
2368          */
2369         if (work->accum_error == 0 && phaseid == PHASE_EXTRACT) {
2370                 struct statfs sfs;
2371                 char *b1;
2372
2373                 asprintf(&b1, "%s/construction", work->basedir);
2374                 if (statfs(b1, &sfs) == 0) {
2375                         wmsg->memuse = (sfs.f_blocks - sfs.f_bfree) *
2376                                        sfs.f_bsize;
2377                         ipcwritemsg(work->fds[0], wmsg);
2378                 }
2379         }
2380
2381         /*
2382          * Update log
2383          */
2384         if (fdlog >= 0) {
2385                 struct stat st;
2386                 int h;
2387                 int m;
2388                 int s;
2389
2390                 last_time = next_time - start_time;
2391                 s = last_time % 60;
2392                 m = last_time / 60 % 60;
2393                 h = last_time / 3600;
2394
2395                 fp = fdopen(fdlog, "a");
2396                 if (fp == NULL) {
2397                         dlog(DLOG_ALL, "[%03d] %s Cannot fdopen fdlog: %s %d\n",
2398                              work->index, pkg->portdir,
2399                              strerror(errno), fstat(fdlog, &st));
2400                         close(fdlog);
2401                         goto skip;
2402                 }
2403
2404                 fprintf(fp, "\n");
2405                 if (work->accum_error) {
2406                         fprintf(fp, "FAILED %02d:%02d:%02d\n", h, m, s);
2407                 } else {
2408                         if (phaseid == PHASE_EXTRACT && wmsg->memuse) {
2409                                 fprintf(fp, "Extracted Memory Use: %6.2fM\n",
2410                                         wmsg->memuse / (1024.0 * 1024.0));
2411                         }
2412                         fprintf(fp, "SUCCEEDED %02d:%02d:%02d\n", h, m, s);
2413                 }
2414                 last_time = next_time - work->start_time;
2415                 s = last_time % 60;
2416                 m = last_time / 60 % 60;
2417                 h = last_time / 3600;
2418                 if (phaseid == PHASE_PACKAGE) {
2419                         fprintf(fp, "TOTAL TIME %02d:%02d:%02d\n", h, m, s);
2420                 }
2421                 fprintf(fp, "\n");
2422                 fclose(fp);
2423 skip:
2424                 ;
2425         }
2426
2427 }
2428
2429 static void
2430 phaseReapAll(void)
2431 {
2432         struct reaper_status rs;
2433         int status;
2434
2435         while (procctl(P_PID, getpid(), PROC_REAP_STATUS, &rs) == 0) {
2436                 if ((rs.flags & PROC_REAP_ACQUIRE) == 0)
2437                         break;
2438                 if (rs.pid_head < 0)
2439                         break;
2440                 if (kill(rs.pid_head, SIGKILL) == 0) {
2441                         while (waitpid(rs.pid_head, &status, 0) < 0)
2442                                 ;
2443                 }
2444         }
2445         while (wait3(&status, 0, NULL) > 0)
2446                 ;
2447 }
2448
2449 static void
2450 phaseTerminateSignal(int sig __unused)
2451 {
2452         if (CopyFileFd >= 0)
2453                 close(CopyFileFd);
2454         if (MasterPtyFd >= 0)
2455                 close(MasterPtyFd);
2456         if (SigPid > 1)
2457                 kill(SigPid, SIGKILL);
2458         phaseReapAll();
2459         if (SigWork)
2460                 DoWorkerUnmounts(SigWork);
2461         exit(1);
2462 }
2463
2464 static
2465 char *
2466 buildskipreason(pkglink_t *parent, pkg_t *pkg)
2467 {
2468         pkglink_t *link;
2469         pkg_t *scan;
2470         char *reason = NULL;
2471         char *ptr;
2472         size_t tot;
2473         size_t len;
2474         pkglink_t stack;
2475
2476         if ((pkg->flags & PKGF_NOBUILD_I) && pkg->ignore)
2477                 asprintf(&reason, "%s ", pkg->ignore);
2478
2479         tot = 0;
2480         PKGLIST_FOREACH(link, &pkg->idepon_list) {
2481 #if 0
2482                 if (link->dep_type > DEP_TYPE_BUILD)
2483                         continue;
2484 #endif
2485                 scan = link->pkg;
2486                 if (scan == NULL)
2487                         continue;
2488                 if ((scan->flags & (PKGF_ERROR | PKGF_NOBUILD)) == 0)
2489                         continue;
2490                 if (scan->flags & PKGF_NOBUILD) {
2491                         stack.pkg = scan;
2492                         stack.next = parent;
2493                         ptr = buildskipreason(&stack, scan);
2494                         len = strlen(scan->portdir) + strlen(ptr) + 8;
2495                         reason = realloc(reason, tot + len);
2496                         snprintf(reason + tot, len, "%s->%s",
2497                                  scan->portdir, ptr);
2498                         free(ptr);
2499                 } else {
2500                         len = strlen(scan->portdir) + 8;
2501                         reason = realloc(reason, tot + len);
2502                         snprintf(reason + tot, len, "%s", scan->portdir);
2503                 }
2504
2505                 /*
2506                  * Don't try to print the entire graph
2507                  */
2508                 if (parent)
2509                         break;
2510                 tot += strlen(reason + tot);
2511                 reason[tot++] = ' ';
2512                 reason[tot] = 0;
2513         }
2514         return (reason);
2515 }
2516
2517 /*
2518  * The master ptyfd is in non-blocking mode.  Drain up to 1024 bytes
2519  * and update wmsg->lines and *wdog_timep as appropriate.
2520  *
2521  * This function will poll, stalling up to 1 second.
2522  */
2523 static int
2524 mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg, time_t *wdog_timep)
2525 {
2526         struct pollfd pfd;
2527         char buf[1024];
2528         ssize_t r;
2529
2530         pfd.fd = ptyfd;
2531         pfd.events = POLLIN;
2532         pfd.revents = 0;
2533
2534         poll(&pfd, 1, 1000);
2535         if (pfd.revents) {
2536                 r = read(ptyfd, buf, sizeof(buf));
2537                 if (r > 0) {
2538                         *wdog_timep = time(NULL);
2539                         if (r > 0 && fdlog >= 0)
2540                                 write(fdlog, buf, r);
2541                         while (--r >= 0) {
2542                                 if (buf[r] == '\n')
2543                                         ++wmsg->lines;
2544                         }
2545                         return MPTY_DATA;
2546                 } else if (r < 0) {
2547                         if (errno != EINTR && errno != EAGAIN)
2548                                 return MPTY_FAILED;
2549                         return MPTY_AGAIN;
2550                 } else if (r == 0) {
2551                         return MPTY_EOF;
2552                 }
2553         }
2554         return MPTY_AGAIN;
2555 }
2556
2557 /*
2558  * Copy a (package) file from (src) to (dst), use an intermediate file and
2559  * rename to ensure that interruption does not leave us with a corrupt
2560  * package file.
2561  *
2562  * This is called by the WORKER process.
2563  *
2564  * (dsynth management thread -> WORKER process -> sub-processes)
2565  */
2566 #define COPYBLKSIZE     32768
2567
2568 static int
2569 copyfile(char *src, char *dst)
2570 {
2571         char *tmp;
2572         char *buf;
2573         int fd1;
2574         int fd2;
2575         int error = 0;
2576         int mask;
2577         ssize_t r;
2578
2579         asprintf(&tmp, "%s.new", dst);
2580         buf = malloc(COPYBLKSIZE);
2581
2582         mask = sigsetmask(sigmask(SIGTERM)|sigmask(SIGINT)|sigmask(SIGHUP));
2583         fd1 = open(src, O_RDONLY|O_CLOEXEC);
2584         fd2 = open(tmp, O_RDWR|O_CREAT|O_TRUNC|O_CLOEXEC, 0644);
2585         CopyFileFd = fd1;
2586         sigsetmask(mask);
2587         while ((r = read(fd1, buf, COPYBLKSIZE)) > 0) {
2588                 if (write(fd2, buf, r) != r)
2589                         error = 1;
2590         }
2591         if (r < 0)
2592                 error = 1;
2593         mask = sigsetmask(sigmask(SIGTERM)|sigmask(SIGINT)|sigmask(SIGHUP));
2594         CopyFileFd = -1;
2595         close(fd1);
2596         close(fd2);
2597         sigsetmask(mask);
2598         if (error) {
2599                 remove(tmp);
2600         } else {
2601                 if (rename(tmp, dst)) {
2602                         error = 1;
2603                         remove(tmp);
2604                 }
2605         }
2606
2607         freestrp(&buf);
2608         freestrp(&tmp);
2609
2610         return error;
2611 }
2612
2613 /*
2614  * doHook()
2615  *
2616  * primary process (threaded) - run_start, run_end, pkg_ignored, pkg_skipped
2617  * worker process  (threaded) - pkg_sucess, pkg_failure
2618  *
2619  * If waitfor is non-zero this hook will be serialized.
2620  */
2621 static void
2622 doHook(pkg_t *pkg, const char *id, const char *path, int waitfor)
2623 {
2624         if (path == NULL)
2625                 return;
2626         while (waitfor && getbulk() != NULL)
2627                 ;
2628         if (pkg)
2629                 queuebulk(pkg->portdir, id, path, pkg->pkgfile);
2630         else
2631                 queuebulk(NULL, id, path, NULL);
2632         while (waitfor && getbulk() != NULL)
2633                 ;
2634 }
2635
2636 /*
2637  * Execute hook (backend)
2638  *
2639  * s1 - portdir
2640  * s2 - id
2641  * s3 - script path
2642  * s4 - pkgfile         (if applicable)
2643  */
2644 static void
2645 childHookRun(bulk_t *bulk)
2646 {
2647         const char *cav[MAXCAC];
2648         buildenv_t benv[MAXCAC];
2649         char buf1[128];
2650         char buf2[128];
2651         char buf3[128];
2652         char buf4[128];
2653         FILE *fp;
2654         char *ptr;
2655         size_t len;
2656         pid_t pid;
2657         int cac;
2658         int bi;
2659
2660         cac = 0;
2661         bi = 0;
2662         bzero(benv, sizeof(benv));
2663
2664         cav[cac++] = bulk->s3;
2665
2666         benv[bi].label = "PROFILE";
2667         benv[bi].data = Profile;
2668         ++bi;
2669
2670         benv[bi].label = "DIR_PACKAGES";
2671         benv[bi].data = PackagesPath;
2672         ++bi;
2673
2674         benv[bi].label = "DIR_REPOSITORY";
2675         benv[bi].data = RepositoryPath;
2676         ++bi;
2677
2678         benv[bi].label = "DIR_PORTS";
2679         benv[bi].data = DPortsPath;
2680         ++bi;
2681
2682         benv[bi].label = "DIR_OPTIONS";
2683         benv[bi].data = OptionsPath;
2684         ++bi;
2685
2686         benv[bi].label = "DIR_DISTFILES";
2687         benv[bi].data = DistFilesPath;
2688         ++bi;
2689
2690         benv[bi].label = "DIR_LOGS";
2691         benv[bi].data = LogsPath;
2692         ++bi;
2693
2694         benv[bi].label = "DIR_BUILDBASE";
2695         benv[bi].data = BuildBase;
2696         ++bi;
2697
2698         if (strcmp(bulk->s2, "hook_run_start") == 0) {
2699                 snprintf(buf1, sizeof(buf1), "%d", BuildTotal);
2700                 benv[bi].label = "PORTS_QUEUED";
2701                 benv[bi].data = buf1;
2702                 ++bi;
2703         } else if (strcmp(bulk->s2, "hook_run_end") == 0) {
2704                 snprintf(buf1, sizeof(buf1), "%d", BuildSuccessCount);
2705                 benv[bi].label = "PORTS_BUILT";
2706                 benv[bi].data = buf1;
2707                 ++bi;
2708                 snprintf(buf2, sizeof(buf2), "%d", BuildFailCount);
2709                 benv[bi].label = "PORTS_FAILED";
2710                 benv[bi].data = buf2;
2711                 ++bi;
2712                 snprintf(buf3, sizeof(buf3), "%d", BuildIgnoreCount);
2713                 benv[bi].label = "PORTS_IGNORED";
2714                 benv[bi].data = buf3;
2715                 ++bi;
2716                 snprintf(buf4, sizeof(buf4), "%d", BuildSkipCount);
2717                 benv[bi].label = "PORTS_SKIPPED";
2718                 benv[bi].data = buf4;
2719                 ++bi;
2720         } else {
2721                 /*
2722                  * success, failure, ignored, skipped
2723                  */
2724                 benv[bi].label = "RESULT";
2725                 if (strcmp(bulk->s2, "hook_pkg_success") == 0) {
2726                         benv[bi].data = "success";
2727                 } else if (strcmp(bulk->s2, "hook_pkg_failure") == 0) {
2728                         benv[bi].data = "failure";
2729                 } else if (strcmp(bulk->s2, "hook_pkg_ignored") == 0) {
2730                         benv[bi].data = "ignored";
2731                 } else if (strcmp(bulk->s2, "hook_pkg_skipped") == 0) {
2732                         benv[bi].data = "skipped";
2733                 } else {
2734                         dfatal("Unknown hook id: %s", bulk->s2);
2735                         /* NOT REACHED */
2736                 }
2737                 ++bi;
2738
2739                 /*
2740                  * For compatibility with synth:
2741                  *
2742                  * ORIGIN does not include any @flavor, thus it is suitable
2743                  * for finding the actual port directory/subdirectory.
2744                  *
2745                  * FLAVOR is set to ORIGIN if there is no flavor, otherwise
2746                  * it is set to only the flavor sans the '@'.
2747                  */
2748                 if ((ptr = strchr(bulk->s1, '@')) != NULL) {
2749                         snprintf(buf1, sizeof(buf1), "%*.*s",
2750                                  (int)(ptr - bulk->s1),
2751                                  (int)(ptr - bulk->s1),
2752                                  bulk->s1);
2753                         benv[bi].label = "ORIGIN";
2754                         benv[bi].data = buf1;
2755                         ++bi;
2756                         benv[bi].label = "FLAVOR";
2757                         benv[bi].data = ptr + 1;
2758                         ++bi;
2759                 } else {
2760                         benv[bi].label = "ORIGIN";
2761                         benv[bi].data = bulk->s1;
2762                         ++bi;
2763                         benv[bi].label = "FLAVOR";
2764                         benv[bi].data = bulk->s1;
2765                         ++bi;
2766                 }
2767                 benv[bi].label = "PKGNAME";
2768                 benv[bi].data = bulk->s4;
2769                 ++bi;
2770         }
2771
2772         benv[bi].label = NULL;
2773         benv[bi].data = NULL;
2774
2775         fp = dexec_open(cav, cac, &pid, benv, 0, 0);
2776         while ((ptr = fgetln(fp, &len)) != NULL)
2777                 ;
2778
2779         if (dexec_close(fp, pid)) {
2780                 dlog(DLOG_ALL,
2781                      "[XXX] %s SCRIPT %s (%s)\n",
2782                      bulk->s1, bulk->s2, bulk->s3);
2783         }
2784 }