usr.bin/dsynth/build.c

   1 /*
   2  * Copyright (c) 2019 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * This code uses concepts and configuration based on 'synth', by
   8  * John R. Marino <draco@marino.st>, which was written in ada.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  *
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in
  18  *    the documentation and/or other materials provided with the
  19  *    distribution.
  20  * 3. Neither the name of The DragonFly Project nor the names of its
  21  *    contributors may be used to endorse or promote products derived
  22  *    from this software without specific, prior written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  25  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  26  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  27  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  28  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  29  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  30  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  31  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  32  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  33  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  34  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  35  * SUCH DAMAGE.
  36  */
  37 #include "dsynth.h"
  38
  39 worker_t WorkerAry[MAXWORKERS];
  40 int BuildInitialized;
  41 int RunningWorkers;
  42 int DynamicMaxWorkers;
  43 int FailedWorkers;
  44 long RunningPkgDepSize;
  45 pthread_mutex_t WorkerMutex;
  46 pthread_cond_t WorkerCond;
  47
  48 static int build_find_leaves(pkg_t *parent, pkg_t *pkg,
  49                         pkg_t ***build_tailp, int *app, int *hasworkp,
  50                         int depth, int first, int first_one_only);
  51 static int buildCalculateDepiDepth(pkg_t *pkg);
  52 static void build_clear_trav(pkg_t *pkg);
  53 static void startbuild(pkg_t **build_listp, pkg_t ***build_tailp);
  54 static int qsort_depi(const void *pkg1, const void *pkg2);
  55 static int qsort_idep(const void *pkg1, const void *pkg2);
  56 static void startworker(pkg_t *pkg, worker_t *work);
  57 static void cleanworker(worker_t *work);
  58 static void waitbuild(int whilematch, int dynamicmax);
  59 static void workercomplete(worker_t *work);
  60 static void *childBuilderThread(void *arg);
  61 static int childInstallPkgDeps(worker_t *work);
  62 static size_t childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list,
  63                         int undoit, int depth, int first_one_only);
  64 static void dophase(worker_t *work, wmsg_t *wmsg,
  65                         int wdog, int phaseid, const char *phase);
  66 static void phaseReapAll(void);
  67 static void phaseTerminateSignal(int sig);
  68 static char *buildskipreason(pkglink_t *parent, pkg_t *pkg);
  69 static int mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg,
  70                         time_t *wdog_timep);
  71 static int copyfile(char *src, char *dst);
  72 static void doHook(pkg_t *pkg, const char *id, const char *path, int waitfor);
  73 static void childHookRun(bulk_t *bulk);
  74
  75 static worker_t *SigWork;
  76 static int MasterPtyFd = -1;
  77 static int CopyFileFd = -1;
  78 static pid_t SigPid;
  79 static const char *WorkerFlavorPrt = "";        /* "" or "@flavor" */
  80
  81 #define MPTY_FAILED     -2
  82 #define MPTY_AGAIN      -1
  83 #define MPTY_EOF        0
  84 #define MPTY_DATA       1
  85
  86 int BuildTotal;
  87 int BuildCount;
  88 int BuildSkipCount;
  89 int BuildIgnoreCount;
  90 int BuildFailCount;
  91 int BuildSuccessCount;
  92
  93 /*
  94  * Initialize the WorkerAry[]
  95  */
  96 void
  97 DoInitBuild(int slot_override)
  98 {
  99         worker_t *work;
 100         struct stat st;
 101         int i;
 102
 103         ddassert(slot_override < 0 || MaxWorkers == 1);
 104
 105         bzero(WorkerAry, MaxWorkers * sizeof(worker_t));
 106         pthread_mutex_init(&WorkerMutex, NULL);
 107
 108         for (i = 0; i < MaxWorkers; ++i) {
 109                 work = &WorkerAry[i];
 110                 work->index = (slot_override >= 0) ? slot_override : i;
 111                 work->state = WORKER_NONE;
 112                 asprintf(&work->basedir, "%s/SL%02d", BuildBase, work->index);
 113                 pthread_cond_init(&work->cond, NULL);
 114         }
 115         BuildCount = 0;
 116
 117         /*
 118          * Create required sub-directories. The base directories must already
 119          * exist as a dsynth configuration safety.
 120          */
 121         if (stat(RepositoryPath, &st) < 0) {
 122                 if (mkdir(RepositoryPath, 0755) < 0)
 123                         dfatal("Cannot mkdir %s\n", RepositoryPath);
 124         }
 125
 126         BuildInitialized = 1;
 127
 128         /*
 129          * slow-start (increases at a rate of 1 per 5 seconds)
 130          */
 131         if (SlowStartOpt > MaxWorkers)
 132                 DynamicMaxWorkers = MaxWorkers;
 133         else if (SlowStartOpt > 0)
 134                 DynamicMaxWorkers = SlowStartOpt;
 135         else
 136                 DynamicMaxWorkers = MaxWorkers;
 137 }
 138
 139 /*
 140  * Called by the frontend to clean-up any hanging mounts.
 141  */
 142 void
 143 DoCleanBuild(int resetlogs)
 144 {
 145         int i;
 146
 147         ddassert(BuildInitialized);
 148
 149         if (resetlogs)
 150                 dlogreset();
 151         for (i = 0; i < MaxWorkers; ++i) {
 152                 DoWorkerUnmounts(&WorkerAry[i]);
 153         }
 154 }
 155
 156 void
 157 DoBuild(pkg_t *pkgs)
 158 {
 159         pkg_t *build_list = NULL;
 160         pkg_t **build_tail = &build_list;
 161         pkg_t *scan;
 162         bulk_t *bulk;
 163         int haswork = 1;
 164         int first = 1;
 165         int newtemplate;
 166         time_t startTime;
 167         time_t t;
 168         int h, m, s;
 169
 170         /*
 171          * We use our bulk system to run hooks.  This will be for
 172          * Skipped and Ignored.  Success and Failure are handled by
 173          * WorkerProcess() which is a separately-exec'd dsynth.
 174          */
 175         if (UsingHooks)
 176                 initbulk(childHookRun, MaxBulk);
 177
 178         /*
 179          * Count up the packages, not counting dummies
 180          */
 181         for (scan = pkgs; scan; scan = scan->bnext) {
 182                 if ((scan->flags & PKGF_DUMMY) == 0)
 183                         ++BuildTotal;
 184         }
 185
 186         doHook(NULL, "hook_run_start", HookRunStart, 1);
 187
 188         /*
 189          * The pkg and pkg-static binaries are needed.  If already present
 190          * then assume that the template is also valid, otherwise build
 191          * both.
 192          */
 193         scan = GetPkgPkg(pkgs);
 194
 195         /*
 196          * Create our template.  The template will be missing pkg
 197          * and pkg-static.
 198          */
 199         if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0) {
 200                 /* force a fresh template */
 201                 newtemplate = DoCreateTemplate(1);
 202         } else {
 203                 newtemplate = DoCreateTemplate(0);
 204         }
 205
 206         /*
 207          * This will clear the screen and set-up our gui, so sleep
 208          * a little first in case the user wants to see what was
 209          * printed before.
 210          */
 211         sleep(2);
 212         pthread_mutex_lock(&WorkerMutex);
 213         startTime = time(NULL);
 214         RunStatsInit();
 215         RunStatsReset();
 216
 217         /*
 218          * Build pkg/pkg-static.
 219          */
 220         if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0) {
 221                 build_list = scan;
 222                 build_tail = &scan->build_next;
 223                 startbuild(&build_list, &build_tail);
 224                 while (RunningWorkers == 1)
 225                         waitbuild(1, 0);
 226
 227                 if (scan->flags & PKGF_NOBUILD)
 228                         dfatal("Unable to build 'pkg'");
 229                 if (scan->flags & PKGF_ERROR)
 230                         dfatal("Error building 'pkg'");
 231                 if ((scan->flags & PKGF_SUCCESS) == 0)
 232                         dfatal("Error building 'pkg'");
 233                 newtemplate = 1;
 234         }
 235
 236         /*
 237          * Install pkg/pkg-static into the template
 238          */
 239         if (newtemplate) {
 240                 char *buf;
 241                 int rc;
 242
 243                 asprintf(&buf,
 244                          "cd %s/Template; "
 245                          "tar --exclude '+*' --exclude '*/man/*' "
 246                          "-xvzpf %s/%s > /dev/null 2>&1",
 247                          BuildBase,
 248                          RepositoryPath,
 249                          scan->pkgfile);
 250                 rc = system(buf);
 251                 if (rc)
 252                         dfatal("Command failed: %s\n", buf);
 253                 freestrp(&buf);
 254         }
 255
 256         /*
 257          * Calculate depi_depth, the longest chain of dependencies
 258          * for who depends on me, weighted by powers of two.
 259          */
 260         for (scan = pkgs; scan; scan = scan->bnext) {
 261                 buildCalculateDepiDepth(scan);
 262         }
 263
 264         /*
 265          * Nominal bulk build sequence
 266          */
 267         while (haswork) {
 268                 haswork = 0;
 269                 fflush(stdout);
 270                 for (scan = pkgs; scan; scan = scan->bnext) {
 271                         ddprintf(0, "SCANLEAVES %08x %s\n",
 272                                  scan->flags, scan->portdir);
 273                         scan->flags |= PKGF_BUILDLOOP;
 274                         /*
 275                          * NOTE: We must still find dependencies if PACKAGED
 276                          *       to fill in the gaps, as some of them may
 277                          *       need to be rebuilt.
 278                          */
 279                         if (scan->flags & (PKGF_SUCCESS | PKGF_FAILURE |
 280                                            PKGF_ERROR)) {
 281 #if 0
 282                                 ddprintf(0, "%s: already built\n",
 283                                          scan->portdir);
 284 #endif
 285                         } else {
 286                                 int ap = 0;
 287                                 build_find_leaves(NULL, scan, &build_tail,
 288                                                   &ap, &haswork, 0, first, 0);
 289                                 ddprintf(0, "TOPLEVEL %s %08x\n",
 290                                          scan->portdir, ap);
 291                         }
 292                         scan->flags &= ~PKGF_BUILDLOOP;
 293                         build_clear_trav(scan);
 294                 }
 295                 first = 0;
 296                 fflush(stdout);
 297                 startbuild(&build_list, &build_tail);
 298
 299                 if (haswork == 0 && RunningWorkers) {
 300                         waitbuild(RunningWorkers, 1);
 301                         haswork = 1;
 302                 }
 303         }
 304         pthread_mutex_unlock(&WorkerMutex);
 305
 306         RunStatsUpdateTop();
 307         RunStatsUpdateLogs();
 308         RunStatsSync();
 309         RunStatsDone();
 310
 311         doHook(NULL, "hook_run_end", HookRunEnd, 1);
 312         if (UsingHooks) {
 313                 while ((bulk = getbulk()) != NULL)
 314                         freebulk(bulk);
 315                 donebulk();
 316         }
 317
 318         t = time(NULL) - startTime;
 319         h = t / 3600;
 320         m = t / 60 % 60;
 321         s = t % 60;
 322
 323         printf("\n");
 324         printf("Initial queue size: %d\n", BuildTotal);
 325         printf("    packages built: %d\n", BuildSuccessCount);
 326         printf("           ignored: %d\n", BuildIgnoreCount);
 327         printf("           skipped: %d\n", BuildSkipCount);
 328         printf("            failed: %d\n", BuildFailCount);
 329         printf("\n");
 330         printf("Duration: %02d:%02d:%02d\n", h, m, s);
 331         printf("\n");
 332 }
 333
 334 /*
 335  * Traverse the packages (pkg) depends on recursively until we find
 336  * a leaf to build or report as unbuildable.  Calculates and assigns a
 337  * dependency count.  Returns all parallel-buildable packages.
 338  *
 339  * (pkg) itself is only added to the list if it is immediately buildable.
 340  */
 341 static
 342 int
 343 build_find_leaves(pkg_t *parent, pkg_t *pkg, pkg_t ***build_tailp,
 344                   int *app, int *hasworkp, int depth, int first,
 345                   int first_one_only)
 346 {
 347         pkglink_t *link;
 348         pkg_t *scan;
 349         int idep_count = 0;
 350         int apsub;
 351         int dfirst_one_only;
 352         int ndepth;
 353         char *buf;
 354
 355         ndepth = depth + 1;
 356
 357         /*
 358          * Already on build list, possibly in-progress, tell caller that
 359          * it is not ready.
 360          */
 361         ddprintf(ndepth, "sbuild_find_leaves %d %s %08x {\n",
 362                  depth, pkg->portdir, pkg->flags);
 363         if (pkg->flags & PKGF_BUILDLIST) {
 364                 ddprintf(ndepth, "} (already on build list)\n");
 365                 *app |= PKGF_NOTREADY;
 366                 return (pkg->idep_count);
 367         }
 368
 369         /*
 370          * Check dependencies
 371          */
 372         PKGLIST_FOREACH(link, &pkg->idepon_list) {
 373                 scan = link->pkg;
 374
 375                 if (scan == NULL) {
 376                         if (first_one_only)
 377                                 break;
 378                         continue;
 379                 }
 380                 ddprintf(ndepth, "check %s %08x\t", scan->portdir, scan->flags);
 381
 382                 /*
 383                  * If this dependency is to a DUMMY node it is a dependency
 384                  * only on the default flavor which is only the first node
 385                  * under this one, not all of them.
 386                  *
 387                  * NOTE: The depth is not being for complex dependency type
 388                  *       tests like it is in childInstallPkgDeps_recurse(),
 389                  *       so we don't have to hicup it like we do in that
 390                  *       procedure.
 391                  */
 392                 dfirst_one_only = (scan->flags & PKGF_DUMMY) ? 1 : 0;
 393
 394                 /*
 395                  * When accounting for a successful build, just bump
 396                  * idep_count by one.  scan->idep_count will heavily
 397                  * overlap packages that we count down multiple branches.
 398                  *
 399                  * We must still recurse through PACKAGED packages as
 400                  * some of their dependencies might be missing.
 401                  */
 402                 if (scan->flags & PKGF_SUCCESS) {
 403                         ddprintf(0, "SUCCESS - OK\n");
 404                         ++idep_count;
 405                         if (first_one_only)
 406                                 break;
 407                         continue;
 408                 }
 409
 410                 /*
 411                  * ERROR includes FAILURE, which is set in numerous situations
 412                  * including when NOBUILD state is processed.  So check for
 413                  * NOBUILD state first.
 414                  *
 415                  * An ERROR in a sub-package causes a NOBUILD in packages
 416                  * that depend on it.
 417                  */
 418                 if (scan->flags & PKGF_NOBUILD) {
 419                         ddprintf(0, "NOBUILD - OK "
 420                                     "(propogate failure upward)\n");
 421                         *app |= PKGF_NOBUILD_S;
 422                         if (first_one_only)
 423                                 break;
 424                         continue;
 425                 }
 426                 if (scan->flags & PKGF_ERROR) {
 427                         ddprintf(0, "ERROR - OK (propogate failure upward)\n");
 428                         *app |= PKGF_NOBUILD_S;
 429                         if (first_one_only)
 430                                 break;
 431                         continue;
 432                 }
 433
 434                 /*
 435                  * If already on build-list this dependency is not ready.
 436                  */
 437                 if (scan->flags & PKGF_BUILDLIST) {
 438                         ddprintf(0, " [BUILDLIST]");
 439                         *app |= PKGF_NOTREADY;
 440                 }
 441
 442                 /*
 443                  * If not packaged this dependency is not ready for
 444                  * the caller.
 445                  */
 446                 if ((scan->flags & PKGF_PACKAGED) == 0) {
 447                         ddprintf(0, " [NOT_PACKAGED]");
 448                         *app |= PKGF_NOTREADY;
 449                 }
 450
 451                 /*
 452                  * Reduce search complexity, if we have already processed
 453                  * scan in the traversal it will either already be on the
 454                  * build list or it will not be buildable.  Either way
 455                  * the parent is not buildable.
 456                  */
 457                 if (scan->flags & PKGF_BUILDTRAV) {
 458                         ddprintf(0, " [BUILDTRAV]\n");
 459                         *app |= PKGF_NOTREADY;
 460                         if (first_one_only)
 461                                 break;
 462                         continue;
 463                 }
 464
 465                 /*
 466                  * Assert on dependency loop
 467                  */
 468                 ++idep_count;
 469                 if (scan->flags & PKGF_BUILDLOOP) {
 470                         dfatal("pkg dependency loop %s -> %s",
 471                                 parent->portdir, scan->portdir);
 472                 }
 473
 474                 /*
 475                  * NOTE: For debug tabbing purposes we use (ndepth + 1)
 476                  *       here (i.e. depth + 2) in our iteration.
 477                  */
 478                 scan->flags |= PKGF_BUILDLOOP;
 479                 apsub = 0;
 480                 ddprintf(0, " SUBRECURSION {\n");
 481                 idep_count += build_find_leaves(pkg, scan, build_tailp,
 482                                                 &apsub, hasworkp,
 483                                                 ndepth + 1, first,
 484                                                 dfirst_one_only);
 485                 scan->flags &= ~PKGF_BUILDLOOP;
 486                 *app |= apsub;
 487                 if (apsub & PKGF_NOBUILD) {
 488                         ddprintf(ndepth, "} (sub-nobuild)\n");
 489                 } else if (apsub & PKGF_ERROR) {
 490                         ddprintf(ndepth, "} (sub-error)\n");
 491                 } else if (apsub & PKGF_NOTREADY) {
 492                         ddprintf(ndepth, "} (sub-notready)\n");
 493                 } else {
 494                         ddprintf(ndepth, "} (sub-ok)\n");
 495                 }
 496                 if (first_one_only)
 497                         break;
 498         }
 499         pkg->idep_count = idep_count;
 500         pkg->flags |= PKGF_BUILDTRAV;
 501
 502         /*
 503          * Incorporate scan results into pkg state.
 504          */
 505         if ((pkg->flags & PKGF_NOBUILD) == 0 && (*app & PKGF_NOBUILD)) {
 506                 *hasworkp = 1;
 507         } else if ((pkg->flags & PKGF_ERROR) == 0 && (*app & PKGF_ERROR)) {
 508                 *hasworkp = 1;
 509         }
 510         pkg->flags |= *app & ~PKGF_NOTREADY;
 511
 512         /*
 513          * Clear PACKAGED bit if sub-dependencies aren't clean
 514          */
 515         if ((pkg->flags & PKGF_PACKAGED) &&
 516             (pkg->flags & (PKGF_NOTREADY|PKGF_ERROR|PKGF_NOBUILD))) {
 517                 pkg->flags &= ~PKGF_PACKAGED;
 518                 ddassert(pkg->pkgfile);
 519                 asprintf(&buf, "%s/%s", RepositoryPath, pkg->pkgfile);
 520                 if (remove(buf) < 0) {
 521                         dlog(DLOG_ALL,
 522                              "[XXX] %s DELETE-PACKAGE %s (failed)\n",
 523                              pkg->portdir, buf);
 524                 } else {
 525                         dlog(DLOG_ALL,
 526                              "[XXX] %s DELETE-PACKAGE %s "
 527                              "(due to dependencies)\n",
 528                              pkg->portdir, buf);
 529                 }
 530                 freestrp(&buf);
 531                 *hasworkp = 1;
 532         }
 533
 534         /*
 535          * Set PKGF_NOBUILD_I if there is IGNORE data
 536          */
 537         if (pkg->ignore)
 538                 pkg->flags |= PKGF_NOBUILD_I;
 539
 540         /*
 541          * Handle propagated flags
 542          */
 543         if (pkg->flags & PKGF_ERROR) {
 544                 /*
 545                  * This can only happen if the ERROR has already been
 546                  * processed and accounted for.
 547                  */
 548                 ddprintf(depth, "} (ERROR - %s)\n", pkg->portdir);
 549         } else if (*app & PKGF_NOTREADY) {
 550                 /*
 551                  * We don't set PKGF_NOTREADY in the pkg, it is strictly
 552                  * a transient flag propagated via build_find_leaves().
 553                  *
 554                  * Just don't add the package to the list.
 555                  *
 556                  * NOTE: Even if NOBUILD is set (meaning we could list it
 557                  *       and let startbuild() finish it up as a skip, we
 558                  *       don't process it to the list because we want to
 559                  *       process all the dependencies, so someone doing a
 560                  *       manual build can get more complete information and
 561                  *       does not have to iterate each failed dependency one
 562                  *       at a time.
 563                  */
 564                 ;
 565         } else if (pkg->flags & PKGF_SUCCESS) {
 566                 ddprintf(depth, "} (SUCCESS - %s)\n", pkg->portdir);
 567         } else if (pkg->flags & PKGF_DUMMY) {
 568                 /*
 569                  * Just mark dummy packages as successful when all of their
 570                  * sub-depends (flavors) complete successfully.  Note that
 571                  * dummy packages are not counted in the total, so do not
 572                  * decrement BuildTotal.
 573                  */
 574                 ddprintf(depth, "} (DUMMY/META - SUCCESS)\n");
 575                 pkg->flags |= PKGF_SUCCESS;
 576                 *hasworkp = 1;
 577                 if (first) {
 578                         dlog(DLOG_ALL | DLOG_FILTER,
 579                              "[XXX] %s META-ALREADY-BUILT\n",
 580                              pkg->portdir);
 581                 } else {
 582                         dlog(DLOG_SUCC, "[XXX] %s meta-node complete\n",
 583                              pkg->portdir);
 584                 }
 585         } else if (pkg->flags & PKGF_PACKAGED) {
 586                 /*
 587                  * We can just mark the pkg successful.  If this is
 588                  * the first pass, we count this as an initial pruning
 589                  * pass and reduce BuildTotal.
 590                  */
 591                 ddprintf(depth, "} (PACKAGED - SUCCESS)\n");
 592                 pkg->flags |= PKGF_SUCCESS;
 593                 *hasworkp = 1;
 594                 if (first) {
 595                         dlog(DLOG_ALL | DLOG_FILTER,
 596                              "[XXX] %s ALREADY-BUILT\n",
 597                              pkg->portdir);
 598                         --BuildTotal;
 599                 }
 600         } else {
 601                 /*
 602                  * All dependencies are successful, queue new work
 603                  * and indicate not-ready to the parent (since our
 604                  * package has to be built).
 605                  *
 606                  * NOTE: The NOBUILD case propagates to here as well
 607                  *       and is ultimately handled by startbuild().
 608                  */
 609                 *hasworkp = 1;
 610                 if (pkg->flags & PKGF_NOBUILD_I)
 611                         ddprintf(depth, "} (ADDLIST(IGNORE/BROKEN) - %s)\n",
 612                                  pkg->portdir);
 613                 else if (pkg->flags & PKGF_NOBUILD)
 614                         ddprintf(depth, "} (ADDLIST(NOBUILD) - %s)\n",
 615                                  pkg->portdir);
 616                 else
 617                         ddprintf(depth, "} (ADDLIST - %s)\n", pkg->portdir);
 618                 pkg->flags |= PKGF_BUILDLIST;
 619                 **build_tailp = pkg;
 620                 *build_tailp = &pkg->build_next;
 621                 *app |= PKGF_NOTREADY;
 622         }
 623
 624         return idep_count;
 625 }
 626
 627 static
 628 void
 629 build_clear_trav(pkg_t *pkg)
 630 {
 631         pkglink_t *link;
 632         pkg_t *scan;
 633
 634         pkg->flags &= ~PKGF_BUILDTRAV;
 635         PKGLIST_FOREACH(link, &pkg->idepon_list) {
 636                 scan = link->pkg;
 637                 if (scan && (scan->flags & PKGF_BUILDTRAV))
 638                         build_clear_trav(scan);
 639         }
 640 }
 641
 642 /*
 643  * Calculate the longest chain of packages that depend on me.  The
 644  * long the chain, the more important my package is to build earlier
 645  * rather than later.
 646  */
 647 static int
 648 buildCalculateDepiDepth(pkg_t *pkg)
 649 {
 650         pkglink_t *link;
 651         pkg_t *scan;
 652         int best_depth = 0;
 653         int res;
 654
 655         if (pkg->depi_depth)
 656                 return(pkg->depi_depth + 1);
 657         pkg->flags |= PKGF_BUILDLOOP;
 658         PKGLIST_FOREACH(link, &pkg->deponi_list) {
 659                 scan = link->pkg;
 660                 if (scan && (scan->flags & PKGF_BUILDLOOP) == 0) {
 661                         res = buildCalculateDepiDepth(scan);
 662                         if (best_depth < res)
 663                                 best_depth = res;
 664                 }
 665         }
 666         pkg->flags &= ~PKGF_BUILDLOOP;
 667         pkg->depi_depth = best_depth;
 668
 669         return (best_depth + 1);
 670 }
 671
 672 /*
 673  * Take a list of pkg ready to go, sort it, and assign it to worker
 674  * slots.  This routine blocks in waitbuild() until it can dispose of
 675  * the entire list.
 676  *
 677  * WorkerMutex is held by the caller.
 678  */
 679 static
 680 void
 681 startbuild(pkg_t **build_listp, pkg_t ***build_tailp)
 682 {
 683         pkg_t *pkg;
 684         pkg_t **idep_ary;
 685         pkg_t **depi_ary;
 686         int count;
 687         int idep_index;
 688         int depi_index;
 689         int i;
 690         int n;
 691         worker_t *work;
 692         static int IterateWorker;
 693
 694         /*
 695          * Nothing to do
 696          */
 697         if (*build_listp == NULL)
 698                 return;
 699
 700         /*
 701          * Sort
 702          */
 703         count = 0;
 704         for (pkg = *build_listp; pkg; pkg = pkg->build_next)
 705                 ++count;
 706         idep_ary = calloc(count, sizeof(pkg_t *));
 707         depi_ary = calloc(count, sizeof(pkg_t *));
 708
 709         count = 0;
 710         for (pkg = *build_listp; pkg; pkg = pkg->build_next) {
 711                 idep_ary[count] = pkg;
 712                 depi_ary[count] = pkg;
 713                 ++count;
 714         }
 715
 716         /*
 717          * idep_ary - sorted by #of dependencies this pkg has.
 718          * depi_ary - sorted by #of other packages that depend on this pkg.
 719          */
 720         qsort(idep_ary, count, sizeof(pkg_t *), qsort_idep);
 721         qsort(depi_ary, count, sizeof(pkg_t *), qsort_depi);
 722
 723         idep_index = 0;
 724         depi_index = 0;
 725
 726         /*
 727          * Half the workers build based on the highest depi count,
 728          * the other half build based on the highest idep count.
 729          *
 730          * This is an attempt to get projects which many other projects
 731          * depend on built first, but to also try to build large projects
 732          * (which tend to have a lot of dependencies) earlier rather than
 733          * later so the end of the bulk run doesn't inefficiently build
 734          * the last few huge projects.
 735          *
 736          * Loop until we manage to assign slots to everyone.  We do not
 737          * wait for build completion.
 738          *
 739          * This is the point where we handle DUMMY packages (these are
 740          * dummy unflavored packages which 'cover' all the flavors for
 741          * a package).  These are not real packages are marked SUCCESS
 742          * at this time because their dependencies (the flavors) have all
 743          * been built.
 744          */
 745         while (idep_index != count || depi_index != count) {
 746                 pkg_t *pkgi;
 747                 pkg_t *ipkg;
 748
 749                 /*
 750                  * Find candidate to start sorted by depi or idep.
 751                  */
 752                 ipkg = NULL;
 753                 while (idep_index < count) {
 754                         ipkg = idep_ary[idep_index];
 755                         if ((ipkg->flags &
 756                              (PKGF_SUCCESS | PKGF_FAILURE |
 757                               PKGF_ERROR | PKGF_RUNNING)) == 0) {
 758                                 break;
 759                         }
 760                         ipkg = NULL;
 761                         ++idep_index;
 762                 }
 763
 764                 pkgi = NULL;
 765                 while (depi_index < count) {
 766                         pkgi = depi_ary[depi_index];
 767                         if ((pkgi->flags &
 768                              (PKGF_SUCCESS | PKGF_FAILURE |
 769                               PKGF_ERROR | PKGF_RUNNING)) == 0) {
 770                                 break;
 771                         }
 772                         pkgi = NULL;
 773                         ++depi_index;
 774                 }
 775
 776                 /*
 777                  * ipkg and pkgi must either both be NULL, or both
 778                  * be non-NULL.
 779                  */
 780                 if (ipkg == NULL && pkgi == NULL)
 781                         break;
 782                 ddassert(ipkg && pkgi);
 783
 784                 /*
 785                  * Handle the NOBUILD case right here, there's no point
 786                  * queueing it anywhere.
 787                  */
 788                 if (ipkg->flags & PKGF_NOBUILD) {
 789                         char *reason;
 790
 791                         ipkg->flags |= PKGF_FAILURE;
 792                         ipkg->flags &= ~PKGF_BUILDLIST;
 793
 794                         reason = buildskipreason(NULL, ipkg);
 795                         if (ipkg->flags & PKGF_NOBUILD_I) {
 796                                 ++BuildIgnoreCount;
 797                                 dlog(DLOG_IGN, "[XXX] %s ignored due to %s\n",
 798                                      ipkg->portdir, reason);
 799                                 doHook(ipkg, "hook_pkg_ignored",
 800                                        HookPkgIgnored, 0);
 801                         } else {
 802                                 ++BuildSkipCount;
 803                                 dlog(DLOG_SKIP, "[XXX] %s skipped due to %s\n",
 804                                      ipkg->portdir, reason);
 805                                 doHook(ipkg, "hook_pkg_skipped",
 806                                        HookPkgSkipped, 0);
 807                         }
 808                         free(reason);
 809                         ++BuildCount;
 810                         continue;
 811                 }
 812                 if (pkgi->flags & PKGF_NOBUILD) {
 813                         char *reason;
 814
 815                         pkgi->flags |= PKGF_FAILURE;
 816                         pkgi->flags &= ~PKGF_BUILDLIST;
 817
 818                         reason = buildskipreason(NULL, pkgi);
 819                         if (pkgi->flags & PKGF_NOBUILD_I) {
 820                                 ++BuildIgnoreCount;
 821                                 dlog(DLOG_IGN, "[XXX] %s ignored due to %s\n",
 822                                      pkgi->portdir, reason);
 823                                 doHook(pkgi, "hook_pkg_ignored",
 824                                        HookPkgIgnored, 0);
 825                         } else {
 826                                 ++BuildSkipCount;
 827                                 dlog(DLOG_SKIP, "[XXX] %s skipped due to %s\n",
 828                                      pkgi->portdir, reason);
 829                                 doHook(pkgi, "hook_pkg_skipped",
 830                                        HookPkgSkipped, 0);
 831                         }
 832                         free(reason);
 833                         ++BuildCount;
 834                         continue;
 835                 }
 836
 837                 /*
 838                  * Block while no slots are available.  waitbuild()
 839                  * will clean out any DONE states.
 840                  */
 841                 while (RunningWorkers >= DynamicMaxWorkers ||
 842                        RunningWorkers >= MaxWorkers - FailedWorkers) {
 843                         waitbuild(RunningWorkers, 1);
 844                 }
 845
 846                 /*
 847                  * Find an available worker slot, there should be at
 848                  * least one.
 849                  */
 850                 for (i = 0; i < MaxWorkers; ++i) {
 851                         n = IterateWorker % MaxWorkers;
 852                         work = &WorkerAry[n];
 853
 854                         if (work->state == WORKER_DONE ||
 855                             work->state == WORKER_FAILED) {
 856                                 workercomplete(work);
 857                         }
 858                         if (work->state == WORKER_NONE ||
 859                             work->state == WORKER_IDLE) {
 860                                 if (n <= MaxWorkers / 2) {
 861                                         startworker(pkgi, work);
 862                                 } else {
 863                                         startworker(ipkg, work);
 864                                 }
 865                                 /*RunStatsUpdate(work);*/
 866                                 break;
 867                         }
 868                         ++IterateWorker;
 869                 }
 870                 ddassert(i != MaxWorkers);
 871         }
 872         RunStatsSync();
 873
 874         /*
 875          * We disposed of the whole list
 876          */
 877         free(idep_ary);
 878         free(depi_ary);
 879         *build_listp = NULL;
 880         *build_tailp = build_listp;
 881 }
 882
 883 typedef const pkg_t *pkg_tt;
 884
 885 static int
 886 qsort_idep(const void *pkg1_arg, const void *pkg2_arg)
 887 {
 888         const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg;
 889         const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg;
 890
 891         return (pkg2->idep_count - pkg1->idep_count);
 892 }
 893
 894 static int
 895 qsort_depi(const void *pkg1_arg, const void *pkg2_arg)
 896 {
 897         const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg;
 898         const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg;
 899
 900         return ((pkg2->depi_count * pkg2->depi_depth) -
 901                 (pkg1->depi_count * pkg1->depi_depth));
 902 }
 903
 904 /*
 905  * Frontend starts a pkg up on a worker
 906  *
 907  * WorkerMutex must be held.
 908  */
 909 static void
 910 startworker(pkg_t *pkg, worker_t *work)
 911 {
 912         switch(work->state) {
 913         case WORKER_NONE:
 914                 pthread_create(&work->td, NULL, childBuilderThread, work);
 915                 work->state = WORKER_IDLE;
 916                 /* fall through */
 917         case WORKER_IDLE:
 918                 work->pkg_dep_size =
 919                 childInstallPkgDeps_recurse(NULL, &pkg->idepon_list, 0, 1, 0);
 920                 childInstallPkgDeps_recurse(NULL, &pkg->idepon_list, 1, 1, 0);
 921                 RunningPkgDepSize += work->pkg_dep_size;
 922
 923                 dlog(DLOG_ALL, "[%03d] START   %s "
 924                                "##idep=%02d depi=%02d/%02d dep=%-4.2fG\n",
 925                      work->index, pkg->portdir,
 926                      pkg->idep_count, pkg->depi_count, pkg->depi_depth,
 927                      (double)work->pkg_dep_size / (double)ONEGB);
 928
 929                 cleanworker(work);
 930                 pkg->flags |= PKGF_RUNNING;
 931                 work->pkg = pkg;
 932                 pthread_cond_signal(&work->cond);
 933                 ++RunningWorkers;
 934                 /*RunStatsUpdate(work);*/
 935                 break;
 936         case WORKER_PENDING:
 937         case WORKER_RUNNING:
 938         case WORKER_DONE:
 939         case WORKER_FAILED:
 940         case WORKER_FROZEN:
 941         case WORKER_EXITING:
 942         default:
 943                 dfatal("startworker: [%03d] Unexpected state %d for worker %d",
 944                        work->index, work->state, work->index);
 945                 break;
 946         }
 947 }
 948
 949 static void
 950 cleanworker(worker_t *work)
 951 {
 952         work->state = WORKER_PENDING;
 953         work->flags = 0;
 954         work->accum_error = 0;
 955         work->start_time = time(NULL);
 956 }
 957
 958 /*
 959  * Frontend finishes up a completed pkg on a worker.
 960  *
 961  * If the worker is in a FAILED state we clean the pkg out but (for now)
 962  * leave it in its failed state so we can debug.  At this point
 963  * workercomplete() will be called every once in a while on the state
 964  * and we have to deal with the NULL pkg.
 965  *
 966  * WorkerMutex must be held.
 967  */
 968 static void
 969 workercomplete(worker_t *work)
 970 {
 971         pkg_t *pkg;
 972         time_t t;
 973         int h;
 974         int m;
 975         int s;
 976
 977         /*
 978          * Steady state FAILED case.
 979          */
 980         if (work->state == WORKER_FAILED) {
 981                 if (work->pkg == NULL)
 982                         return;
 983         }
 984
 985         t = time(NULL) - work->start_time;
 986         h = t / 3600;
 987         m = t / 60 % 60;
 988         s = t % 60;
 989
 990         /*
 991          * Reduce total dep size
 992          */
 993         RunningPkgDepSize -= work->pkg_dep_size;
 994         RunningPkgDepSize -= work->memuse;
 995         work->pkg_dep_size = 0;
 996         work->memuse = 0;
 997
 998         /*
 999          * Process pkg out of the worker
1000          */
1001         pkg = work->pkg;
1002         if (pkg->flags & (PKGF_ERROR|PKGF_NOBUILD)) {
1003                 pkg->flags |= PKGF_FAILURE;
1004
1005                 /*
1006                  * This NOBUILD condition XXX can occur if the package is
1007                  * not allowed to be built.
1008                  */
1009                 if (pkg->flags & PKGF_NOBUILD) {
1010                         char *reason;
1011
1012                         reason = buildskipreason(NULL, pkg);
1013                         if (pkg->flags & PKGF_NOBUILD_I) {
1014                                 ++BuildIgnoreCount;
1015                                 dlog(DLOG_SKIP, "[%03d] IGNORD %s - %s\n",
1016                                      work->index, pkg->portdir, reason);
1017                                 doHook(pkg, "hook_pkg_ignored",
1018                                        HookPkgIgnored, 0);
1019                         } else {
1020                                 ++BuildSkipCount;
1021                                 dlog(DLOG_SKIP, "[%03d] SKIPPD %s - %s\n",
1022                                      work->index, pkg->portdir, reason);
1023                                 doHook(pkg, "hook_pkg_skipped",
1024                                        HookPkgSkipped, 0);
1025                         }
1026                         free(reason);
1027                 } else {
1028                         ++BuildFailCount;
1029                         dlog(DLOG_FAIL | DLOG_RED,
1030                              "[%03d] FAILURE %s ##%16.16s %02d:%02d:%02d\n",
1031                              work->index, pkg->portdir,
1032                              getphasestr(work->phase),
1033                              h, m, s);
1034                         doHook(pkg, "hook_pkg_failure", HookPkgFailure, 0);
1035                 }
1036         } else {
1037                 pkg->flags |= PKGF_SUCCESS;
1038                 ++BuildSuccessCount;
1039                 dlog(DLOG_SUCC | DLOG_GRN,
1040                      "[%03d] SUCCESS %s ##%02d:%02d:%02d\n",
1041                      work->index, pkg->portdir, h, m, s);
1042                 doHook(pkg, "hook_pkg_success", HookPkgSuccess, 0);
1043         }
1044         ++BuildCount;
1045         pkg->flags &= ~PKGF_BUILDLIST;
1046         pkg->flags &= ~PKGF_RUNNING;
1047         work->pkg = NULL;
1048         --RunningWorkers;
1049
1050         if (work->state == WORKER_FAILED) {
1051                 dlog(DLOG_ALL, "[%03d] XXX/XXX WORKER IS IN A FAILED STATE\n",
1052                      work->index);
1053                 ++FailedWorkers;
1054         } else if (work->flags & WORKERF_FREEZE) {
1055                 dlog(DLOG_ALL, "[%03d] FROZEN(DEBUG) %s\n",
1056                      work->index, pkg->portdir);
1057                 work->state = WORKER_FROZEN;
1058         } else {
1059                 work->state = WORKER_IDLE;
1060         }
1061 }
1062
1063 /*
1064  * Wait for one or more workers to complete.
1065  *
1066  * WorkerMutex must be held.
1067  */
1068 static void
1069 waitbuild(int whilematch, int dynamicmax)
1070 {
1071         static time_t wblast_time;
1072         static time_t dmlast_time;
1073         struct timespec ts;
1074         worker_t *work;
1075         time_t t;
1076         int i;
1077
1078         if (whilematch == 0)
1079                 whilematch = 1;
1080
1081         while (RunningWorkers == whilematch) {
1082                 for (i = 0; i < MaxWorkers; ++i) {
1083                         work = &WorkerAry[i];
1084                         if (work->state == WORKER_DONE ||
1085                             work->state == WORKER_FAILED) {
1086                                 workercomplete(work);
1087                         } else {
1088                                 pthread_cond_signal(&work->cond);
1089                         }
1090                         RunStatsUpdate(work);
1091                 }
1092                 RunStatsUpdateTop();
1093                 RunStatsUpdateLogs();
1094                 RunStatsSync();
1095                 if (RunningWorkers == whilematch) {
1096                         clock_gettime(CLOCK_REALTIME, &ts);
1097                         ts.tv_sec += 1;
1098                         ts.tv_nsec = 0;
1099                         pthread_cond_timedwait(&WorkerCond, &WorkerMutex, &ts);
1100                 }
1101
1102                 /*
1103                  * Dynamically reduce MaxWorkers based on the load.  When
1104                  * the load exceeds 2 x ncpus we reduce the number of workers
1105                  * up to 75% of MaxWorkers @ (5 x ncpus) load.
1106                  *
1107                  * Dynamically reduce MaxWorkers based on swap use, starting
1108                  * at 10% swap and up to 75% of MaxWorkers at 40% swap.
1109                  *
1110                  * NOTE! Generally speaking this allows more workers to be
1111                  *       configured which helps in two ways.  First it allows
1112                  *       a higher build rate for smaller packages.  Second
1113                  *       it allows dsynth to ratchet-down the number of slots
1114                  *       when large packages are forcing the load up.
1115                  *
1116                  *       A high load doesn't hurt efficiency, but swap usage
1117                  *       due to loading the tmpfs in lots of worker slots up
1118                  *       with tons of pkg install's (pre-reqs for a build)
1119                  *       does.  Reducing the number of worker slots has a
1120                  *       huge beneficial effect on reducing swap use / paging.
1121                  */
1122                 t = time(NULL);
1123                 if (dynamicmax && (wblast_time == 0 ||
1124                                    (unsigned)(t - wblast_time) >= 5)) {
1125                         double min_load = 1.5 * NumCores;
1126                         double max_load = 5.0 * NumCores;
1127                         double min_swap = 0.10;
1128                         double max_swap = 0.40;
1129                         double dload[3];
1130                         double dswap;
1131                         int max1;
1132                         int max2;
1133                         int max3;
1134                         int max_sel;
1135                         int noswap;
1136
1137                         wblast_time = t;
1138
1139                         /*
1140                          * Cap based on load.  This is back-loaded.
1141                          */
1142                         getloadavg(dload, 3);
1143                         if (dload[0] < min_load) {
1144                                 max1 = MaxWorkers;
1145                         } else if (dload[0] <= max_load) {
1146                                 max1 = MaxWorkers -
1147                                        MaxWorkers * 0.75 *
1148                                        (dload[0] - min_load) /
1149                                        (max_load - min_load);
1150                         } else {
1151                                 max1 = MaxWorkers * 25 / 100;
1152                         }
1153
1154                         /*
1155                          * Cap based on swap use.  This is back-loaded.
1156                          */
1157                         dswap = getswappct(&noswap);
1158                         if (dswap < min_swap) {
1159                                 max2 = MaxWorkers;
1160                         } else if (dswap <= max_swap) {
1161                                 max2 = MaxWorkers -
1162                                        MaxWorkers * 0.75 *
1163                                        (dswap - min_swap) /
1164                                        (max_swap - min_swap);
1165                         } else {
1166                                 max2 = MaxWorkers * 25 / 100;
1167                         }
1168
1169                         /*
1170                          * Cap based on aggregate pkg-dependency memory
1171                          * use installed in worker slots.  This is
1172                          * front-loaded.
1173                          *
1174                          * Since it can take a while for workers to retire
1175                          * (to reduce RunningPkgDepSize), just set our
1176                          * target 1 below the current run count to allow
1177                          * jobs to retire without being replaced with new
1178                          * jobs.
1179                          *
1180                          * In addition, in order to avoid a paging 'shock',
1181                          * We enforce a 30 second-per-increment slow-start
1182                          * once RunningPkgDepSize exceeds 1/2 the target.
1183                          */
1184                         if (RunningPkgDepSize > PkgDepMemoryTarget) {
1185                                 max3 = RunningWorkers - 1;
1186                         } else if (RunningPkgDepSize > PkgDepMemoryTarget / 2) {
1187                                 if (dmlast_time == 0 ||
1188                                     (unsigned)(t - dmlast_time) >= 30) {
1189                                         dmlast_time = t;
1190                                         max3 = RunningWorkers + 1;
1191                                 } else {
1192                                         max3 = RunningWorkers;
1193                                 }
1194                         } else {
1195                                 max3 = MaxWorkers;
1196                         }
1197
1198                         /*
1199                          * Priority reduction, convert to DynamicMaxWorkers
1200                          */
1201                         max_sel = max1;
1202                         if (max_sel > max2)
1203                                 max_sel = max2;
1204                         if (max_sel > max3)
1205                                 max_sel = max3;
1206
1207                         /*
1208                          * Restrict to allowed range, and also handle
1209                          * slow-start.
1210                          */
1211                         if (max_sel < 1)
1212                                 max_sel = 1;
1213                         if (max_sel > DynamicMaxWorkers + 1)
1214                                 max_sel = DynamicMaxWorkers + 1;
1215                         if (max_sel > MaxWorkers)
1216                                 max_sel = MaxWorkers;
1217
1218                         /*
1219                          * Stop waiting if DynamicMaxWorkers is going to
1220                          * increase.
1221                          */
1222                         if (DynamicMaxWorkers < max1)
1223                                 whilematch = -1;
1224
1225                         /*
1226                          * And adjust
1227                          */
1228                         if (DynamicMaxWorkers != max1) {
1229                                 dlog(DLOG_ALL | DLOG_FILTER,
1230                                      "[XXX] Load=%-6.2f(%2d) "
1231                                      "Swap=%-3.2f%%(%2d) "
1232                                      "Mem=%3.2fG(%2d) "
1233                                      "Adjust Workers %d->%d\n",
1234                                      dload[0], max1,
1235                                      dswap * 100.0, max2,
1236                                      RunningPkgDepSize / (double)ONEGB, max3,
1237                                      DynamicMaxWorkers, max_sel);
1238                                 DynamicMaxWorkers = max_sel;
1239                         }
1240                 }
1241         }
1242 }
1243
1244
1245 /*
1246  * Worker pthread (WorkerAry)
1247  *
1248  * This thread belongs to the dsynth master process and handled a worker slot.
1249  * (this_thread) -> WORKER fork/exec (WorkerPocess) -> (pty) -> sub-processes.
1250  */
1251 static void *
1252 childBuilderThread(void *arg)
1253 {
1254         char *envary[1] = { NULL };
1255         worker_t *work = arg;
1256         wmsg_t wmsg;
1257         pkg_t *pkg;
1258         pid_t pid;
1259         int status;
1260         volatile int dowait;
1261         char slotbuf[8];
1262         char fdbuf[8];
1263         char flagsbuf[16];
1264
1265         pthread_mutex_lock(&WorkerMutex);
1266         while (work->terminate == 0) {
1267                 dowait = 1;
1268
1269                 switch(work->state) {
1270                 case WORKER_IDLE:
1271                         break;
1272                 case WORKER_PENDING:
1273                         /*
1274                          * Fork the management process for the pkg operation
1275                          * on this worker slot.
1276                          *
1277                          * This process will set up the environment, do the
1278                          * mounts, will become the reaper, and will process
1279                          * pipe commands and handle chroot operations.
1280                          *
1281                          * NOTE: If SOCK_CLOEXEC is not supported WorkerMutex
1282                          *       is sufficient to interlock F_SETFD FD_CLOEXEC
1283                          *       operations.
1284                          */
1285                         ddassert(work->pkg);
1286                         if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC,
1287                                        PF_UNSPEC, work->fds)) {
1288                                 dfatal_errno("socketpair() during worker fork");
1289                         }
1290                         snprintf(slotbuf, sizeof(slotbuf),
1291                                  "%d", work->index);
1292                         snprintf(fdbuf, sizeof(fdbuf),
1293                                  "3");
1294                         snprintf(flagsbuf, sizeof(flagsbuf),
1295                                  "%d", WorkerProcFlags);
1296
1297                         /*
1298                          * fds[0] - master
1299                          * fds[1] - slave
1300                          *
1301                          * We pass the salve descriptor in fd 3 and close all
1302                          * other descriptors for security.
1303                          */
1304                         pthread_mutex_unlock(&WorkerMutex);
1305                         pid = vfork();
1306                         if (pid == 0) {
1307                                 close(work->fds[0]);
1308                                 dup2(work->fds[1], 3);
1309                                 closefrom(4);
1310                                 fcntl(3, F_SETFD, 0);
1311                                 execle(DSynthExecPath, DSynthExecPath,
1312                                        "WORKER", slotbuf, fdbuf,
1313                                        work->pkg->portdir, work->pkg->pkgfile,
1314                                        flagsbuf,
1315                                        NULL, envary);
1316                                 write(2, "EXECLE FAILURE\n", 15);
1317                                 _exit(1);
1318                         }
1319                         pthread_mutex_lock(&WorkerMutex);
1320                         close(work->fds[1]);
1321                         work->phase = PHASE_PENDING;
1322                         work->lines = 0;
1323                         work->memuse = 0;
1324                         work->pid = pid;
1325                         work->state = WORKER_RUNNING;
1326                         /* fall through */
1327                 case WORKER_RUNNING:
1328                         /*
1329                          * Poll for status updates, if NULL is returned
1330                          * and status is non-zero, the communications link
1331                          * failed unexpectedly.
1332                          */
1333                         pkg = work->pkg;
1334                         pthread_mutex_unlock(&WorkerMutex);
1335                         status = ipcreadmsg(work->fds[0], &wmsg);
1336                         pthread_mutex_lock(&WorkerMutex);
1337
1338                         if (status == 0) {
1339                                 /*
1340                                  * Normal message, can include normal
1341                                  * termination which changes us over
1342                                  * to another state.
1343                                  */
1344                                 dowait = 0;
1345                                 switch(wmsg.cmd) {
1346                                 case WMSG_CMD_INSTALL_PKGS:
1347                                         wmsg.cmd = WMSG_RES_INSTALL_PKGS;
1348                                         wmsg.status = childInstallPkgDeps(work);
1349                                         pthread_mutex_unlock(&WorkerMutex);
1350                                         ipcwritemsg(work->fds[0], &wmsg);
1351                                         pthread_mutex_lock(&WorkerMutex);
1352                                         break;
1353                                 case WMSG_CMD_STATUS_UPDATE:
1354                                         work->phase = wmsg.phase;
1355                                         work->lines = wmsg.lines;
1356                                         if (work->memuse != wmsg.memuse) {
1357                                                 RunningPkgDepSize +=
1358                                                 wmsg.memuse - work->memuse;
1359                                                 work->memuse = wmsg.memuse;
1360                                         }
1361                                         break;
1362                                 case WMSG_CMD_SUCCESS:
1363                                         work->flags |= WORKERF_SUCCESS;
1364                                         break;
1365                                 case WMSG_CMD_FAILURE:
1366                                         work->flags |= WORKERF_FAILURE;
1367                                         break;
1368                                 case WMSG_CMD_FREEZEWORKER:
1369                                         work->flags |= WORKERF_FREEZE;
1370                                         break;
1371                                 default:
1372                                         break;
1373                                 }
1374                                 RunStatsUpdate(work);
1375                                 RunStatsSync();
1376                         } else {
1377                                 close(work->fds[0]);
1378                                 pthread_mutex_unlock(&WorkerMutex);
1379                                 while (waitpid(work->pid, &status, 0) < 0 &&
1380                                        errno == EINTR) {
1381                                         ;
1382                                 }
1383                                 pthread_mutex_lock(&WorkerMutex);
1384
1385                                 if (work->flags & WORKERF_SUCCESS) {
1386                                         pkg->flags |= PKGF_SUCCESS;
1387                                         work->state = WORKER_DONE;
1388                                 } else if (work->flags & WORKERF_FAILURE) {
1389                                         pkg->flags |= PKGF_FAILURE;
1390                                         work->state = WORKER_DONE;
1391                                 } else {
1392                                         pkg->flags |= PKGF_FAILURE;
1393                                         work->state = WORKER_FAILED;
1394                                 }
1395                                 work->flags |= WORKERF_STATUS_UPDATE;
1396                                 pthread_cond_signal(&WorkerCond);
1397                         }
1398                         break;
1399                 case WORKER_DONE:
1400                         /*
1401                          * pkg remains attached until frontend processes the
1402                          * completion.  The frontend will then set the state
1403                          * back to idle.
1404                          */
1405                         break;
1406                 case WORKER_FAILED:
1407                         /*
1408                          * A worker failure means that the worker did not
1409                          * send us a WMSG_CMD_SUCCESS or WMSG_CMD_FAILURE
1410                          * ipc before terminating.
1411                          *
1412                          * We just sit in this state until the front-end
1413                          * does something about it.
1414                          */
1415                         break;
1416                 default:
1417                         dfatal("worker: [%03d] Unexpected state %d for worker %d",
1418                                work->index, work->state, work->index);
1419                         /* NOT REACHED */
1420                         break;
1421                 }
1422
1423                 /*
1424                  * The dsynth frontend will poll us approximately once
1425                  * a second (its variable).
1426                  */
1427                 if (dowait)
1428                         pthread_cond_wait(&work->cond, &WorkerMutex);
1429         }
1430
1431         /*
1432          * Scrap the comm socket if running, this should cause the worker
1433          * process to kill its sub-programs and cleanup.
1434          */
1435         if (work->state == WORKER_RUNNING) {
1436                 pthread_mutex_unlock(&WorkerMutex);
1437                 close(work->fds[0]);
1438                 while (waitpid(work->pid, &status, 0) < 0 &&
1439                        errno == EINTR);
1440                 pthread_mutex_lock(&WorkerMutex);
1441         }
1442
1443         /*
1444          * Final handshake
1445          */
1446         work->state = WORKER_EXITING;
1447         pthread_cond_signal(&WorkerCond);
1448         pthread_mutex_unlock(&WorkerMutex);
1449
1450         return NULL;
1451 }
1452
1453 /*
1454  * Install all the binary packages (we have already built them) that
1455  * the current work package depends on, without duplicates, in a script
1456  * which will be run from within the specified work jail.
1457  *
1458  * Locked by WorkerMutex (global)
1459  */
1460 static int
1461 childInstallPkgDeps(worker_t *work)
1462 {
1463         char *buf;
1464         FILE *fp;
1465
1466         if (PKGLIST_EMPTY(&work->pkg->idepon_list))
1467                 return 0;
1468
1469         asprintf(&buf, "%s/tmp/dsynth_install_pkgs", work->basedir);
1470         fp = fopen(buf, "w");
1471         ddassert(fp != NULL);
1472         fprintf(fp, "#!/bin/sh\n");
1473         fprintf(fp, "#\n");
1474         fchmod(fileno(fp), 0755);
1475
1476         childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 0, 1, 0);
1477         childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 1, 1, 0);
1478         fprintf(fp, "\nexit 0\n");
1479         fclose(fp);
1480         freestrp(&buf);
1481
1482         return 1;
1483 }
1484
1485 static size_t
1486 childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list, int undoit,
1487                             int depth, int first_one_only)
1488 {
1489         pkglink_t *link;
1490         pkg_t *pkg;
1491         size_t tot = 0;
1492         int ndepth;
1493         int nfirst;
1494
1495         PKGLIST_FOREACH(link, list) {
1496                 pkg = link->pkg;
1497
1498                 /*
1499                  * We don't want to mess up our depth test just below if
1500                  * a DUMMY node had to be inserted.  The nodes under the
1501                  * dummy node.
1502                  *
1503                  * The elements under a dummy node represent all the flabor,
1504                  * a dependency that directly references a dummy node only
1505                  * uses the first flavor (first_one_only / nfirst).
1506                  */
1507                 ndepth = (pkg->flags & PKGF_DUMMY) ? depth : depth + 1;
1508                 nfirst = (pkg->flags & PKGF_DUMMY) ? 1 : 0;
1509
1510                 /*
1511                  * We only need all packages for the top-level dependencies.
1512                  * The deeper ones only need DEP_TYPE_LIB and DEP_TYPE_RUN
1513                  * (types greater than DEP_TYPE_BUILD) since they are already
1514                  * built.
1515                  */
1516                 if (depth > 1 && link->dep_type <= DEP_TYPE_BUILD) {
1517                         if (first_one_only)
1518                                 break;
1519                         continue;
1520                 }
1521
1522                 if (undoit) {
1523                         if (pkg->dsynth_install_flg == 1) {
1524                                 pkg->dsynth_install_flg = 0;
1525                                 tot += childInstallPkgDeps_recurse(fp,
1526                                                             &pkg->idepon_list,
1527                                                             undoit,
1528                                                             ndepth, nfirst);
1529                         }
1530                         if (first_one_only)
1531                                 break;
1532                         continue;
1533                 }
1534                 if (pkg->dsynth_install_flg) {
1535                         if (DebugOpt >= 2 && pkg->pkgfile && fp) {
1536                                 fprintf(fp, "echo 'AlreadyHave %s'\n",
1537                                         pkg->pkgfile);
1538                         }
1539                         if (first_one_only)
1540                                 break;
1541                         continue;
1542                 }
1543
1544                 tot += childInstallPkgDeps_recurse(fp, &pkg->idepon_list,
1545                                                    undoit, ndepth, nfirst);
1546                 if (pkg->dsynth_install_flg) {
1547                         if (first_one_only)
1548                                 break;
1549                         continue;
1550                 }
1551                 pkg->dsynth_install_flg = 1;
1552
1553                 /*
1554                  * If this is a dummy node with no package, the originator
1555                  * is requesting a flavored package.  We select the default
1556                  * flavor which we presume is the first one.
1557                  */
1558                 if (pkg->pkgfile == NULL && (pkg->flags & PKGF_DUMMY)) {
1559                         pkg_t *spkg = pkg->idepon_list.next->pkg;
1560
1561                         if (spkg) {
1562                                 pkg = spkg;
1563                                 if (fp) {
1564                                         fprintf(fp,
1565                                                 "echo 'DUMMY use %s (%p)'\n",
1566                                                 pkg->portdir, pkg->pkgfile);
1567                                 }
1568                         } else {
1569                                 if (fp) {
1570                                         fprintf(fp,
1571                                                 "echo 'CANNOT FIND DEFAULT "
1572                                                 "FLAVOR FOR %s'\n",
1573                                                 pkg->portdir);
1574                                 }
1575                         }
1576                 }
1577
1578                 /*
1579                  * Generate package installation command
1580                  */
1581                 if (fp && pkg->pkgfile) {
1582                         fprintf(fp, "echo 'Installing /packages/All/%s'\n",
1583                                 pkg->pkgfile);
1584                         fprintf(fp, "pkg install -q -y /packages/All/%s "
1585                                 "|| exit 1\n",
1586                                 pkg->pkgfile);
1587                 } else if (fp) {
1588                         fprintf(fp, "echo 'CANNOT FIND PKG FOR %s'\n",
1589                                 pkg->portdir);
1590                 }
1591
1592                 if (pkg->pkgfile) {
1593                         struct stat st;
1594                         char *path;
1595                         char *ptr;
1596
1597                         asprintf(&path, "%s/%s", RepositoryPath, pkg->pkgfile);
1598                         ptr = strrchr(pkg->pkgfile, '.');
1599                         if (stat(path, &st) == 0) {
1600                                 if (strcmp(ptr, ".tar") == 0)
1601                                         tot += st.st_size;
1602                                 else if (strcmp(ptr, ".tgz") == 0)
1603                                         tot += st.st_size * 3;
1604                                 else if (strcmp(ptr, ".txz") == 0)
1605                                         tot += st.st_size * 5;
1606                                 else if (strcmp(ptr, ".tbz") == 0)
1607                                         tot += st.st_size * 3;
1608                                 else
1609                                         tot += st.st_size * 2;
1610                         }
1611                         free(path);
1612                 }
1613                 if (first_one_only)
1614                         break;
1615         }
1616         return tot;
1617 }
1618
1619 /*
1620  * Worker process interactions.
1621  *
1622  * The worker process is responsible for managing the build of a single
1623  * package.  It is exec'd by the master dsynth and only loads the
1624  * configuration.
1625  *
1626  * This process does not run in the chroot.  It will become the reaper for
1627  * all sub-processes and it will enter the chroot to execute various phases.
1628  * It catches SIGINTR, SIGHUP, and SIGPIPE and will iterate, terminate, and
1629  * reap all sub-process upon kill or exit.
1630  *
1631  * The command line forwarded to this function is:
1632  *
1633  *      WORKER slot# socketfd portdir/subdir
1634  *
1635  * TERM=dumb
1636  * USER=root
1637  * HOME=/root
1638  * LANG=C
1639  * SSL_NO_VERIFY_PEER=1
1640  * USE_PACKAGE_DEPENDS_ONLY=1   (exec_phase_depends)
1641  * PORTSDIR=/xports
1642  * PORT_DBDIR=/options          For ports options
1643  * PACKAGE_BUILDING=yes         Don't build packages that aren't legally
1644  *                              buildable for a binary repo.
1645  * PKG_DBDIR=/var/db/pkg
1646  * PKG_CACHEDIR=/var/cache/pkg
1647  * PKG_CREATE_VERBOSE=yes       Ensure periodic output during packaging
1648  * (custom environment)
1649  * PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
1650  * UNAME_s=DragonFly            (example)
1651  * UNAME_v=DragonFly 5.7-SYNTH  (example)
1652  * UNAME_p=x86_64               (example)
1653  * UNAME_m=x86_64               (example)
1654  * UNAME_r=5.7-SYNTH            (example)
1655  * NO_DEPENDS=yes               (exec_phase)
1656  * DISTDIR=/distfiles
1657  * WRKDIRPREFIX=/construction
1658  * BATCH=yes
1659  * MAKE_JOBS_NUMBER=n
1660  *
1661  * SETUP:
1662  *      ldconfig -R
1663  *      /usr/local/sbin/pkg-static install /packages/All/<the pkg pkg>
1664  *      /usr/local/sbin/pkg-static install /packages/All/<pkg>
1665  *                      (for all dependencies)
1666  *
1667  * PHASES:              make -C path FLAVOR=flavor <phase>
1668  *      check-sanity
1669  *      pkg-depends
1670  *      fetch-depends
1671  *      fetch
1672  *      checksum
1673  *      extract-depends
1674  *      extract
1675  *      patch-depends
1676  *      patch
1677  *      build-depends
1678  *      lib-depends
1679  *      configure
1680  *      build
1681  *      run-depends
1682  *      stage
1683  *      test            (skipped)
1684  *      check-plist
1685  *      package          e.g. /construction/lang/perl5.28/pkg/perl5-5.28.2.txz
1686  *      install-mtree   (skipped)
1687  *      install         (skipped)
1688  *      deinstall       (skipped)
1689  */
1690 void
1691 WorkerProcess(int ac, char **av)
1692 {
1693         wmsg_t wmsg;
1694         int fd;
1695         int slot;
1696         int tmpfd;
1697         int pkgpkg = 0;
1698         int status;
1699         int len;
1700         int do_install_phase;
1701         char *portdir;
1702         char *pkgfile;
1703         char *flavor;
1704         char *buf;
1705         worker_t *work;
1706         bulk_t *bulk;
1707         pkg_t pkg;
1708         buildenv_t *benv;
1709         FILE *fp;
1710
1711         /*
1712          * Parse arguments
1713          */
1714         if (ac != 6) {
1715                 dlog(DLOG_ALL, "WORKER PROCESS %d- bad arguments\n", getpid());
1716                 exit(1);
1717         }
1718         slot = strtol(av[1], NULL, 0);
1719         fd = strtol(av[2], NULL, 0);    /* master<->slave messaging */
1720         portdir = av[3];
1721         pkgfile = av[4];
1722         flavor = strchr(portdir, '@');
1723         if (flavor) {
1724                 *flavor++ = 0;
1725                 asprintf(&buf, "@%s", flavor);
1726                 WorkerFlavorPrt = buf;
1727                 buf = NULL;     /* safety */
1728         }
1729         WorkerProcFlags = strtol(av[5], NULL, 0);
1730
1731         bzero(&wmsg, sizeof(wmsg));
1732
1733         setproctitle("[%02d] WORKER STARTUP  %s%s",
1734                      slot, portdir, WorkerFlavorPrt);
1735
1736         if (strcmp(portdir, "ports-mgmt/pkg") == 0)
1737                 pkgpkg = 1;
1738
1739         signal(SIGTERM, phaseTerminateSignal);
1740         signal(SIGINT, phaseTerminateSignal);
1741         signal(SIGHUP, phaseTerminateSignal);
1742
1743         /*
1744          * Set up the environment
1745          */
1746         setenv("TERM", "dumb", 1);
1747         setenv("USER", "root", 1);
1748         setenv("HOME", "/root", 1);
1749         setenv("LANG", "C", 1);
1750         setenv("SSL_NO_VERIFY_PEER", "1", 1);
1751
1752         addbuildenv("USE_PACKAGE_DEPENDS_ONLY", "yes", BENV_MAKECONF);
1753         addbuildenv("PORTSDIR", "/xports", BENV_MAKECONF);
1754         addbuildenv("PORT_DBDIR", "/options", BENV_MAKECONF);
1755         addbuildenv("PKG_DBDIR", "/var/db/pkg", BENV_MAKECONF);
1756         addbuildenv("PKG_CACHEDIR", "/var/cache/pkg", BENV_MAKECONF);
1757         addbuildenv("PKG_SUFX", USE_PKG_SUFX, BENV_MAKECONF);
1758         if (WorkerProcFlags & WORKER_PROC_DEVELOPER)
1759                 addbuildenv("DEVELOPER", "1", BENV_MAKECONF);
1760
1761         /*
1762          * CCache is a horrible unreliable hack but... leave the
1763          * mechanism in-place in case someone has a death wish.
1764          */
1765         if (UseCCache) {
1766                 addbuildenv("WITH_CCACHE_BUILD", "yes", BENV_MAKECONF);
1767                 addbuildenv("CCACHE_DIR", "/ccache", BENV_MAKECONF);
1768         }
1769
1770         addbuildenv("UID", "0", BENV_MAKECONF);
1771         addbuildenv("ARCH", ArchitectureName, BENV_MAKECONF);
1772
1773 #ifdef __DragonFly__
1774         addbuildenv("OPSYS", "DragonFly", BENV_MAKECONF);
1775         addbuildenv("DFLYVERSION", VersionFromParamHeader, BENV_MAKECONF);
1776         addbuildenv("OSVERSION", "9999999", BENV_MAKECONF);
1777 #else
1778 #error "Need OS-specific data to generate make.conf"
1779 #endif
1780
1781         addbuildenv("OSREL", ReleaseName, BENV_MAKECONF);
1782         addbuildenv("_OSRELEASE", VersionOnlyName, BENV_MAKECONF);
1783
1784         setenv("PATH",
1785                "/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin",
1786                1);
1787
1788         setenv("UNAME_s", OperatingSystemName, 1);
1789         setenv("UNAME_v", VersionName, 1);
1790         setenv("UNAME_p", ArchitectureName, 1);
1791         setenv("UNAME_m", MachineName, 1);
1792         setenv("UNAME_r", ReleaseName, 1);
1793
1794         addbuildenv("NO_DEPENDS", "yes", BENV_MAKECONF);
1795         addbuildenv("DISTDIR", "/distfiles", BENV_MAKECONF);
1796         addbuildenv("WRKDIRPREFIX", "/construction", BENV_MAKECONF);
1797         addbuildenv("BATCH", "yes", BENV_MAKECONF);
1798
1799         /*
1800          * Special consideration
1801          *
1802          * PACKAGE_BUILDING     - Disallow packaging ports which do not allow
1803          *                        for binary distribution.
1804          *
1805          * PKG_CREATE_VERBOSE   - Ensure periodic output during the packaging
1806          *                        process to avoid a watchdog timeout.
1807          *
1808          */
1809         addbuildenv("PACKAGE_BUILDING", "yes", BENV_MAKECONF);
1810         addbuildenv("PKG_CREATE_VERBOSE", "yes", BENV_MAKECONF);
1811         asprintf(&buf, "%d", MaxJobs);
1812         addbuildenv("MAKE_JOBS_NUMBER", buf, BENV_MAKECONF);
1813         freestrp(&buf);
1814
1815         if (flavor)
1816                 setenv("FLAVOR", flavor, 1);
1817
1818         /*
1819          * Become the reaper
1820          */
1821         if (procctl(P_PID, getpid(), PROC_REAP_ACQUIRE, NULL) < 0)
1822                 dfatal_errno("procctl() - Cannot become reaper");
1823
1824         /*
1825          * Initialize a worker structure
1826          */
1827         DoInitBuild(slot);
1828
1829         bzero(&pkg, sizeof(pkg));
1830         pkg.portdir = portdir;          /* sans flavor */
1831         pkg.pkgfile = pkgfile;
1832         if (strchr(portdir, '/'))
1833                 len = strchr(portdir, '/') - portdir;
1834         else
1835                 len = 0;
1836
1837         /*
1838          * Setup the logfile
1839          */
1840         asprintf(&pkg.logfile,
1841                  "%s/%*.*s___%s%s%s.log",
1842                  LogsPath, len, len, portdir,
1843                  ((portdir[len] == '/') ? portdir + len + 1 : portdir + len),
1844                  (flavor ? "@" : ""),
1845                  (flavor ? flavor : ""));
1846         tmpfd = open(pkg.logfile, O_RDWR|O_CREAT|O_TRUNC, 0666);
1847         if (tmpfd >= 0) {
1848                 if (DebugOpt >= 2) {
1849                         dlog(DLOG_ALL, "[%03d] %s LOGFILE %s\n",
1850                              slot, pkg.portdir, pkg.logfile);
1851                 }
1852                 close(tmpfd);
1853         } else {
1854                 dlog(DLOG_ALL, "[%03d] LOGFILE %s (create failed)\n",
1855                      slot, pkg.logfile);
1856         }
1857
1858         /*
1859          * Setup the work structure.  Because this is an exec'd sub-process,
1860          * there is only one work structure.
1861          */
1862         work = &WorkerAry[0];
1863         work->flavor = flavor;
1864         work->fds[0] = fd;
1865         work->pkg = &pkg;
1866         work->start_time = time(NULL);
1867
1868         /*
1869          * Do mounts
1870          */
1871         SigWork = work;
1872         setproctitle("[%02d] WORKER MOUNTS   %s%s",
1873                      slot, portdir, WorkerFlavorPrt);
1874         DoWorkerMounts(work);
1875
1876         /*
1877          * Generate an /etc/make.conf in the build base
1878          */
1879         asprintf(&buf, "%s/etc/make.conf", work->basedir);
1880         fp = fopen(buf, "w");
1881         dassert_errno(fp, "Unable to create %s\n", buf);
1882         for (benv = BuildEnv; benv; benv = benv->next) {
1883                 if (benv->type & BENV_PKGLIST)
1884                         continue;
1885                 if ((benv->type & BENV_CMDMASK) == BENV_MAKECONF) {
1886                         if (DebugOpt >= 2) {
1887                                 dlog(DLOG_ALL, "[%03d] ENV %s=%s\n",
1888                                      slot, benv->label, benv->data);
1889                         }
1890                         fprintf(fp, "%s=%s\n", benv->label, benv->data);
1891                 }
1892         }
1893         fclose(fp);
1894         freestrp(&buf);
1895
1896         /*
1897          * Set up our hooks
1898          */
1899         if (UsingHooks)
1900                 initbulk(childHookRun, MaxBulk);
1901
1902         /*
1903          * Start phases
1904          */
1905         wmsg.cmd = WMSG_CMD_INSTALL_PKGS;
1906         ipcwritemsg(fd, &wmsg);
1907         status = ipcreadmsg(fd, &wmsg);
1908         if (status < 0 || wmsg.cmd != WMSG_RES_INSTALL_PKGS)
1909                 dfatal("pkg installation handshake failed");
1910         do_install_phase = wmsg.status;
1911
1912         wmsg.cmd = WMSG_CMD_STATUS_UPDATE;
1913         wmsg.phase = PHASE_INSTALL_PKGS;
1914         wmsg.lines = 0;
1915
1916         status = ipcwritemsg(fd, &wmsg);
1917
1918         if (pkgpkg) {
1919                 dophase(work, &wmsg,
1920                         WDOG5, PHASE_PACKAGE, "package");
1921         } else {
1922                 if (do_install_phase) {
1923                         dophase(work, &wmsg,
1924                                 WDOG4, PHASE_INSTALL_PKGS, "setup");
1925                 }
1926                 dophase(work, &wmsg,
1927                         WDOG2, PHASE_CHECK_SANITY, "check-sanity");
1928                 dophase(work, &wmsg,
1929                         WDOG2, PHASE_PKG_DEPENDS, "pkg-depends");
1930                 dophase(work, &wmsg,
1931                         WDOG7, PHASE_FETCH_DEPENDS, "fetch-depends");
1932                 dophase(work, &wmsg,
1933                         WDOG7, PHASE_FETCH, "fetch");
1934                 dophase(work, &wmsg,
1935                         WDOG2, PHASE_CHECKSUM, "checksum");
1936                 dophase(work, &wmsg,
1937                         WDOG3, PHASE_EXTRACT_DEPENDS, "extract-depends");
1938                 dophase(work, &wmsg,
1939                         WDOG3, PHASE_EXTRACT, "extract");
1940                 dophase(work, &wmsg,
1941                         WDOG2, PHASE_PATCH_DEPENDS, "patch-depends");
1942                 dophase(work, &wmsg,
1943                         WDOG2, PHASE_PATCH, "patch");
1944                 dophase(work, &wmsg,
1945                         WDOG5, PHASE_BUILD_DEPENDS, "build-depends");
1946                 dophase(work, &wmsg,
1947                         WDOG5, PHASE_LIB_DEPENDS, "lib-depends");
1948                 dophase(work, &wmsg,
1949                         WDOG3, PHASE_CONFIGURE, "configure");
1950                 dophase(work, &wmsg,
1951                         WDOG9, PHASE_BUILD, "build");
1952                 dophase(work, &wmsg,
1953                         WDOG5, PHASE_RUN_DEPENDS, "run-depends");
1954                 dophase(work, &wmsg,
1955                         WDOG5, PHASE_STAGE, "stage");
1956 #if 0
1957                 dophase(work, &wmsg,
1958                         WDOG5, PHASE_TEST, "test");
1959 #endif
1960                 dophase(work, &wmsg,
1961                         WDOG1, PHASE_CHECK_PLIST, "check-plist");
1962                 dophase(work, &wmsg,
1963                         WDOG5, PHASE_PACKAGE, "package");
1964 #if 0
1965                 dophase(work, &wmsg,
1966                         WDOG5, PHASE_INSTALL_MTREE, "install-mtree");
1967                 dophase(work, &wmsg,
1968                         WDOG5, PHASE_INSTALL, "install");
1969                 dophase(work, &wmsg,
1970                         WDOG5, PHASE_DEINSTALL, "deinstall");
1971 #endif
1972         }
1973
1974         if (MasterPtyFd >= 0) {
1975                 close(MasterPtyFd);
1976                 MasterPtyFd = -1;
1977         }
1978
1979         setproctitle("[%02d] WORKER CLEANUP  %s%s",
1980                      slot, portdir, WorkerFlavorPrt);
1981
1982         /*
1983          * Copy the package to the repo.
1984          */
1985         if (work->accum_error == 0) {
1986                 char *b1;
1987                 char *b2;
1988
1989                 asprintf(&b1, "%s/construction/%s/pkg/%s",
1990                          work->basedir, pkg.portdir, pkg.pkgfile);
1991                 asprintf(&b2, "%s/%s", RepositoryPath, pkg.pkgfile);
1992                 if (copyfile(b1, b2)) {
1993                         ++work->accum_error;
1994                         dlog(DLOG_ALL, "[%03d] %s Unable to copy %s to %s\n",
1995                              work->index, pkg.portdir, b1, b2);
1996                 }
1997                 free(b1);
1998                 free(b2);
1999         }
2000
2001         /*
2002          * Unmount, unless we are in DebugStopMode.
2003          */
2004         if ((WorkerProcFlags & WORKER_PROC_DEBUGSTOP) == 0)
2005                 DoWorkerUnmounts(work);
2006
2007         /*
2008          * Send completion status to master dsynth worker thread.
2009          */
2010         if (work->accum_error) {
2011                 wmsg.cmd = WMSG_CMD_FAILURE;
2012         } else {
2013                 wmsg.cmd = WMSG_CMD_SUCCESS;
2014         }
2015         ipcwritemsg(fd, &wmsg);
2016         if (WorkerProcFlags & WORKER_PROC_DEBUGSTOP) {
2017                 wmsg.cmd = WMSG_CMD_FREEZEWORKER;
2018                 ipcwritemsg(fd, &wmsg);
2019         }
2020         if (UsingHooks) {
2021                 while ((bulk = getbulk()) != NULL)
2022                         freebulk(bulk);
2023                 donebulk();
2024         }
2025 }
2026
2027 static void
2028 dophase(worker_t *work, wmsg_t *wmsg, int wdog, int phaseid, const char *phase)
2029 {
2030         pkg_t *pkg = work->pkg;
2031         char buf[1024];
2032         pid_t pid;
2033         int status;
2034         int ms;
2035         pid_t wpid;
2036         int wpid_reaped;
2037         int fdlog;
2038         time_t start_time;
2039         time_t last_time;
2040         time_t next_time;
2041         time_t wdog_time;
2042         FILE *fp;
2043
2044         if (work->accum_error)
2045                 return;
2046         setproctitle("[%02d] WORKER %-8.8s %s%s",
2047                      work->index, phase, pkg->portdir, WorkerFlavorPrt);
2048         wmsg->phase = phaseid;
2049         if (ipcwritemsg(work->fds[0], wmsg) < 0) {
2050                 dlog(DLOG_ALL, "[%03d] %s Lost Communication with dsynth, "
2051                      "aborting worker\n",
2052                      work->index, pkg->portdir);
2053                 ++work->accum_error;
2054                 return;
2055         }
2056
2057         /*
2058          * Execute the port make command in chroot on a pty.
2059          */
2060         fflush(stdout);
2061         fflush(stderr);
2062         if (MasterPtyFd >= 0) {
2063                 int slavefd;
2064                 char ttybuf[2];
2065
2066                 /*
2067                  * NOTE: We can't open the slave in the child because the
2068                  *       master may race a disconnection test.  If we open
2069                  *       it in the parent our close() will flush any pending
2070                  *       output not read by the master (which is the same
2071                  *       parent process) and deadlock.
2072                  *
2073                  *       Solve this by hand-shaking the slave tty to give
2074                  *       the master time to close its slavefd.
2075                  *
2076                  *       Leave the tty defaults intact, which also likely
2077                  *       means it will be in line-buffered mode, so handshake
2078                  *       with a full line.
2079                  */
2080                 slavefd = open(ptsname(MasterPtyFd), O_RDWR);
2081                 dassert_errno(slavefd >= 0, "Cannot open slave pty");
2082                 pid = fork();
2083                 if (pid == 0) {
2084                         login_tty(slavefd);
2085                         /* login_tty() closes slavefd */
2086                         read(0, ttybuf, 2);
2087                 } else {
2088                         close(slavefd);
2089                         write(MasterPtyFd, "x\n", 2);
2090                 }
2091         } else {
2092                 pid = forkpty(&MasterPtyFd, NULL, NULL, NULL);
2093         }
2094
2095         if (pid == 0) {
2096                 struct termios tio;
2097
2098                 /*
2099                  * We are going through a pty, so set the tty modes to
2100                  * Set tty modes so we do not get ^M's in the log files.
2101                  *
2102                  * This isn't fatal if it doesn't work.  Remember that
2103                  * our output goes through the pty to the management
2104                  * process which will log it.
2105                  */
2106                 if (tcgetattr(1, &tio) == 0) {
2107                         tio.c_oflag |= OPOST | ONOCR;
2108                         tio.c_oflag &= ~(OCRNL | ONLCR);
2109                         tio.c_iflag |= ICRNL;
2110                         tio.c_iflag &= ~(INLCR|IGNCR);
2111                         if (tcsetattr(1, TCSANOW, &tio)) {
2112                                 printf("tcsetattr failed: %s\n",
2113                                        strerror(errno));
2114                         }
2115                 } else {
2116                         printf("tcgetattr failed: %s\n", strerror(errno));
2117                 }
2118
2119                 /*
2120                  * Clean-up, chdir, and chroot.
2121                  */
2122                 closefrom(3);
2123                 if (chdir(work->basedir) < 0)
2124                         dfatal_errno("chdir in phase initialization");
2125                 if (chroot(work->basedir) < 0)
2126                         dfatal_errno("chroot in phase initialization");
2127
2128                 /*
2129                  * We have a choice here on how to handle stdin (fd 0).
2130                  * We can leave it connected to the pty in which case
2131                  * the build will just block if it tries to ask a
2132                  * question (and the watchdog will kill it, eventually),
2133                  * or we can try to EOF the pty, or we can attach /dev/null
2134                  * to descriptor 0.
2135                  */
2136                 if (NullStdinOpt) {
2137                         int fd;
2138
2139                         fd = open("/dev/null", O_RDWR);
2140                         dassert_errno(fd >= 0, "cannot open /dev/null");
2141                         if (fd != 0) {
2142                                 dup2(fd, 0);
2143                                 close(fd);
2144                         }
2145                 }
2146
2147                 /*
2148                  * Execute the appropriate command.
2149                  */
2150                 switch(phaseid) {
2151                 case PHASE_INSTALL_PKGS:
2152                         snprintf(buf, sizeof(buf), "/tmp/dsynth_install_pkgs");
2153                         execl(buf, buf, NULL);
2154                         break;
2155                 default:
2156                         snprintf(buf, sizeof(buf), "/xports/%s", pkg->portdir);
2157                         execl(MAKE_BINARY, MAKE_BINARY, "-C", buf, phase, NULL);
2158                         break;
2159                 }
2160                 _exit(1);
2161         }
2162         fcntl(MasterPtyFd, F_SETFL, O_NONBLOCK);
2163
2164         if (pid < 0) {
2165                 dlog(DLOG_ALL, "[%03d] %s Fork Failed: %s\n",
2166                      work->index, pkg->logfile, strerror(errno));
2167                 ++work->accum_error;
2168                 return;
2169         }
2170
2171         SigPid = pid;
2172
2173         fdlog = open(pkg->logfile, O_RDWR|O_CREAT|O_APPEND, 0644);
2174         if (fdlog < 0) {
2175                 dlog(DLOG_ALL, "[%03d] %s Cannot open logfile '%s': %s\n",
2176                      work->index, pkg->portdir,
2177                      pkg->logfile, strerror(errno));
2178         }
2179
2180         snprintf(buf, sizeof(buf),
2181                  "----------------------------------------"
2182                  "---------------------------------------\n"
2183                  "-- Phase: %s\n"
2184                  "----------------------------------------"
2185                  "---------------------------------------\n",
2186                  phase);
2187         write(fdlog, buf, strlen(buf));
2188
2189         start_time = time(NULL);
2190         last_time = start_time;
2191         wdog_time = start_time;
2192         wpid_reaped = 0;
2193
2194         status = 0;
2195         for (;;) {
2196                 ms = mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time);
2197                 if (ms == MPTY_FAILED) {
2198                         dlog(DLOG_ALL,
2199                              "[%03d] %s lost pty in phase %s, terminating\n",
2200                              work->index, pkg->portdir, phase);
2201                         break;
2202                 }
2203                 if (ms == MPTY_EOF)
2204                         break;
2205
2206                 /*
2207                  * Generally speaking update status once a second.
2208                  * This also allows us to detect if the management
2209                  * dsynth process has gone away.
2210                  */
2211                 next_time = time(NULL);
2212                 if (next_time != last_time) {
2213                         double dload[3];
2214                         double dv;
2215                         int wdog_scaled;
2216
2217                         /*
2218                          * Send status update to the worker management thread
2219                          * in the master dsynth process.  Remember, *WE* are
2220                          * the worker management process sub-fork.
2221                          */
2222                         if (ipcwritemsg(work->fds[0], wmsg) < 0)
2223                                 break;
2224                         last_time = next_time;
2225
2226                         /*
2227                          * Watchdog scaling
2228                          */
2229                         getloadavg(dload, 3);
2230                         dv = dload[2] / NumCores;
2231                         if (dv < (double)NumCores) {
2232                                 wdog_scaled = wdog;
2233                         } else {
2234                                 if (dv > 4.0 * NumCores)
2235                                         dv = 4.0 * NumCores;
2236                                 wdog_scaled = wdog * dv / NumCores;
2237                         }
2238
2239                         /*
2240                          * Watchdog
2241                          */
2242                         if (next_time - wdog_time >= wdog_scaled * 60) {
2243                                 snprintf(buf, sizeof(buf),
2244                                          "\n--------\n"
2245                                          "WATCHDOG TIMEOUT FOR %s in %s "
2246                                          "after %d minutes\n"
2247                                          "Killing pid %d\n"
2248                                          "--------\n",
2249                                          pkg->portdir, phase, wdog_scaled, pid);
2250                                 if (fdlog >= 0)
2251                                         write(fdlog, buf, strlen(buf));
2252                                 dlog(DLOG_ALL,
2253                                      "[%03d] %s WATCHDOG TIMEOUT in %s "
2254                                      "after %d minutes (%d min scaled)\n",
2255                                      work->index, pkg->portdir, phase,
2256                                      wdog, wdog_scaled);
2257                                 kill(pid, SIGKILL);
2258                                 ++work->accum_error;
2259                                 break;
2260                         }
2261                 }
2262
2263                 /*
2264                  * Check process exit.  Normally the pty will EOF
2265                  * but if background processes remain we need to
2266                  * check here to see if our primary exec is done,
2267                  * so we can break out and reap those processes.
2268                  *
2269                  * Generally reap any other processes we have inherited
2270                  * while we are here.
2271                  */
2272                 do {
2273                         wpid = wait3(&status, WNOHANG, NULL);
2274                 } while (wpid > 0 && wpid != pid);
2275                 if (wpid == pid && WIFEXITED(status)) {
2276                         wpid_reaped = 1;
2277                         break;
2278                 }
2279         }
2280
2281         next_time = time(NULL);
2282
2283         setproctitle("[%02d] WORKER EXITREAP %s%s",
2284                      work->index, pkg->portdir, WorkerFlavorPrt);
2285
2286         /*
2287          * We usually get here due to a mpty EOF, but not always as there
2288          * could be persistent processes still holding the slave.  Finish
2289          * up getting the exit status for the main process we are waiting
2290          * on and clean out any data left on the MasterPtyFd (as it could
2291          * be blocking the exit).
2292          */
2293         while (wpid_reaped == 0) {
2294                 (void)mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time);
2295                 wpid = waitpid(pid, &status, WNOHANG);
2296                 if (wpid == pid && WIFEXITED(status)) {
2297                         wpid_reaped = 1;
2298                         break;
2299                 }
2300                 if (wpid < 0 && errno != EINTR) {
2301                         break;
2302                 }
2303
2304                 /*
2305                  * Safety.  The normal phase waits until the fork/exec'd
2306                  * pid finishes, causing a pty EOF on exit (the slave
2307                  * descriptor is closed by the kernel on exit so the
2308                  * process should already have exited).
2309                  *
2310                  * However, it is also possible to get here if the pty fails
2311                  * for some reason.  In this case, make sure that the process
2312                  * is killed.
2313                  */
2314                 kill(pid, SIGKILL);
2315         }
2316
2317         /*
2318          * Clean out anything left on the pty but don't wait around
2319          * because there could be background processes preventing the
2320          * slave side from closing.
2321          */
2322         while (mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time) == MPTY_DATA)
2323                 ;
2324
2325         /*
2326          * Report on the exit condition.  If the pid was somehow lost
2327          * (probably due to someone gdb'ing the process), assume an error.
2328          */
2329         if (wpid_reaped) {
2330                 if (WEXITSTATUS(status)) {
2331                         dlog(DLOG_ALL | DLOG_FILTER,
2332                              "[%03d] %s Build phase '%s' failed exit %d\n",
2333                              work->index, pkg->portdir, phase,
2334                              WEXITSTATUS(status));
2335                         ++work->accum_error;
2336                 }
2337         } else {
2338                 dlog(DLOG_ALL, "[%03d] %s Build phase '%s' failed - lost pid\n",
2339                      work->index, pkg->portdir, phase);
2340                 ++work->accum_error;
2341         }
2342
2343         /*
2344          * Kill any processes still running (sometimes processes end up in
2345          * the background during a dports build), and clean up any other
2346          * children that we have inherited.
2347          */
2348         phaseReapAll();
2349
2350         /*
2351          * After the extraction phase add the space used by /construction
2352          * to the memory use.  This helps us reduce the amount of paging
2353          * we do due to extremely large package extractions (languages,
2354          * chromium, etc).
2355          *
2356          * (dsynth already estimated the space used by the package deps
2357          * up front, but this will help us further).
2358          */
2359         if (work->accum_error == 0 && phaseid == PHASE_EXTRACT) {
2360                 struct statfs sfs;
2361                 char *b1;
2362
2363                 asprintf(&b1, "%s/construction", work->basedir);
2364                 if (statfs(b1, &sfs) == 0) {
2365                         wmsg->memuse = (sfs.f_blocks - sfs.f_bfree) *
2366                                        sfs.f_bsize;
2367                         ipcwritemsg(work->fds[0], wmsg);
2368                 }
2369         }
2370
2371         /*
2372          * Update log
2373          */
2374         if (fdlog >= 0) {
2375                 struct stat st;
2376                 int h;
2377                 int m;
2378                 int s;
2379
2380                 last_time = next_time - start_time;
2381                 s = last_time % 60;
2382                 m = last_time / 60 % 60;
2383                 h = last_time / 3600;
2384
2385                 fp = fdopen(fdlog, "a");
2386                 if (fp == NULL) {
2387                         dlog(DLOG_ALL, "[%03d] %s Cannot fdopen fdlog: %s %d\n",
2388                              work->index, pkg->portdir,
2389                              strerror(errno), fstat(fdlog, &st));
2390                         close(fdlog);
2391                         goto skip;
2392                 }
2393
2394                 fprintf(fp, "\n");
2395                 if (work->accum_error) {
2396                         fprintf(fp, "FAILED %02d:%02d:%02d\n", h, m, s);
2397                 } else {
2398                         if (phaseid == PHASE_EXTRACT && wmsg->memuse) {
2399                                 fprintf(fp, "Extracted Memory Use: %6.2fM\n",
2400                                         wmsg->memuse / (1024.0 * 1024.0));
2401                         }
2402                         fprintf(fp, "SUCCEEDED %02d:%02d:%02d\n", h, m, s);
2403                 }
2404                 last_time = next_time - work->start_time;
2405                 s = last_time % 60;
2406                 m = last_time / 60 % 60;
2407                 h = last_time / 3600;
2408                 if (phaseid == PHASE_PACKAGE) {
2409                         fprintf(fp, "TOTAL TIME %02d:%02d:%02d\n", h, m, s);
2410                 }
2411                 fprintf(fp, "\n");
2412                 fclose(fp);
2413 skip:
2414                 ;
2415         }
2416
2417 }
2418
2419 static void
2420 phaseReapAll(void)
2421 {
2422         struct reaper_status rs;
2423         int status;
2424
2425         while (procctl(P_PID, getpid(), PROC_REAP_STATUS, &rs) == 0) {
2426                 if ((rs.flags & PROC_REAP_ACQUIRE) == 0)
2427                         break;
2428                 if (rs.pid_head < 0)
2429                         break;
2430                 if (kill(rs.pid_head, SIGKILL) == 0) {
2431                         while (waitpid(rs.pid_head, &status, 0) < 0)
2432                                 ;
2433                 }
2434         }
2435         while (wait3(&status, 0, NULL) > 0)
2436                 ;
2437 }
2438
2439 static void
2440 phaseTerminateSignal(int sig __unused)
2441 {
2442         if (CopyFileFd >= 0)
2443                 close(CopyFileFd);
2444         if (MasterPtyFd >= 0)
2445                 close(MasterPtyFd);
2446         if (SigPid > 1)
2447                 kill(SigPid, SIGKILL);
2448         phaseReapAll();
2449         if (SigWork)
2450                 DoWorkerUnmounts(SigWork);
2451         exit(1);
2452 }
2453
2454 static
2455 char *
2456 buildskipreason(pkglink_t *parent, pkg_t *pkg)
2457 {
2458         pkglink_t *link;
2459         pkg_t *scan;
2460         char *reason = NULL;
2461         char *ptr;
2462         size_t tot;
2463         size_t len;
2464         pkglink_t stack;
2465
2466         if ((pkg->flags & PKGF_NOBUILD_I) && pkg->ignore)
2467                 asprintf(&reason, "%s ", pkg->ignore);
2468
2469         tot = 0;
2470         PKGLIST_FOREACH(link, &pkg->idepon_list) {
2471 #if 0
2472                 if (link->dep_type > DEP_TYPE_BUILD)
2473                         continue;
2474 #endif
2475                 scan = link->pkg;
2476                 if (scan == NULL)
2477                         continue;
2478                 if ((scan->flags & (PKGF_ERROR | PKGF_NOBUILD)) == 0)
2479                         continue;
2480                 if (scan->flags & PKGF_NOBUILD) {
2481                         stack.pkg = scan;
2482                         stack.next = parent;
2483                         ptr = buildskipreason(&stack, scan);
2484                         len = strlen(scan->portdir) + strlen(ptr) + 8;
2485                         reason = realloc(reason, tot + len);
2486                         snprintf(reason + tot, len, "%s->%s",
2487                                  scan->portdir, ptr);
2488                         free(ptr);
2489                 } else {
2490                         len = strlen(scan->portdir) + 8;
2491                         reason = realloc(reason, tot + len);
2492                         snprintf(reason + tot, len, "%s", scan->portdir);
2493                 }
2494
2495                 /*
2496                  * Don't try to print the entire graph
2497                  */
2498                 if (parent)
2499                         break;
2500                 tot += strlen(reason + tot);
2501                 reason[tot++] = ' ';
2502                 reason[tot] = 0;
2503         }
2504         return (reason);
2505 }
2506
2507 /*
2508  * The master ptyfd is in non-blocking mode.  Drain up to 1024 bytes
2509  * and update wmsg->lines and *wdog_timep as appropriate.
2510  *
2511  * This function will poll, stalling up to 1 second.
2512  */
2513 static int
2514 mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg, time_t *wdog_timep)
2515 {
2516         struct pollfd pfd;
2517         char buf[1024];
2518         ssize_t r;
2519
2520         pfd.fd = ptyfd;
2521         pfd.events = POLLIN;
2522         pfd.revents = 0;
2523
2524         poll(&pfd, 1, 1000);
2525         if (pfd.revents) {
2526                 r = read(ptyfd, buf, sizeof(buf));
2527                 if (r > 0) {
2528                         *wdog_timep = time(NULL);
2529                         if (r > 0 && fdlog >= 0)
2530                                 write(fdlog, buf, r);
2531                         while (--r >= 0) {
2532                                 if (buf[r] == '\n')
2533                                         ++wmsg->lines;
2534                         }
2535                         return MPTY_DATA;
2536                         return 1;
2537                 } else if (r < 0) {
2538                         if (errno != EINTR && errno != EAGAIN)
2539                                 return MPTY_FAILED;
2540                         return MPTY_AGAIN;
2541                 } else if (r == 0) {
2542                         return MPTY_EOF;
2543                 }
2544         }
2545         return MPTY_AGAIN;
2546 }
2547
2548 /*
2549  * Copy a (package) file from (src) to (dst), use an intermediate file and
2550  * rename to ensure that interruption does not leave us with a corrupt
2551  * package file.
2552  *
2553  * This is called by the WORKER process.
2554  *
2555  * (dsynth management thread -> WORKER process -> sub-processes)
2556  */
2557 #define COPYBLKSIZE     32768
2558
2559 static int
2560 copyfile(char *src, char *dst)
2561 {
2562         char *tmp;
2563         char *buf;
2564         int fd1;
2565         int fd2;
2566         int error = 0;
2567         int mask;
2568         ssize_t r;
2569
2570         asprintf(&tmp, "%s.new", dst);
2571         buf = malloc(COPYBLKSIZE);
2572
2573         mask = sigsetmask(sigmask(SIGTERM)|sigmask(SIGINT)|sigmask(SIGHUP));
2574         fd1 = open(src, O_RDONLY|O_CLOEXEC);
2575         fd2 = open(tmp, O_RDWR|O_CREAT|O_TRUNC|O_CLOEXEC, 0644);
2576         CopyFileFd = fd1;
2577         sigsetmask(mask);
2578         while ((r = read(fd1, buf, COPYBLKSIZE)) > 0) {
2579                 if (write(fd2, buf, r) != r)
2580                         error = 1;
2581         }
2582         if (r < 0)
2583                 error = 1;
2584         mask = sigsetmask(sigmask(SIGTERM)|sigmask(SIGINT)|sigmask(SIGHUP));
2585         CopyFileFd = -1;
2586         close(fd1);
2587         close(fd2);
2588         sigsetmask(mask);
2589         if (error) {
2590                 remove(tmp);
2591         } else {
2592                 if (rename(tmp, dst)) {
2593                         error = 1;
2594                         remove(tmp);
2595                 }
2596         }
2597
2598         freestrp(&buf);
2599         freestrp(&tmp);
2600
2601         return error;
2602 }
2603
2604 /*
2605  * doHook()
2606  *
2607  * primary process (threaded) - run_start, run_end, pkg_ignored, pkg_skipped
2608  * worker process  (threaded) - pkg_sucess, pkg_failure
2609  *
2610  * If waitfor is non-zero this hook will be serialized.
2611  */
2612 static void
2613 doHook(pkg_t *pkg, const char *id, const char *path, int waitfor)
2614 {
2615         if (path == NULL)
2616                 return;
2617         while (waitfor && getbulk() != NULL)
2618                 ;
2619         if (pkg)
2620                 queuebulk(pkg->portdir, id, path, pkg->pkgfile);
2621         else
2622                 queuebulk(NULL, id, path, NULL);
2623         while (waitfor && getbulk() != NULL)
2624                 ;
2625 }
2626
2627 /*
2628  * Execute hook (backend)
2629  *
2630  * s1 - portdir
2631  * s2 - id
2632  * s3 - script path
2633  * s4 - pkgfile         (if applicable)
2634  */
2635 static void
2636 childHookRun(bulk_t *bulk)
2637 {
2638         const char *cav[MAXCAC];
2639         buildenv_t benv[MAXCAC];
2640         char buf1[128];
2641         char buf2[128];
2642         char buf3[128];
2643         char buf4[128];
2644         FILE *fp;
2645         char *ptr;
2646         size_t len;
2647         pid_t pid;
2648         int cac;
2649         int bi;
2650
2651         cac = 0;
2652         bi = 0;
2653         bzero(benv, sizeof(benv));
2654
2655         cav[cac++] = bulk->s3;
2656
2657         benv[bi].label = "PROFILE";
2658         benv[bi].data = Profile;
2659         ++bi;
2660
2661         benv[bi].label = "DIR_PACKAGES";
2662         benv[bi].data = PackagesPath;
2663         ++bi;
2664
2665         benv[bi].label = "DIR_REPOSITORY";
2666         benv[bi].data = RepositoryPath;
2667         ++bi;
2668
2669         benv[bi].label = "DIR_PORTS";
2670         benv[bi].data = DPortsPath;
2671         ++bi;
2672
2673         benv[bi].label = "DIR_OPTIONS";
2674         benv[bi].data = OptionsPath;
2675         ++bi;
2676
2677         benv[bi].label = "DIR_DISTFILES";
2678         benv[bi].data = DistFilesPath;
2679         ++bi;
2680
2681         benv[bi].label = "DIR_LOGS";
2682         benv[bi].data = LogsPath;
2683         ++bi;
2684
2685         benv[bi].label = "DIR_BUILDBASE";
2686         benv[bi].data = BuildBase;
2687         ++bi;
2688
2689         if (strcmp(bulk->s2, "hook_run_start") == 0) {
2690                 snprintf(buf1, sizeof(buf1), "%d", BuildTotal);
2691                 benv[bi].label = "PORTS_QUEUED";
2692                 benv[bi].data = buf1;
2693                 ++bi;
2694         } else if (strcmp(bulk->s2, "hook_run_end") == 0) {
2695                 snprintf(buf1, sizeof(buf1), "%d", BuildSuccessCount);
2696                 benv[bi].label = "PORTS_BUILT";
2697                 benv[bi].data = buf1;
2698                 ++bi;
2699                 snprintf(buf2, sizeof(buf2), "%d", BuildFailCount);
2700                 benv[bi].label = "PORTS_FAILED";
2701                 benv[bi].data = buf2;
2702                 ++bi;
2703                 snprintf(buf3, sizeof(buf3), "%d", BuildIgnoreCount);
2704                 benv[bi].label = "PORTS_IGNORED";
2705                 benv[bi].data = buf3;
2706                 ++bi;
2707                 snprintf(buf4, sizeof(buf4), "%d", BuildSkipCount);
2708                 benv[bi].label = "PORTS_SKIPPED";
2709                 benv[bi].data = buf4;
2710                 ++bi;
2711         } else {
2712                 /*
2713                  * success, failure, ignored, skipped
2714                  */
2715                 benv[bi].label = "RESULT";
2716                 if (strcmp(bulk->s2, "hook_pkg_success") == 0) {
2717                         benv[bi].data = "success";
2718                 } else if (strcmp(bulk->s2, "hook_pkg_failure") == 0) {
2719                         benv[bi].data = "failure";
2720                 } else if (strcmp(bulk->s2, "hook_pkg_ignored") == 0) {
2721                         benv[bi].data = "ignored";
2722                 } else if (strcmp(bulk->s2, "hook_pkg_skipped") == 0) {
2723                         benv[bi].data = "skipped";
2724                 } else {
2725                         dfatal("Unknown hook id: %s", bulk->s2);
2726                         /* NOT REACHED */
2727                 }
2728                 ++bi;
2729
2730                 /*
2731                  * For compatibility with synth:
2732                  *
2733                  * ORIGIN does not include any @flavor, thus it is suitable
2734                  * for finding the actual port directory/subdirectory.
2735                  *
2736                  * FLAVOR is set to ORIGIN if there is no flavor, otherwise
2737                  * it is set to only the flavor sans the '@'.
2738                  */
2739                 if ((ptr = strchr(bulk->s1, '@')) != NULL) {
2740                         snprintf(buf1, sizeof(buf1), "%*.*s",
2741                                  (int)(ptr - bulk->s1),
2742                                  (int)(ptr - bulk->s1),
2743                                  bulk->s1);
2744                         benv[bi].label = "ORIGIN";
2745                         benv[bi].data = buf1;
2746                         ++bi;
2747                         benv[bi].label = "FLAVOR";
2748                         benv[bi].data = ptr + 1;
2749                         ++bi;
2750                 } else {
2751                         benv[bi].label = "ORIGIN";
2752                         benv[bi].data = bulk->s1;
2753                         ++bi;
2754                         benv[bi].label = "FLAVOR";
2755                         benv[bi].data = bulk->s1;
2756                         ++bi;
2757                 }
2758                 benv[bi].label = "PKGNAME";
2759                 benv[bi].data = bulk->s4;
2760                 ++bi;
2761         }
2762
2763         benv[bi].label = NULL;
2764         benv[bi].data = NULL;
2765
2766         fp = dexec_open(cav, cac, &pid, benv, 0, 0);
2767         while ((ptr = fgetln(fp, &len)) != NULL)
2768                 ;
2769
2770         if (dexec_close(fp, pid)) {
2771                 dlog(DLOG_ALL,
2772                      "[XXX] %s SCRIPT %s (%s)\n",
2773                      bulk->s1, bulk->s2, bulk->s3);
2774         }
2775 }