2 * Copyright (c) 2019 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * This code uses concepts and configuration based on 'synth', by
8 * John R. Marino <draco@marino.st>, which was written in ada.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in
18 * the documentation and/or other materials provided with the
20 * 3. Neither the name of The DragonFly Project nor the names of its
21 * contributors may be used to endorse or promote products derived
22 * from this software without specific, prior written permission.
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39 worker_t WorkerAry[MAXWORKERS];
42 int DynamicMaxWorkers;
44 long RunningPkgDepSize;
45 pthread_mutex_t WorkerMutex;
46 pthread_cond_t WorkerCond;
48 static int build_find_leaves(pkg_t *parent, pkg_t *pkg,
49 pkg_t ***build_tailp, int *app, int *hasworkp,
50 int depth, int first, int first_one_only);
51 static int buildCalculateDepiDepth(pkg_t *pkg);
52 static void build_clear_trav(pkg_t *pkg);
53 static void startbuild(pkg_t **build_listp, pkg_t ***build_tailp);
54 static int qsort_depi(const void *pkg1, const void *pkg2);
55 static int qsort_idep(const void *pkg1, const void *pkg2);
56 static void startworker(pkg_t *pkg, worker_t *work);
57 static void cleanworker(worker_t *work);
58 static void waitbuild(int whilematch, int dynamicmax);
59 static void workercomplete(worker_t *work);
60 static void *childBuilderThread(void *arg);
61 static int childInstallPkgDeps(worker_t *work);
62 static size_t childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list,
63 int undoit, int depth, int first_one_only);
64 static void dophase(worker_t *work, wmsg_t *wmsg,
65 int wdog, int phaseid, const char *phase);
66 static void phaseReapAll(void);
67 static void phaseTerminateSignal(int sig);
68 static char *buildskipreason(pkglink_t *parent, pkg_t *pkg);
69 static int mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg,
71 static int copyfile(char *src, char *dst);
72 static void doHook(pkg_t *pkg, const char *id, const char *path, int waitfor);
73 static void childHookRun(bulk_t *bulk);
75 static worker_t *SigWork;
76 static int MasterPtyFd = -1;
77 static int CopyFileFd = -1;
79 static const char *WorkerFlavorPrt = ""; /* "" or "@flavor" */
81 #define MPTY_FAILED -2
91 int BuildSuccessCount;
94 * Initialize the WorkerAry[]
97 DoInitBuild(int slot_override)
103 ddassert(slot_override < 0 || MaxWorkers == 1);
105 bzero(WorkerAry, MaxWorkers * sizeof(worker_t));
106 pthread_mutex_init(&WorkerMutex, NULL);
108 for (i = 0; i < MaxWorkers; ++i) {
109 work = &WorkerAry[i];
110 work->index = (slot_override >= 0) ? slot_override : i;
111 work->state = WORKER_NONE;
112 asprintf(&work->basedir, "%s/SL%02d", BuildBase, work->index);
113 pthread_cond_init(&work->cond, NULL);
118 * Create required sub-directories. The base directories must already
119 * exist as a dsynth configuration safety.
121 if (stat(RepositoryPath, &st) < 0) {
122 if (mkdir(RepositoryPath, 0755) < 0)
123 dfatal("Cannot mkdir %s\n", RepositoryPath);
126 BuildInitialized = 1;
129 * slow-start (increases at a rate of 1 per 5 seconds)
131 if (SlowStartOpt > MaxWorkers)
132 DynamicMaxWorkers = MaxWorkers;
133 else if (SlowStartOpt > 0)
134 DynamicMaxWorkers = SlowStartOpt;
136 DynamicMaxWorkers = MaxWorkers;
140 * Called by the frontend to clean-up any hanging mounts.
143 DoCleanBuild(int resetlogs)
147 ddassert(BuildInitialized);
151 for (i = 0; i < MaxWorkers; ++i) {
152 DoWorkerUnmounts(&WorkerAry[i]);
159 pkg_t *build_list = NULL;
160 pkg_t **build_tail = &build_list;
171 * We use our bulk system to run hooks. This will be for
172 * Skipped and Ignored. Success and Failure are handled by
173 * WorkerProcess() which is a separately-exec'd dsynth.
176 initbulk(childHookRun, MaxBulk);
179 * Count up the packages, not counting dummies
181 for (scan = pkgs; scan; scan = scan->bnext) {
182 if ((scan->flags & PKGF_DUMMY) == 0)
186 doHook(NULL, "hook_run_start", HookRunStart, 1);
189 * The pkg and pkg-static binaries are needed. If already present
190 * then assume that the template is also valid, otherwise build
193 scan = GetPkgPkg(pkgs);
196 * Create our template. The template will be missing pkg
199 if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0) {
200 /* force a fresh template */
201 newtemplate = DoCreateTemplate(1);
203 newtemplate = DoCreateTemplate(0);
207 * This will clear the screen and set-up our gui, so sleep
208 * a little first in case the user wants to see what was
212 pthread_mutex_lock(&WorkerMutex);
213 startTime = time(NULL);
218 * Build pkg/pkg-static.
220 if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0) {
222 build_tail = &scan->build_next;
223 startbuild(&build_list, &build_tail);
224 while (RunningWorkers == 1)
227 if (scan->flags & PKGF_NOBUILD)
228 dfatal("Unable to build 'pkg'");
229 if (scan->flags & PKGF_ERROR)
230 dfatal("Error building 'pkg'");
231 if ((scan->flags & PKGF_SUCCESS) == 0)
232 dfatal("Error building 'pkg'");
237 * Install pkg/pkg-static into the template
245 "tar --exclude '+*' --exclude '*/man/*' "
246 "-xvzpf %s/%s > /dev/null 2>&1",
252 dfatal("Command failed: %s\n", buf);
257 * Calculate depi_depth, the longest chain of dependencies
258 * for who depends on me, weighted by powers of two.
260 for (scan = pkgs; scan; scan = scan->bnext) {
261 buildCalculateDepiDepth(scan);
265 * Nominal bulk build sequence
270 for (scan = pkgs; scan; scan = scan->bnext) {
271 ddprintf(0, "SCANLEAVES %08x %s\n",
272 scan->flags, scan->portdir);
273 scan->flags |= PKGF_BUILDLOOP;
275 * NOTE: We must still find dependencies if PACKAGED
276 * to fill in the gaps, as some of them may
277 * need to be rebuilt.
279 if (scan->flags & (PKGF_SUCCESS | PKGF_FAILURE |
282 ddprintf(0, "%s: already built\n",
287 build_find_leaves(NULL, scan, &build_tail,
288 &ap, &haswork, 0, first, 0);
289 ddprintf(0, "TOPLEVEL %s %08x\n",
292 scan->flags &= ~PKGF_BUILDLOOP;
293 build_clear_trav(scan);
297 startbuild(&build_list, &build_tail);
299 if (haswork == 0 && RunningWorkers) {
300 waitbuild(RunningWorkers, 1);
304 pthread_mutex_unlock(&WorkerMutex);
307 RunStatsUpdateLogs();
311 doHook(NULL, "hook_run_end", HookRunEnd, 1);
313 while ((bulk = getbulk()) != NULL)
318 t = time(NULL) - startTime;
324 printf("Initial queue size: %d\n", BuildTotal);
325 printf(" packages built: %d\n", BuildSuccessCount);
326 printf(" ignored: %d\n", BuildIgnoreCount);
327 printf(" skipped: %d\n", BuildSkipCount);
328 printf(" failed: %d\n", BuildFailCount);
330 printf("Duration: %02d:%02d:%02d\n", h, m, s);
335 * Traverse the packages (pkg) depends on recursively until we find
336 * a leaf to build or report as unbuildable. Calculates and assigns a
337 * dependency count. Returns all parallel-buildable packages.
339 * (pkg) itself is only added to the list if it is immediately buildable.
343 build_find_leaves(pkg_t *parent, pkg_t *pkg, pkg_t ***build_tailp,
344 int *app, int *hasworkp, int depth, int first,
358 * Already on build list, possibly in-progress, tell caller that
361 ddprintf(ndepth, "sbuild_find_leaves %d %s %08x {\n",
362 depth, pkg->portdir, pkg->flags);
363 if (pkg->flags & PKGF_BUILDLIST) {
364 ddprintf(ndepth, "} (already on build list)\n");
365 *app |= PKGF_NOTREADY;
366 return (pkg->idep_count);
372 PKGLIST_FOREACH(link, &pkg->idepon_list) {
380 ddprintf(ndepth, "check %s %08x\t", scan->portdir, scan->flags);
383 * If this dependency is to a DUMMY node it is a dependency
384 * only on the default flavor which is only the first node
385 * under this one, not all of them.
387 * NOTE: The depth is not being for complex dependency type
388 * tests like it is in childInstallPkgDeps_recurse(),
389 * so we don't have to hicup it like we do in that
392 dfirst_one_only = (scan->flags & PKGF_DUMMY) ? 1 : 0;
395 * When accounting for a successful build, just bump
396 * idep_count by one. scan->idep_count will heavily
397 * overlap packages that we count down multiple branches.
399 * We must still recurse through PACKAGED packages as
400 * some of their dependencies might be missing.
402 if (scan->flags & PKGF_SUCCESS) {
403 ddprintf(0, "SUCCESS - OK\n");
411 * ERROR includes FAILURE, which is set in numerous situations
412 * including when NOBUILD state is processed. So check for
413 * NOBUILD state first.
415 * An ERROR in a sub-package causes a NOBUILD in packages
418 if (scan->flags & PKGF_NOBUILD) {
419 ddprintf(0, "NOBUILD - OK "
420 "(propogate failure upward)\n");
421 *app |= PKGF_NOBUILD_S;
426 if (scan->flags & PKGF_ERROR) {
427 ddprintf(0, "ERROR - OK (propogate failure upward)\n");
428 *app |= PKGF_NOBUILD_S;
435 * If already on build-list this dependency is not ready.
437 if (scan->flags & PKGF_BUILDLIST) {
438 ddprintf(0, " [BUILDLIST]");
439 *app |= PKGF_NOTREADY;
443 * If not packaged this dependency is not ready for
446 if ((scan->flags & PKGF_PACKAGED) == 0) {
447 ddprintf(0, " [NOT_PACKAGED]");
448 *app |= PKGF_NOTREADY;
452 * Reduce search complexity, if we have already processed
453 * scan in the traversal it will either already be on the
454 * build list or it will not be buildable. Either way
455 * the parent is not buildable.
457 if (scan->flags & PKGF_BUILDTRAV) {
458 ddprintf(0, " [BUILDTRAV]\n");
459 *app |= PKGF_NOTREADY;
466 * Assert on dependency loop
469 if (scan->flags & PKGF_BUILDLOOP) {
470 dfatal("pkg dependency loop %s -> %s",
471 parent->portdir, scan->portdir);
475 * NOTE: For debug tabbing purposes we use (ndepth + 1)
476 * here (i.e. depth + 2) in our iteration.
478 scan->flags |= PKGF_BUILDLOOP;
480 ddprintf(0, " SUBRECURSION {\n");
481 idep_count += build_find_leaves(pkg, scan, build_tailp,
485 scan->flags &= ~PKGF_BUILDLOOP;
487 if (apsub & PKGF_NOBUILD) {
488 ddprintf(ndepth, "} (sub-nobuild)\n");
489 } else if (apsub & PKGF_ERROR) {
490 ddprintf(ndepth, "} (sub-error)\n");
491 } else if (apsub & PKGF_NOTREADY) {
492 ddprintf(ndepth, "} (sub-notready)\n");
494 ddprintf(ndepth, "} (sub-ok)\n");
499 pkg->idep_count = idep_count;
500 pkg->flags |= PKGF_BUILDTRAV;
503 * Incorporate scan results into pkg state.
505 if ((pkg->flags & PKGF_NOBUILD) == 0 && (*app & PKGF_NOBUILD)) {
507 } else if ((pkg->flags & PKGF_ERROR) == 0 && (*app & PKGF_ERROR)) {
510 pkg->flags |= *app & ~PKGF_NOTREADY;
513 * Clear PACKAGED bit if sub-dependencies aren't clean
515 if ((pkg->flags & PKGF_PACKAGED) &&
516 (pkg->flags & (PKGF_NOTREADY|PKGF_ERROR|PKGF_NOBUILD))) {
517 pkg->flags &= ~PKGF_PACKAGED;
518 ddassert(pkg->pkgfile);
519 asprintf(&buf, "%s/%s", RepositoryPath, pkg->pkgfile);
520 if (remove(buf) < 0) {
522 "[XXX] %s DELETE-PACKAGE %s (failed)\n",
526 "[XXX] %s DELETE-PACKAGE %s "
527 "(due to dependencies)\n",
535 * Set PKGF_NOBUILD_I if there is IGNORE data
538 pkg->flags |= PKGF_NOBUILD_I;
541 * Handle propagated flags
543 if (pkg->flags & PKGF_ERROR) {
545 * This can only happen if the ERROR has already been
546 * processed and accounted for.
548 ddprintf(depth, "} (ERROR - %s)\n", pkg->portdir);
549 } else if (*app & PKGF_NOTREADY) {
551 * We don't set PKGF_NOTREADY in the pkg, it is strictly
552 * a transient flag propagated via build_find_leaves().
554 * Just don't add the package to the list.
556 * NOTE: Even if NOBUILD is set (meaning we could list it
557 * and let startbuild() finish it up as a skip, we
558 * don't process it to the list because we want to
559 * process all the dependencies, so someone doing a
560 * manual build can get more complete information and
561 * does not have to iterate each failed dependency one
565 } else if (pkg->flags & PKGF_SUCCESS) {
566 ddprintf(depth, "} (SUCCESS - %s)\n", pkg->portdir);
567 } else if (pkg->flags & PKGF_DUMMY) {
569 * Just mark dummy packages as successful when all of their
570 * sub-depends (flavors) complete successfully. Note that
571 * dummy packages are not counted in the total, so do not
572 * decrement BuildTotal.
574 ddprintf(depth, "} (DUMMY/META - SUCCESS)\n");
575 pkg->flags |= PKGF_SUCCESS;
578 dlog(DLOG_ALL | DLOG_FILTER,
579 "[XXX] %s META-ALREADY-BUILT\n",
582 dlog(DLOG_SUCC, "[XXX] %s meta-node complete\n",
585 } else if (pkg->flags & PKGF_PACKAGED) {
587 * We can just mark the pkg successful. If this is
588 * the first pass, we count this as an initial pruning
589 * pass and reduce BuildTotal.
591 ddprintf(depth, "} (PACKAGED - SUCCESS)\n");
592 pkg->flags |= PKGF_SUCCESS;
595 dlog(DLOG_ALL | DLOG_FILTER,
596 "[XXX] %s ALREADY-BUILT\n",
602 * All dependencies are successful, queue new work
603 * and indicate not-ready to the parent (since our
604 * package has to be built).
606 * NOTE: The NOBUILD case propagates to here as well
607 * and is ultimately handled by startbuild().
610 if (pkg->flags & PKGF_NOBUILD_I)
611 ddprintf(depth, "} (ADDLIST(IGNORE/BROKEN) - %s)\n",
613 else if (pkg->flags & PKGF_NOBUILD)
614 ddprintf(depth, "} (ADDLIST(NOBUILD) - %s)\n",
617 ddprintf(depth, "} (ADDLIST - %s)\n", pkg->portdir);
618 pkg->flags |= PKGF_BUILDLIST;
620 *build_tailp = &pkg->build_next;
621 *app |= PKGF_NOTREADY;
629 build_clear_trav(pkg_t *pkg)
634 pkg->flags &= ~PKGF_BUILDTRAV;
635 PKGLIST_FOREACH(link, &pkg->idepon_list) {
637 if (scan && (scan->flags & PKGF_BUILDTRAV))
638 build_clear_trav(scan);
643 * Calculate the longest chain of packages that depend on me. The
644 * long the chain, the more important my package is to build earlier
648 buildCalculateDepiDepth(pkg_t *pkg)
656 return(pkg->depi_depth + 1);
657 pkg->flags |= PKGF_BUILDLOOP;
658 PKGLIST_FOREACH(link, &pkg->deponi_list) {
660 if (scan && (scan->flags & PKGF_BUILDLOOP) == 0) {
661 res = buildCalculateDepiDepth(scan);
662 if (best_depth < res)
666 pkg->flags &= ~PKGF_BUILDLOOP;
667 pkg->depi_depth = best_depth;
669 return (best_depth + 1);
673 * Take a list of pkg ready to go, sort it, and assign it to worker
674 * slots. This routine blocks in waitbuild() until it can dispose of
677 * WorkerMutex is held by the caller.
681 startbuild(pkg_t **build_listp, pkg_t ***build_tailp)
692 static int IterateWorker;
697 if (*build_listp == NULL)
704 for (pkg = *build_listp; pkg; pkg = pkg->build_next)
706 idep_ary = calloc(count, sizeof(pkg_t *));
707 depi_ary = calloc(count, sizeof(pkg_t *));
710 for (pkg = *build_listp; pkg; pkg = pkg->build_next) {
711 idep_ary[count] = pkg;
712 depi_ary[count] = pkg;
717 * idep_ary - sorted by #of dependencies this pkg has.
718 * depi_ary - sorted by #of other packages that depend on this pkg.
720 qsort(idep_ary, count, sizeof(pkg_t *), qsort_idep);
721 qsort(depi_ary, count, sizeof(pkg_t *), qsort_depi);
727 * Half the workers build based on the highest depi count,
728 * the other half build based on the highest idep count.
730 * This is an attempt to get projects which many other projects
731 * depend on built first, but to also try to build large projects
732 * (which tend to have a lot of dependencies) earlier rather than
733 * later so the end of the bulk run doesn't inefficiently build
734 * the last few huge projects.
736 * Loop until we manage to assign slots to everyone. We do not
737 * wait for build completion.
739 * This is the point where we handle DUMMY packages (these are
740 * dummy unflavored packages which 'cover' all the flavors for
741 * a package). These are not real packages are marked SUCCESS
742 * at this time because their dependencies (the flavors) have all
745 while (idep_index != count || depi_index != count) {
750 * Find candidate to start sorted by depi or idep.
753 while (idep_index < count) {
754 ipkg = idep_ary[idep_index];
756 (PKGF_SUCCESS | PKGF_FAILURE |
757 PKGF_ERROR | PKGF_RUNNING)) == 0) {
765 while (depi_index < count) {
766 pkgi = depi_ary[depi_index];
768 (PKGF_SUCCESS | PKGF_FAILURE |
769 PKGF_ERROR | PKGF_RUNNING)) == 0) {
777 * ipkg and pkgi must either both be NULL, or both
780 if (ipkg == NULL && pkgi == NULL)
782 ddassert(ipkg && pkgi);
785 * Handle the NOBUILD case right here, there's no point
786 * queueing it anywhere.
788 if (ipkg->flags & PKGF_NOBUILD) {
791 ipkg->flags |= PKGF_FAILURE;
792 ipkg->flags &= ~PKGF_BUILDLIST;
794 reason = buildskipreason(NULL, ipkg);
795 if (ipkg->flags & PKGF_NOBUILD_I) {
797 dlog(DLOG_IGN, "[XXX] %s ignored due to %s\n",
798 ipkg->portdir, reason);
799 doHook(pkg, "hook_pkg_ignored",
803 dlog(DLOG_SKIP, "[XXX] %s skipped due to %s\n",
804 ipkg->portdir, reason);
805 doHook(pkg, "hook_pkg_skipped",
812 if (pkgi->flags & PKGF_NOBUILD) {
815 pkgi->flags |= PKGF_FAILURE;
816 pkgi->flags &= ~PKGF_BUILDLIST;
818 reason = buildskipreason(NULL, pkgi);
819 if (pkgi->flags & PKGF_NOBUILD_I) {
821 dlog(DLOG_IGN, "[XXX] %s ignored due to %s\n",
822 pkgi->portdir, reason);
823 doHook(pkg, "hook_pkg_ignored",
827 dlog(DLOG_SKIP, "[XXX] %s skipped due to %s\n",
828 pkgi->portdir, reason);
829 doHook(pkg, "hook_pkg_skipped",
838 * Block while no slots are available. waitbuild()
839 * will clean out any DONE states.
841 while (RunningWorkers >= DynamicMaxWorkers ||
842 RunningWorkers >= MaxWorkers - FailedWorkers) {
843 waitbuild(RunningWorkers, 1);
847 * Find an available worker slot, there should be at
850 for (i = 0; i < MaxWorkers; ++i) {
851 n = IterateWorker % MaxWorkers;
852 work = &WorkerAry[n];
854 if (work->state == WORKER_DONE ||
855 work->state == WORKER_FAILED) {
856 workercomplete(work);
858 if (work->state == WORKER_NONE ||
859 work->state == WORKER_IDLE) {
860 if (n <= MaxWorkers / 2) {
861 startworker(pkgi, work);
863 startworker(ipkg, work);
865 /*RunStatsUpdate(work);*/
870 ddassert(i != MaxWorkers);
875 * We disposed of the whole list
880 *build_tailp = build_listp;
883 typedef const pkg_t *pkg_tt;
886 qsort_idep(const void *pkg1_arg, const void *pkg2_arg)
888 const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg;
889 const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg;
891 return (pkg2->idep_count - pkg1->idep_count);
895 qsort_depi(const void *pkg1_arg, const void *pkg2_arg)
897 const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg;
898 const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg;
900 return ((pkg2->depi_count * pkg2->depi_depth) -
901 (pkg1->depi_count * pkg1->depi_depth));
905 * Frontend starts a pkg up on a worker
907 * WorkerMutex must be held.
910 startworker(pkg_t *pkg, worker_t *work)
912 switch(work->state) {
914 pthread_create(&work->td, NULL, childBuilderThread, work);
915 work->state = WORKER_IDLE;
919 childInstallPkgDeps_recurse(NULL, &pkg->idepon_list, 0, 1, 0);
920 childInstallPkgDeps_recurse(NULL, &pkg->idepon_list, 1, 1, 0);
921 RunningPkgDepSize += work->pkg_dep_size;
923 dlog(DLOG_ALL, "[%03d] START %s "
924 "##idep=%02d depi=%02d/%02d dep=%-4.2fG\n",
925 work->index, pkg->portdir,
926 pkg->idep_count, pkg->depi_count, pkg->depi_depth,
927 (double)work->pkg_dep_size / (double)ONEGB);
930 pkg->flags |= PKGF_RUNNING;
932 pthread_cond_signal(&work->cond);
934 /*RunStatsUpdate(work);*/
943 dfatal("startworker: [%03d] Unexpected state %d for worker %d",
944 work->index, work->state, work->index);
950 cleanworker(worker_t *work)
952 work->state = WORKER_PENDING;
954 work->accum_error = 0;
955 work->start_time = time(NULL);
959 * Frontend finishes up a completed pkg on a worker.
961 * If the worker is in a FAILED state we clean the pkg out but (for now)
962 * leave it in its failed state so we can debug. At this point
963 * workercomplete() will be called every once in a while on the state
964 * and we have to deal with the NULL pkg.
966 * WorkerMutex must be held.
969 workercomplete(worker_t *work)
978 * Steady state FAILED case.
980 if (work->state == WORKER_FAILED) {
981 if (work->pkg == NULL)
985 t = time(NULL) - work->start_time;
991 * Reduce total dep size
993 RunningPkgDepSize -= work->pkg_dep_size;
994 RunningPkgDepSize -= work->memuse;
995 work->pkg_dep_size = 0;
999 * Process pkg out of the worker
1002 if (pkg->flags & (PKGF_ERROR|PKGF_NOBUILD)) {
1003 pkg->flags |= PKGF_FAILURE;
1006 * This NOBUILD condition XXX can occur if the package is
1007 * not allowed to be built.
1009 if (pkg->flags & PKGF_NOBUILD) {
1012 reason = buildskipreason(NULL, pkg);
1013 if (pkg->flags & PKGF_NOBUILD_I) {
1015 dlog(DLOG_SKIP, "[%03d] IGNORD %s - %s\n",
1016 work->index, pkg->portdir, reason);
1017 doHook(pkg, "hook_pkg_ignored",
1021 dlog(DLOG_SKIP, "[%03d] SKIPPD %s - %s\n",
1022 work->index, pkg->portdir, reason);
1023 doHook(pkg, "hook_pkg_skipped",
1029 dlog(DLOG_FAIL | DLOG_RED,
1030 "[%03d] FAILURE %s ##%16.16s %02d:%02d:%02d\n",
1031 work->index, pkg->portdir,
1032 getphasestr(work->phase),
1034 doHook(pkg, "hook_pkg_failure", HookPkgFailure, 0);
1037 pkg->flags |= PKGF_SUCCESS;
1038 ++BuildSuccessCount;
1039 dlog(DLOG_SUCC | DLOG_GRN,
1040 "[%03d] SUCCESS %s ##%02d:%02d:%02d\n",
1041 work->index, pkg->portdir, h, m, s);
1042 doHook(pkg, "hook_pkg_success", HookPkgSuccess, 0);
1045 pkg->flags &= ~PKGF_BUILDLIST;
1046 pkg->flags &= ~PKGF_RUNNING;
1050 if (work->state == WORKER_FAILED) {
1051 dlog(DLOG_ALL, "[%03d] XXX/XXX WORKER IS IN A FAILED STATE\n",
1054 } else if (work->flags & WORKERF_FREEZE) {
1055 dlog(DLOG_ALL, "[%03d] FROZEN(DEBUG) %s\n",
1056 work->index, pkg->portdir);
1057 work->state = WORKER_FROZEN;
1059 work->state = WORKER_IDLE;
1064 * Wait for one or more workers to complete.
1066 * WorkerMutex must be held.
1069 waitbuild(int whilematch, int dynamicmax)
1071 static time_t wblast_time;
1072 static time_t dmlast_time;
1078 if (whilematch == 0)
1081 while (RunningWorkers == whilematch) {
1082 for (i = 0; i < MaxWorkers; ++i) {
1083 work = &WorkerAry[i];
1084 if (work->state == WORKER_DONE ||
1085 work->state == WORKER_FAILED) {
1086 workercomplete(work);
1088 pthread_cond_signal(&work->cond);
1090 RunStatsUpdate(work);
1092 RunStatsUpdateTop();
1093 RunStatsUpdateLogs();
1095 if (RunningWorkers == whilematch) {
1096 clock_gettime(CLOCK_REALTIME, &ts);
1099 pthread_cond_timedwait(&WorkerCond, &WorkerMutex, &ts);
1103 * Dynamically reduce MaxWorkers based on the load. When
1104 * the load exceeds 2 x ncpus we reduce the number of workers
1105 * up to 75% of MaxWorkers @ (5 x ncpus) load.
1107 * Dynamically reduce MaxWorkers based on swap use, starting
1108 * at 10% swap and up to 75% of MaxWorkers at 40% swap.
1110 * NOTE! Generally speaking this allows more workers to be
1111 * configured which helps in two ways. First it allows
1112 * a higher build rate for smaller packages. Second
1113 * it allows dsynth to ratchet-down the number of slots
1114 * when large packages are forcing the load up.
1116 * A high load doesn't hurt efficiency, but swap usage
1117 * due to loading the tmpfs in lots of worker slots up
1118 * with tons of pkg install's (pre-reqs for a build)
1119 * does. Reducing the number of worker slots has a
1120 * huge beneficial effect on reducing swap use / paging.
1123 if (dynamicmax && (wblast_time == 0 ||
1124 (unsigned)(t - wblast_time) >= 5)) {
1125 double min_load = 1.5 * NumCores;
1126 double max_load = 5.0 * NumCores;
1127 double min_swap = 0.10;
1128 double max_swap = 0.40;
1140 * Cap based on load. This is back-loaded.
1142 getloadavg(dload, 3);
1143 if (dload[0] < min_load) {
1145 } else if (dload[0] <= max_load) {
1148 (dload[0] - min_load) /
1149 (max_load - min_load);
1151 max1 = MaxWorkers * 25 / 100;
1155 * Cap based on swap use. This is back-loaded.
1157 dswap = getswappct(&noswap);
1158 if (dswap < min_swap) {
1160 } else if (dswap <= max_swap) {
1163 (dswap - min_swap) /
1164 (max_swap - min_swap);
1166 max2 = MaxWorkers * 25 / 100;
1170 * Cap based on aggregate pkg-dependency memory
1171 * use installed in worker slots. This is
1174 * Since it can take a while for workers to retire
1175 * (to reduce RunningPkgDepSize), just set our
1176 * target 1 below the current run count to allow
1177 * jobs to retire without being replaced with new
1180 * In addition, in order to avoid a paging 'shock',
1181 * We enforce a 30 second-per-increment slow-start
1182 * once RunningPkgDepSize exceeds 1/2 the target.
1184 if (RunningPkgDepSize > PkgDepMemoryTarget) {
1185 max3 = RunningWorkers - 1;
1186 } else if (RunningPkgDepSize > PkgDepMemoryTarget / 2) {
1187 if (dmlast_time == 0 ||
1188 (unsigned)(t - dmlast_time) >= 30) {
1190 max3 = RunningWorkers + 1;
1192 max3 = RunningWorkers;
1199 * Priority reduction, convert to DynamicMaxWorkers
1208 * Restrict to allowed range, and also handle
1213 if (max_sel > DynamicMaxWorkers + 1)
1214 max_sel = DynamicMaxWorkers + 1;
1215 if (max_sel > MaxWorkers)
1216 max_sel = MaxWorkers;
1219 * Stop waiting if DynamicMaxWorkers is going to
1222 if (DynamicMaxWorkers < max1)
1228 if (DynamicMaxWorkers != max1) {
1229 dlog(DLOG_ALL | DLOG_FILTER,
1230 "[XXX] Load=%-6.2f(%2d) "
1231 "Swap=%-3.2f%%(%2d) "
1233 "Adjust Workers %d->%d\n",
1235 dswap * 100.0, max2,
1236 RunningPkgDepSize / (double)ONEGB, max3,
1237 DynamicMaxWorkers, max_sel);
1238 DynamicMaxWorkers = max_sel;
1246 * Worker pthread (WorkerAry)
1248 * This thread belongs to the dsynth master process and handled a worker slot.
1249 * (this_thread) -> WORKER fork/exec (WorkerPocess) -> (pty) -> sub-processes.
1252 childBuilderThread(void *arg)
1254 char *envary[1] = { NULL };
1255 worker_t *work = arg;
1260 volatile int dowait;
1265 pthread_mutex_lock(&WorkerMutex);
1266 while (work->terminate == 0) {
1269 switch(work->state) {
1272 case WORKER_PENDING:
1274 * Fork the management process for the pkg operation
1275 * on this worker slot.
1277 * This process will set up the environment, do the
1278 * mounts, will become the reaper, and will process
1279 * pipe commands and handle chroot operations.
1281 * NOTE: If SOCK_CLOEXEC is not supported WorkerMutex
1282 * is sufficient to interlock F_SETFD FD_CLOEXEC
1285 ddassert(work->pkg);
1286 if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC,
1287 PF_UNSPEC, work->fds)) {
1288 dfatal_errno("socketpair() during worker fork");
1290 snprintf(slotbuf, sizeof(slotbuf),
1292 snprintf(fdbuf, sizeof(fdbuf),
1294 snprintf(flagsbuf, sizeof(flagsbuf),
1295 "%d", WorkerProcFlags);
1301 * We pass the salve descriptor in fd 3 and close all
1302 * other descriptors for security.
1304 pthread_mutex_unlock(&WorkerMutex);
1307 close(work->fds[0]);
1308 dup2(work->fds[1], 3);
1310 fcntl(3, F_SETFD, 0);
1311 execle(DSynthExecPath, DSynthExecPath,
1312 "WORKER", slotbuf, fdbuf,
1313 work->pkg->portdir, work->pkg->pkgfile,
1316 write(2, "EXECLE FAILURE\n", 15);
1319 pthread_mutex_lock(&WorkerMutex);
1320 close(work->fds[1]);
1321 work->phase = PHASE_PENDING;
1325 work->state = WORKER_RUNNING;
1327 case WORKER_RUNNING:
1329 * Poll for status updates, if NULL is returned
1330 * and status is non-zero, the communications link
1331 * failed unexpectedly.
1334 pthread_mutex_unlock(&WorkerMutex);
1335 status = ipcreadmsg(work->fds[0], &wmsg);
1336 pthread_mutex_lock(&WorkerMutex);
1340 * Normal message, can include normal
1341 * termination which changes us over
1346 case WMSG_CMD_INSTALL_PKGS:
1347 wmsg.cmd = WMSG_RES_INSTALL_PKGS;
1348 wmsg.status = childInstallPkgDeps(work);
1349 pthread_mutex_unlock(&WorkerMutex);
1350 ipcwritemsg(work->fds[0], &wmsg);
1351 pthread_mutex_lock(&WorkerMutex);
1353 case WMSG_CMD_STATUS_UPDATE:
1354 work->phase = wmsg.phase;
1355 work->lines = wmsg.lines;
1356 if (work->memuse != wmsg.memuse) {
1357 RunningPkgDepSize +=
1358 wmsg.memuse - work->memuse;
1359 work->memuse = wmsg.memuse;
1362 case WMSG_CMD_SUCCESS:
1363 work->flags |= WORKERF_SUCCESS;
1365 case WMSG_CMD_FAILURE:
1366 work->flags |= WORKERF_FAILURE;
1368 case WMSG_CMD_FREEZEWORKER:
1369 work->flags |= WORKERF_FREEZE;
1374 RunStatsUpdate(work);
1377 close(work->fds[0]);
1378 pthread_mutex_unlock(&WorkerMutex);
1379 while (waitpid(work->pid, &status, 0) < 0 &&
1383 pthread_mutex_lock(&WorkerMutex);
1385 if (work->flags & WORKERF_SUCCESS) {
1386 pkg->flags |= PKGF_SUCCESS;
1387 work->state = WORKER_DONE;
1388 } else if (work->flags & WORKERF_FAILURE) {
1389 pkg->flags |= PKGF_FAILURE;
1390 work->state = WORKER_DONE;
1392 pkg->flags |= PKGF_FAILURE;
1393 work->state = WORKER_FAILED;
1395 work->flags |= WORKERF_STATUS_UPDATE;
1396 pthread_cond_signal(&WorkerCond);
1401 * pkg remains attached until frontend processes the
1402 * completion. The frontend will then set the state
1408 * A worker failure means that the worker did not
1409 * send us a WMSG_CMD_SUCCESS or WMSG_CMD_FAILURE
1410 * ipc before terminating.
1412 * We just sit in this state until the front-end
1413 * does something about it.
1417 dfatal("worker: [%03d] Unexpected state %d for worker %d",
1418 work->index, work->state, work->index);
1424 * The dsynth frontend will poll us approximately once
1425 * a second (its variable).
1428 pthread_cond_wait(&work->cond, &WorkerMutex);
1432 * Scrap the comm socket if running, this should cause the worker
1433 * process to kill its sub-programs and cleanup.
1435 if (work->state == WORKER_RUNNING) {
1436 pthread_mutex_unlock(&WorkerMutex);
1437 close(work->fds[0]);
1438 while (waitpid(work->pid, &status, 0) < 0 &&
1440 pthread_mutex_lock(&WorkerMutex);
1446 work->state = WORKER_EXITING;
1447 pthread_cond_signal(&WorkerCond);
1448 pthread_mutex_unlock(&WorkerMutex);
1454 * Install all the binary packages (we have already built them) that
1455 * the current work package depends on, without duplicates, in a script
1456 * which will be run from within the specified work jail.
1458 * Locked by WorkerMutex (global)
1461 childInstallPkgDeps(worker_t *work)
1466 if (PKGLIST_EMPTY(&work->pkg->idepon_list))
1469 asprintf(&buf, "%s/tmp/dsynth_install_pkgs", work->basedir);
1470 fp = fopen(buf, "w");
1471 ddassert(fp != NULL);
1472 fprintf(fp, "#!/bin/sh\n");
1474 fchmod(fileno(fp), 0755);
1476 childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 0, 1, 0);
1477 childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 1, 1, 0);
1478 fprintf(fp, "\nexit 0\n");
1486 childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list, int undoit,
1487 int depth, int first_one_only)
1495 PKGLIST_FOREACH(link, list) {
1499 * We don't want to mess up our depth test just below if
1500 * a DUMMY node had to be inserted. The nodes under the
1503 * The elements under a dummy node represent all the flabor,
1504 * a dependency that directly references a dummy node only
1505 * uses the first flavor (first_one_only / nfirst).
1507 ndepth = (pkg->flags & PKGF_DUMMY) ? depth : depth + 1;
1508 nfirst = (pkg->flags & PKGF_DUMMY) ? 1 : 0;
1511 * We only need all packages for the top-level dependencies.
1512 * The deeper ones only need DEP_TYPE_LIB and DEP_TYPE_RUN
1513 * (types greater than DEP_TYPE_BUILD) since they are already
1516 if (depth > 1 && link->dep_type <= DEP_TYPE_BUILD) {
1523 if (pkg->dsynth_install_flg == 1) {
1524 pkg->dsynth_install_flg = 0;
1525 tot += childInstallPkgDeps_recurse(fp,
1534 if (pkg->dsynth_install_flg) {
1535 if (DebugOpt >= 2 && pkg->pkgfile && fp) {
1536 fprintf(fp, "echo 'AlreadyHave %s'\n",
1544 tot += childInstallPkgDeps_recurse(fp, &pkg->idepon_list,
1545 undoit, ndepth, nfirst);
1546 if (pkg->dsynth_install_flg) {
1551 pkg->dsynth_install_flg = 1;
1554 * If this is a dummy node with no package, the originator
1555 * is requesting a flavored package. We select the default
1556 * flavor which we presume is the first one.
1558 if (pkg->pkgfile == NULL && (pkg->flags & PKGF_DUMMY)) {
1559 pkg_t *spkg = pkg->idepon_list.next->pkg;
1565 "echo 'DUMMY use %s (%p)'\n",
1566 pkg->portdir, pkg->pkgfile);
1571 "echo 'CANNOT FIND DEFAULT "
1579 * Generate package installation command
1581 if (fp && pkg->pkgfile) {
1582 fprintf(fp, "echo 'Installing /packages/All/%s'\n",
1584 fprintf(fp, "pkg install -q -y /packages/All/%s "
1588 fprintf(fp, "echo 'CANNOT FIND PKG FOR %s'\n",
1597 asprintf(&path, "%s/%s", RepositoryPath, pkg->pkgfile);
1598 ptr = strrchr(pkg->pkgfile, '.');
1599 if (stat(path, &st) == 0) {
1600 if (strcmp(ptr, ".tar") == 0)
1602 else if (strcmp(ptr, ".tgz") == 0)
1603 tot += st.st_size * 3;
1604 else if (strcmp(ptr, ".txz") == 0)
1605 tot += st.st_size * 5;
1606 else if (strcmp(ptr, ".tbz") == 0)
1607 tot += st.st_size * 3;
1609 tot += st.st_size * 2;
1620 * Worker process interactions.
1622 * The worker process is responsible for managing the build of a single
1623 * package. It is exec'd by the master dsynth and only loads the
1626 * This process does not run in the chroot. It will become the reaper for
1627 * all sub-processes and it will enter the chroot to execute various phases.
1628 * It catches SIGINTR, SIGHUP, and SIGPIPE and will iterate, terminate, and
1629 * reap all sub-process upon kill or exit.
1631 * The command line forwarded to this function is:
1633 * WORKER slot# socketfd portdir/subdir
1639 * SSL_NO_VERIFY_PEER=1
1640 * USE_PACKAGE_DEPENDS_ONLY=1 (exec_phase_depends)
1642 * PORT_DBDIR=/options For ports options
1643 * PACKAGE_BUILDING=yes Don't build packages that aren't legally
1644 * buildable for a binary repo.
1645 * PKG_DBDIR=/var/db/pkg
1646 * PKG_CACHEDIR=/var/cache/pkg
1647 * PKG_CREATE_VERBOSE=yes Ensure periodic output during packaging
1648 * (custom environment)
1649 * PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
1650 * UNAME_s=DragonFly (example)
1651 * UNAME_v=DragonFly 5.7-SYNTH (example)
1652 * UNAME_p=x86_64 (example)
1653 * UNAME_m=x86_64 (example)
1654 * UNAME_r=5.7-SYNTH (example)
1655 * NO_DEPENDS=yes (exec_phase)
1656 * DISTDIR=/distfiles
1657 * WRKDIRPREFIX=/construction
1659 * MAKE_JOBS_NUMBER=n
1663 * /usr/local/sbin/pkg-static install /packages/All/<the pkg pkg>
1664 * /usr/local/sbin/pkg-static install /packages/All/<pkg>
1665 * (for all dependencies)
1667 * PHASES: make -C path FLAVOR=flavor <phase>
1685 * package e.g. /construction/lang/perl5.28/pkg/perl5-5.28.2.txz
1686 * install-mtree (skipped)
1688 * deinstall (skipped)
1691 WorkerProcess(int ac, char **av)
1700 int do_install_phase;
1715 dlog(DLOG_ALL, "WORKER PROCESS %d- bad arguments\n", getpid());
1718 slot = strtol(av[1], NULL, 0);
1719 fd = strtol(av[2], NULL, 0); /* master<->slave messaging */
1722 flavor = strchr(portdir, '@');
1725 asprintf(&buf, "@%s", flavor);
1726 WorkerFlavorPrt = buf;
1727 buf = NULL; /* safety */
1729 WorkerProcFlags = strtol(av[5], NULL, 0);
1731 bzero(&wmsg, sizeof(wmsg));
1733 setproctitle("[%02d] WORKER STARTUP %s%s",
1734 slot, portdir, WorkerFlavorPrt);
1736 if (strcmp(portdir, "ports-mgmt/pkg") == 0)
1739 signal(SIGTERM, phaseTerminateSignal);
1740 signal(SIGINT, phaseTerminateSignal);
1741 signal(SIGHUP, phaseTerminateSignal);
1744 * Set up the environment
1746 setenv("TERM", "dumb", 1);
1747 setenv("USER", "root", 1);
1748 setenv("HOME", "/root", 1);
1749 setenv("LANG", "C", 1);
1750 setenv("SSL_NO_VERIFY_PEER", "1", 1);
1752 addbuildenv("USE_PACKAGE_DEPENDS_ONLY", "yes", BENV_MAKECONF);
1753 addbuildenv("PORTSDIR", "/xports", BENV_MAKECONF);
1754 addbuildenv("PORT_DBDIR", "/options", BENV_MAKECONF);
1755 addbuildenv("PKG_DBDIR", "/var/db/pkg", BENV_MAKECONF);
1756 addbuildenv("PKG_CACHEDIR", "/var/cache/pkg", BENV_MAKECONF);
1757 addbuildenv("PKG_SUFX", USE_PKG_SUFX, BENV_MAKECONF);
1758 if (WorkerProcFlags & WORKER_PROC_DEVELOPER)
1759 addbuildenv("DEVELOPER", "1", BENV_MAKECONF);
1762 * CCache is a horrible unreliable hack but... leave the
1763 * mechanism in-place in case someone has a death wish.
1766 addbuildenv("WITH_CCACHE_BUILD", "yes", BENV_MAKECONF);
1767 addbuildenv("CCACHE_DIR", "/ccache", BENV_MAKECONF);
1771 setenv("_PERL5_FROM_BIN", "5.28.2", 1);
1772 setenv("OPSYS", OperatingSystemName, 1);
1775 setenv("DFLYVERSION", "5.7.0", 1);
1776 setenv("OSVERSION", "9999999", 1);
1780 "/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin",
1783 setenv("UNAME_s", OperatingSystemName, 1);
1784 setenv("UNAME_v", VersionName, 1);
1785 setenv("UNAME_p", ArchitectureName, 1);
1786 setenv("UNAME_m", MachineName, 1);
1787 setenv("UNAME_r", ReleaseName, 1);
1789 addbuildenv("NO_DEPENDS", "yes", BENV_MAKECONF);
1790 addbuildenv("DISTDIR", "/distfiles", BENV_MAKECONF);
1791 addbuildenv("WRKDIRPREFIX", "/construction", BENV_MAKECONF);
1792 addbuildenv("BATCH", "yes", BENV_MAKECONF);
1795 * Special consideration
1797 * PACKAGE_BUILDING - Disallow packaging ports which do not allow
1798 * for binary distribution.
1800 * PKG_CREATE_VERBOSE - Ensure periodic output during the packaging
1801 * process to avoid a watchdog timeout.
1804 addbuildenv("PACKAGE_BUILDING", "yes", BENV_MAKECONF);
1805 addbuildenv("PKG_CREATE_VERBOSE", "yes", BENV_MAKECONF);
1806 asprintf(&buf, "%d", MaxJobs);
1807 addbuildenv("MAKE_JOBS_NUMBER", buf, BENV_MAKECONF);
1811 setenv("FLAVOR", flavor, 1);
1816 if (procctl(P_PID, getpid(), PROC_REAP_ACQUIRE, NULL) < 0)
1817 dfatal_errno("procctl() - Cannot become reaper");
1820 * Initialize a worker structure
1824 bzero(&pkg, sizeof(pkg));
1825 pkg.portdir = portdir; /* sans flavor */
1826 pkg.pkgfile = pkgfile;
1827 if (strchr(portdir, '/'))
1828 len = strchr(portdir, '/') - portdir;
1835 asprintf(&pkg.logfile,
1836 "%s/%*.*s___%s%s%s.log",
1837 LogsPath, len, len, portdir,
1838 ((portdir[len] == '/') ? portdir + len + 1 : portdir + len),
1839 (flavor ? "@" : ""),
1840 (flavor ? flavor : ""));
1841 tmpfd = open(pkg.logfile, O_RDWR|O_CREAT|O_TRUNC, 0666);
1843 if (DebugOpt >= 2) {
1844 dlog(DLOG_ALL, "[%03d] %s LOGFILE %s\n",
1845 slot, pkg.portdir, pkg.logfile);
1849 dlog(DLOG_ALL, "[%03d] LOGFILE %s (create failed)\n",
1854 * Setup the work structure. Because this is an exec'd sub-process,
1855 * there is only one work structure.
1857 work = &WorkerAry[0];
1858 work->flavor = flavor;
1861 work->start_time = time(NULL);
1867 setproctitle("[%02d] WORKER MOUNTS %s%s",
1868 slot, portdir, WorkerFlavorPrt);
1869 DoWorkerMounts(work);
1872 * Generate an /etc/make.conf in the build base
1874 asprintf(&buf, "%s/etc/make.conf", work->basedir);
1875 fp = fopen(buf, "w");
1876 dassert_errno(fp, "Unable to create %s\n", buf);
1877 for (benv = BuildEnv; benv; benv = benv->next) {
1878 if (benv->type & BENV_PKGLIST)
1880 if ((benv->type & BENV_CMDMASK) == BENV_MAKECONF) {
1881 if (DebugOpt >= 2) {
1882 dlog(DLOG_ALL, "[%03d] ENV %s=%s\n",
1883 slot, benv->label, benv->data);
1885 fprintf(fp, "%s=%s\n", benv->label, benv->data);
1895 initbulk(childHookRun, MaxBulk);
1900 wmsg.cmd = WMSG_CMD_INSTALL_PKGS;
1901 ipcwritemsg(fd, &wmsg);
1902 status = ipcreadmsg(fd, &wmsg);
1903 if (status < 0 || wmsg.cmd != WMSG_RES_INSTALL_PKGS)
1904 dfatal("pkg installation handshake failed");
1905 do_install_phase = wmsg.status;
1907 wmsg.cmd = WMSG_CMD_STATUS_UPDATE;
1908 wmsg.phase = PHASE_INSTALL_PKGS;
1911 status = ipcwritemsg(fd, &wmsg);
1914 dophase(work, &wmsg,
1915 WDOG5, PHASE_PACKAGE, "package");
1917 if (do_install_phase) {
1918 dophase(work, &wmsg,
1919 WDOG4, PHASE_INSTALL_PKGS, "setup");
1921 dophase(work, &wmsg,
1922 WDOG2, PHASE_CHECK_SANITY, "check-sanity");
1923 dophase(work, &wmsg,
1924 WDOG2, PHASE_PKG_DEPENDS, "pkg-depends");
1925 dophase(work, &wmsg,
1926 WDOG7, PHASE_FETCH_DEPENDS, "fetch-depends");
1927 dophase(work, &wmsg,
1928 WDOG7, PHASE_FETCH, "fetch");
1929 dophase(work, &wmsg,
1930 WDOG2, PHASE_CHECKSUM, "checksum");
1931 dophase(work, &wmsg,
1932 WDOG3, PHASE_EXTRACT_DEPENDS, "extract-depends");
1933 dophase(work, &wmsg,
1934 WDOG3, PHASE_EXTRACT, "extract");
1935 dophase(work, &wmsg,
1936 WDOG2, PHASE_PATCH_DEPENDS, "patch-depends");
1937 dophase(work, &wmsg,
1938 WDOG2, PHASE_PATCH, "patch");
1939 dophase(work, &wmsg,
1940 WDOG5, PHASE_BUILD_DEPENDS, "build-depends");
1941 dophase(work, &wmsg,
1942 WDOG5, PHASE_LIB_DEPENDS, "lib-depends");
1943 dophase(work, &wmsg,
1944 WDOG3, PHASE_CONFIGURE, "configure");
1945 dophase(work, &wmsg,
1946 WDOG9, PHASE_BUILD, "build");
1947 dophase(work, &wmsg,
1948 WDOG5, PHASE_RUN_DEPENDS, "run-depends");
1949 dophase(work, &wmsg,
1950 WDOG5, PHASE_STAGE, "stage");
1952 dophase(work, &wmsg,
1953 WDOG5, PHASE_TEST, "test");
1955 dophase(work, &wmsg,
1956 WDOG1, PHASE_CHECK_PLIST, "check-plist");
1957 dophase(work, &wmsg,
1958 WDOG5, PHASE_PACKAGE, "package");
1960 dophase(work, &wmsg,
1961 WDOG5, PHASE_INSTALL_MTREE, "install-mtree");
1962 dophase(work, &wmsg,
1963 WDOG5, PHASE_INSTALL, "install");
1964 dophase(work, &wmsg,
1965 WDOG5, PHASE_DEINSTALL, "deinstall");
1969 if (MasterPtyFd >= 0) {
1974 setproctitle("[%02d] WORKER CLEANUP %s%s",
1975 slot, portdir, WorkerFlavorPrt);
1978 * Copy the package to the repo.
1980 if (work->accum_error == 0) {
1984 asprintf(&b1, "%s/construction/%s/pkg/%s",
1985 work->basedir, pkg.portdir, pkg.pkgfile);
1986 asprintf(&b2, "%s/%s", RepositoryPath, pkg.pkgfile);
1987 if (copyfile(b1, b2)) {
1988 ++work->accum_error;
1989 dlog(DLOG_ALL, "[%03d] %s Unable to copy %s to %s\n",
1990 work->index, pkg.portdir, b1, b2);
1997 * Unmount, unless we are in DebugStopMode.
1999 if ((WorkerProcFlags & WORKER_PROC_DEBUGSTOP) == 0)
2000 DoWorkerUnmounts(work);
2003 * Send completion status to master dsynth worker thread.
2005 if (work->accum_error) {
2006 wmsg.cmd = WMSG_CMD_FAILURE;
2008 wmsg.cmd = WMSG_CMD_SUCCESS;
2010 ipcwritemsg(fd, &wmsg);
2011 if (WorkerProcFlags & WORKER_PROC_DEBUGSTOP) {
2012 wmsg.cmd = WMSG_CMD_FREEZEWORKER;
2013 ipcwritemsg(fd, &wmsg);
2016 while ((bulk = getbulk()) != NULL)
2023 dophase(worker_t *work, wmsg_t *wmsg, int wdog, int phaseid, const char *phase)
2025 pkg_t *pkg = work->pkg;
2039 if (work->accum_error)
2041 setproctitle("[%02d] WORKER %-8.8s %s%s",
2042 work->index, phase, pkg->portdir, WorkerFlavorPrt);
2043 wmsg->phase = phaseid;
2044 if (ipcwritemsg(work->fds[0], wmsg) < 0) {
2045 dlog(DLOG_ALL, "[%03d] %s Lost Communication with dsynth, "
2046 "aborting worker\n",
2047 work->index, pkg->portdir);
2048 ++work->accum_error;
2053 * Execute the port make command in chroot on a pty.
2057 if (MasterPtyFd >= 0) {
2062 * NOTE: We can't open the slave in the child because the
2063 * master may race a disconnection test. If we open
2064 * it in the parent our close() will flush any pending
2065 * output not read by the master (which is the same
2066 * parent process) and deadlock.
2068 * Solve this by hand-shaking the slave tty to give
2069 * the master time to close its slavefd.
2071 * Leave the tty defaults intact, which also likely
2072 * means it will be in line-buffered mode, so handshake
2075 slavefd = open(ptsname(MasterPtyFd), O_RDWR);
2076 dassert_errno(slavefd >= 0, "Cannot open slave pty");
2080 /* login_tty() closes slavefd */
2084 write(MasterPtyFd, "x\n", 2);
2087 pid = forkpty(&MasterPtyFd, NULL, NULL, NULL);
2094 * We are going through a pty, so set the tty modes to
2095 * Set tty modes so we do not get ^M's in the log files.
2097 * This isn't fatal if it doesn't work. Remember that
2098 * our output goes through the pty to the management
2099 * process which will log it.
2101 if (tcgetattr(1, &tio) == 0) {
2102 tio.c_oflag |= OPOST | ONOCR;
2103 tio.c_oflag &= ~(OCRNL | ONLCR);
2104 tio.c_iflag |= ICRNL;
2105 tio.c_iflag &= ~(INLCR|IGNCR);
2106 if (tcsetattr(1, TCSANOW, &tio)) {
2107 printf("tcsetattr failed: %s\n",
2111 printf("tcgetattr failed: %s\n", strerror(errno));
2115 * Clean-up, chdir, and chroot.
2118 if (chdir(work->basedir) < 0)
2119 dfatal_errno("chdir in phase initialization");
2120 if (chroot(work->basedir) < 0)
2121 dfatal_errno("chroot in phase initialization");
2124 * We have a choice here on how to handle stdin (fd 0).
2125 * We can leave it connected to the pty in which case
2126 * the build will just block if it tries to ask a
2127 * question (and the watchdog will kill it, eventually),
2128 * or we can try to EOF the pty, or we can attach /dev/null
2134 fd = open("/dev/null", O_RDWR);
2135 dassert_errno(fd >= 0, "cannot open /dev/null");
2143 * Execute the appropriate command.
2146 case PHASE_INSTALL_PKGS:
2147 snprintf(buf, sizeof(buf), "/tmp/dsynth_install_pkgs");
2148 execl(buf, buf, NULL);
2151 snprintf(buf, sizeof(buf), "/xports/%s", pkg->portdir);
2152 execl(MAKE_BINARY, MAKE_BINARY, "-C", buf, phase, NULL);
2157 fcntl(MasterPtyFd, F_SETFL, O_NONBLOCK);
2160 dlog(DLOG_ALL, "[%03d] %s Fork Failed: %s\n",
2161 work->index, pkg->logfile, strerror(errno));
2162 ++work->accum_error;
2168 fdlog = open(pkg->logfile, O_RDWR|O_CREAT|O_APPEND, 0644);
2170 dlog(DLOG_ALL, "[%03d] %s Cannot open logfile '%s': %s\n",
2171 work->index, pkg->portdir,
2172 pkg->logfile, strerror(errno));
2175 snprintf(buf, sizeof(buf),
2176 "----------------------------------------"
2177 "---------------------------------------\n"
2179 "----------------------------------------"
2180 "---------------------------------------\n",
2182 write(fdlog, buf, strlen(buf));
2184 start_time = time(NULL);
2185 last_time = start_time;
2186 wdog_time = start_time;
2191 ms = mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time);
2192 if (ms == MPTY_FAILED) {
2194 "[%03d] %s lost pty in phase %s, terminating\n",
2195 work->index, pkg->portdir, phase);
2202 * Generally speaking update status once a second.
2203 * This also allows us to detect if the management
2204 * dsynth process has gone away.
2206 next_time = time(NULL);
2207 if (next_time != last_time) {
2213 * Send status update to the worker management thread
2214 * in the master dsynth process. Remember, *WE* are
2215 * the worker management process sub-fork.
2217 if (ipcwritemsg(work->fds[0], wmsg) < 0)
2219 last_time = next_time;
2224 getloadavg(dload, 3);
2225 dv = dload[2] / NumCores;
2226 if (dv < (double)NumCores) {
2229 if (dv > 4.0 * NumCores)
2230 dv = 4.0 * NumCores;
2231 wdog_scaled = wdog * dv / NumCores;
2237 if (next_time - wdog_time >= wdog_scaled * 60) {
2238 snprintf(buf, sizeof(buf),
2240 "WATCHDOG TIMEOUT FOR %s in %s "
2241 "after %d minutes\n"
2244 pkg->portdir, phase, wdog_scaled, pid);
2246 write(fdlog, buf, strlen(buf));
2248 "[%03d] %s WATCHDOG TIMEOUT in %s "
2249 "after %d minutes (%d min scaled)\n",
2250 work->index, pkg->portdir, phase,
2253 ++work->accum_error;
2259 * Check process exit. Normally the pty will EOF
2260 * but if background processes remain we need to
2261 * check here to see if our primary exec is done,
2262 * so we can break out and reap those processes.
2264 * Generally reap any other processes we have inherited
2265 * while we are here.
2268 wpid = wait3(&status, WNOHANG, NULL);
2269 } while (wpid > 0 && wpid != pid);
2270 if (wpid == pid && WIFEXITED(status)) {
2276 next_time = time(NULL);
2278 setproctitle("[%02d] WORKER EXITREAP %s%s",
2279 work->index, pkg->portdir, WorkerFlavorPrt);
2282 * We usually get here due to a mpty EOF, but not always as there
2283 * could be persistent processes still holding the slave. Finish
2284 * up getting the exit status for the main process we are waiting
2285 * on and clean out any data left on the MasterPtyFd (as it could
2286 * be blocking the exit).
2288 while (wpid_reaped == 0) {
2289 (void)mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time);
2290 wpid = waitpid(pid, &status, WNOHANG);
2291 if (wpid == pid && WIFEXITED(status)) {
2295 if (wpid < 0 && errno != EINTR) {
2300 * Safety. The normal phase waits until the fork/exec'd
2301 * pid finishes, causing a pty EOF on exit (the slave
2302 * descriptor is closed by the kernel on exit so the
2303 * process should already have exited).
2305 * However, it is also possible to get here if the pty fails
2306 * for some reason. In this case, make sure that the process
2313 * Clean out anything left on the pty but don't wait around
2314 * because there could be background processes preventing the
2315 * slave side from closing.
2317 while (mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time) == MPTY_DATA)
2321 * Report on the exit condition. If the pid was somehow lost
2322 * (probably due to someone gdb'ing the process), assume an error.
2325 if (WEXITSTATUS(status)) {
2326 dlog(DLOG_ALL | DLOG_FILTER,
2327 "[%03d] %s Build phase '%s' failed exit %d\n",
2328 work->index, pkg->portdir, phase,
2329 WEXITSTATUS(status));
2330 ++work->accum_error;
2333 dlog(DLOG_ALL, "[%03d] %s Build phase '%s' failed - lost pid\n",
2334 work->index, pkg->portdir, phase);
2335 ++work->accum_error;
2339 * Kill any processes still running (sometimes processes end up in
2340 * the background during a dports build), and clean up any other
2341 * children that we have inherited.
2346 * After the extraction phase add the space used by /construction
2347 * to the memory use. This helps us reduce the amount of paging
2348 * we do due to extremely large package extractions (languages,
2351 * (dsynth already estimated the space used by the package deps
2352 * up front, but this will help us further).
2354 if (work->accum_error == 0 && phaseid == PHASE_EXTRACT) {
2358 asprintf(&b1, "%s/construction", work->basedir);
2359 if (statfs(b1, &sfs) == 0) {
2360 wmsg->memuse = (sfs.f_blocks - sfs.f_bfree) *
2362 ipcwritemsg(work->fds[0], wmsg);
2375 last_time = next_time - start_time;
2377 m = last_time / 60 % 60;
2378 h = last_time / 3600;
2380 fp = fdopen(fdlog, "a");
2382 dlog(DLOG_ALL, "[%03d] %s Cannot fdopen fdlog: %s %d\n",
2383 work->index, pkg->portdir,
2384 strerror(errno), fstat(fdlog, &st));
2390 if (work->accum_error) {
2391 fprintf(fp, "FAILED %02d:%02d:%02d\n", h, m, s);
2393 if (phaseid == PHASE_EXTRACT && wmsg->memuse) {
2394 fprintf(fp, "Extracted Memory Use: %6.2fM\n",
2395 wmsg->memuse / (1024.0 * 1024.0));
2397 fprintf(fp, "SUCCEEDED %02d:%02d:%02d\n", h, m, s);
2399 last_time = next_time - work->start_time;
2401 m = last_time / 60 % 60;
2402 h = last_time / 3600;
2403 if (phaseid == PHASE_PACKAGE) {
2404 fprintf(fp, "TOTAL TIME %02d:%02d:%02d\n", h, m, s);
2417 struct reaper_status rs;
2420 while (procctl(P_PID, getpid(), PROC_REAP_STATUS, &rs) == 0) {
2421 if ((rs.flags & PROC_REAP_ACQUIRE) == 0)
2423 if (rs.pid_head < 0)
2425 if (kill(rs.pid_head, SIGKILL) == 0) {
2426 while (waitpid(rs.pid_head, &status, 0) < 0)
2430 while (wait3(&status, 0, NULL) > 0)
2435 phaseTerminateSignal(int sig __unused)
2437 if (CopyFileFd >= 0)
2439 if (MasterPtyFd >= 0)
2442 kill(SigPid, SIGKILL);
2445 DoWorkerUnmounts(SigWork);
2451 buildskipreason(pkglink_t *parent, pkg_t *pkg)
2455 char *reason = NULL;
2461 if ((pkg->flags & PKGF_NOBUILD_I) && pkg->ignore)
2462 asprintf(&reason, "%s ", pkg->ignore);
2465 PKGLIST_FOREACH(link, &pkg->idepon_list) {
2467 if (link->dep_type > DEP_TYPE_BUILD)
2473 if ((scan->flags & (PKGF_ERROR | PKGF_NOBUILD)) == 0)
2475 if (scan->flags & PKGF_NOBUILD) {
2477 stack.next = parent;
2478 ptr = buildskipreason(&stack, scan);
2479 len = strlen(scan->portdir) + strlen(ptr) + 8;
2480 reason = realloc(reason, tot + len);
2481 snprintf(reason + tot, len, "%s->%s",
2482 scan->portdir, ptr);
2485 len = strlen(scan->portdir) + 8;
2486 reason = realloc(reason, tot + len);
2487 snprintf(reason + tot, len, "%s", scan->portdir);
2491 * Don't try to print the entire graph
2495 tot += strlen(reason + tot);
2496 reason[tot++] = ' ';
2503 * The master ptyfd is in non-blocking mode. Drain up to 1024 bytes
2504 * and update wmsg->lines and *wdog_timep as appropriate.
2506 * This function will poll, stalling up to 1 second.
2509 mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg, time_t *wdog_timep)
2516 pfd.events = POLLIN;
2519 poll(&pfd, 1, 1000);
2521 r = read(ptyfd, buf, sizeof(buf));
2523 *wdog_timep = time(NULL);
2524 if (r > 0 && fdlog >= 0)
2525 write(fdlog, buf, r);
2533 if (errno != EINTR && errno != EAGAIN)
2536 } else if (r == 0) {
2544 * Copy a (package) file from (src) to (dst), use an intermediate file and
2545 * rename to ensure that interruption does not leave us with a corrupt
2548 * This is called by the WORKER process.
2550 * (dsynth management thread -> WORKER process -> sub-processes)
2552 #define COPYBLKSIZE 32768
2555 copyfile(char *src, char *dst)
2565 asprintf(&tmp, "%s.new", dst);
2566 buf = malloc(COPYBLKSIZE);
2568 mask = sigsetmask(sigmask(SIGTERM)|sigmask(SIGINT)|sigmask(SIGHUP));
2569 fd1 = open(src, O_RDONLY|O_CLOEXEC);
2570 fd2 = open(tmp, O_RDWR|O_CREAT|O_TRUNC|O_CLOEXEC, 0644);
2573 while ((r = read(fd1, buf, COPYBLKSIZE)) > 0) {
2574 if (write(fd2, buf, r) != r)
2579 mask = sigsetmask(sigmask(SIGTERM)|sigmask(SIGINT)|sigmask(SIGHUP));
2587 if (rename(tmp, dst)) {
2602 * primary process (threaded) - run_start, run_end, pkg_ignored, pkg_skipped
2603 * worker process (threaded) - pkg_sucess, pkg_failure
2605 * If waitfor is non-zero this hook will be serialized.
2608 doHook(pkg_t *pkg, const char *id, const char *path, int waitfor)
2612 while (waitfor && getbulk() != NULL)
2615 queuebulk(pkg->portdir, id, path, pkg->pkgfile);
2617 queuebulk(NULL, id, path, NULL);
2618 while (waitfor && getbulk() != NULL)
2623 * Execute hook (backend)
2628 * s4 - pkgfile (if applicable)
2631 childHookRun(bulk_t *bulk)
2633 const char *cav[MAXCAC];
2634 buildenv_t benv[MAXCAC];
2648 bzero(benv, sizeof(benv));
2650 cav[cac++] = bulk->s3;
2652 benv[bi].label = "PROFILE";
2653 benv[bi].data = Profile;
2656 benv[bi].label = "DIR_PACKAGES";
2657 benv[bi].data = PackagesPath;
2660 benv[bi].label = "DIR_REPOSITORY";
2661 benv[bi].data = RepositoryPath;
2664 benv[bi].label = "DIR_PORTS";
2665 benv[bi].data = DPortsPath;
2668 benv[bi].label = "DIR_OPTIONS";
2669 benv[bi].data = OptionsPath;
2672 benv[bi].label = "DIR_DISTFILES";
2673 benv[bi].data = DistFilesPath;
2676 benv[bi].label = "DIR_LOGS";
2677 benv[bi].data = LogsPath;
2680 benv[bi].label = "DIR_BUILDBASE";
2681 benv[bi].data = BuildBase;
2684 if (strcmp(bulk->s2, "hook_run_start") == 0) {
2685 snprintf(buf1, sizeof(buf1), "%d", BuildTotal);
2686 benv[bi].label = "PORTS_QUEUED";
2687 benv[bi].data = buf1;
2689 } else if (strcmp(bulk->s2, "hook_run_end") == 0) {
2690 snprintf(buf1, sizeof(buf1), "%d", BuildSuccessCount);
2691 benv[bi].label = "PORTS_BUILT";
2692 benv[bi].data = buf1;
2694 snprintf(buf2, sizeof(buf2), "%d", BuildFailCount);
2695 benv[bi].label = "PORTS_FAILED";
2696 benv[bi].data = buf2;
2698 snprintf(buf3, sizeof(buf3), "%d", BuildIgnoreCount);
2699 benv[bi].label = "PORTS_IGNORED";
2700 benv[bi].data = buf3;
2702 snprintf(buf4, sizeof(buf4), "%d", BuildSkipCount);
2703 benv[bi].label = "PORTS_SKIPPED";
2704 benv[bi].data = buf4;
2708 * success, failure, ignored, skipped
2710 benv[bi].label = "RESULT";
2711 if (strcmp(bulk->s2, "hook_pkg_success") == 0) {
2712 benv[bi].data = "success";
2713 } else if (strcmp(bulk->s2, "hook_pkg_failure") == 0) {
2714 benv[bi].data = "failure";
2715 } else if (strcmp(bulk->s2, "hook_pkg_ignored") == 0) {
2716 benv[bi].data = "ignored";
2717 } else if (strcmp(bulk->s2, "hook_pkg_skipped") == 0) {
2718 benv[bi].data = "skipped";
2720 dfatal("Unknown hook id: %s", bulk->s2);
2726 * For compatibility with synth:
2728 * ORIGIN does not include any @flavor, thus it is suitable
2729 * for finding the actual port directory/subdirectory.
2731 * FLAVOR is set to ORIGIN if there is no flavor, otherwise
2732 * it is set to only the flavor sans the '@'.
2734 if ((ptr = strchr(bulk->s1, '@')) != NULL) {
2735 snprintf(buf1, sizeof(buf1), "%*.*s",
2736 (int)(ptr - bulk->s1),
2737 (int)(ptr - bulk->s1),
2739 benv[bi].label = "ORIGIN";
2740 benv[bi].data = buf1;
2742 benv[bi].label = "FLAVOR";
2743 benv[bi].data = ptr + 1;
2746 benv[bi].label = "ORIGIN";
2747 benv[bi].data = bulk->s1;
2749 benv[bi].label = "FLAVOR";
2750 benv[bi].data = bulk->s1;
2753 benv[bi].label = "PKGNAME";
2754 benv[bi].data = bulk->s4;
2758 benv[bi].label = NULL;
2759 benv[bi].data = NULL;
2761 fp = dexec_open(cav, cac, &pid, benv, 0, 0);
2762 while ((ptr = fgetln(fp, &len)) != NULL)
2765 if (dexec_close(fp, pid)) {
2767 "[XXX] %s SCRIPT %s (%s)\n",
2768 bulk->s1, bulk->s2, bulk->s3);