2 * Copyright (c) 2019 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * This code uses concepts and configuration based on 'synth', by
8 * John R. Marino <draco@marino.st>, which was written in ada.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in
18 * the documentation and/or other materials provided with the
20 * 3. Neither the name of The DragonFly Project nor the names of its
21 * contributors may be used to endorse or promote products derived
22 * from this software without specific, prior written permission.
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39 worker_t WorkerAry[MAXWORKERS];
42 int DynamicMaxWorkers;
43 pthread_mutex_t WorkerMutex;
44 pthread_cond_t WorkerCond;
46 static int build_find_leaves(pkg_t *parent, pkg_t *pkg, pkg_t ***build_tailp,
47 int *app, int *hasworkp, int level, int first);
48 static void build_clear_trav(pkg_t *pkg);
49 static void startbuild(pkg_t **build_listp, pkg_t ***build_tailp);
50 static int qsort_depi(const void *pkg1, const void *pkg2);
51 static int qsort_idep(const void *pkg1, const void *pkg2);
52 static void startworker(pkg_t *pkg, worker_t *work);
53 static void cleanworker(worker_t *work);
54 static void waitbuild(int whilematch, int dynamicmax);
55 static void workercomplete(worker_t *work);
56 static void *childBuilderThread(void *arg);
57 static int childInstallPkgDeps(worker_t *work);
58 static void childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list, int undoit);
59 static void dophase(worker_t *work, wmsg_t *wmsg,
60 int wdog, int phaseid, const char *phase);
61 static void phaseReapAll(void);
62 static void phaseTerminateSignal(int sig);
63 static char *buildskipreason(pkglink_t *parent, pkg_t *pkg);
64 static int mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg,
66 static int copyfile(char *src, char *dst);
68 static worker_t *SigWork;
69 static int MasterPtyFd = -1;
72 #define MPTY_FAILED -2
81 int BuildSuccessCount;
84 * Initialize the WorkerAry[]
87 DoInitBuild(int slot_override)
93 ddassert(slot_override < 0 || MaxWorkers == 1);
95 bzero(WorkerAry, MaxWorkers * sizeof(worker_t));
96 pthread_mutex_init(&WorkerMutex, NULL);
98 for (i = 0; i < MaxWorkers; ++i) {
100 work->index = (slot_override >= 0) ? slot_override : i;
101 work->state = WORKER_NONE;
102 asprintf(&work->basedir, "%s/SL%02d", BuildBase, work->index);
103 pthread_cond_init(&work->cond, NULL);
108 * Create required sub-directories. The base directories must already
109 * exist as a dsynth configuration safety.
111 if (stat(RepositoryPath, &st) < 0) {
112 if (mkdir(RepositoryPath, 0755) < 0)
113 dfatal("Cannot mkdir %s\n", RepositoryPath);
116 BuildInitialized = 1;
119 * slow-start (increases at a rate of 1 per 5 seconds)
121 if (SlowStartOpt > MaxWorkers)
122 DynamicMaxWorkers = MaxWorkers;
123 else if (SlowStartOpt > 0)
124 DynamicMaxWorkers = SlowStartOpt;
126 DynamicMaxWorkers = MaxWorkers;
130 * Called by the frontend to clean-up any hanging mounts.
137 ddassert(BuildInitialized);
139 for (i = 0; i < MaxWorkers; ++i) {
140 DoWorkerUnmounts(&WorkerAry[i]);
148 pkg_t *build_list = NULL;
149 pkg_t **build_tail = &build_list;
154 for (scan = pkgs; scan; scan = scan->bnext)
158 * The pkg and pkg-static binaries are needed. If already present
159 * then assume that the template is also valid, otherwise build
162 scan = GetPkgPkg(pkgs);
165 * Create our template. The template will be missing pkg
168 if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0) {
169 DoCreateTemplate(1); /* force a fresh template */
175 * This will clear the screen and set-up our gui, so sleep
176 * a little first in case the user wants to see what was
180 pthread_mutex_lock(&WorkerMutex);
184 if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0) {
189 * Build pkg/pkg-static.
192 build_tail = &scan->build_next;
193 startbuild(&build_list, &build_tail);
194 while (RunningWorkers == 1)
197 if (scan->flags & PKGF_NOBUILD)
198 dfatal("Unable to build 'pkg'");
199 if (scan->flags & PKGF_ERROR)
200 dfatal("Error building 'pkg'");
201 if ((scan->flags & PKGF_SUCCESS) == 0)
202 dfatal("Error building 'pkg'");
205 * Install pkg/pkg-static into the template
209 "tar --exclude '+*' --exclude '*/man/*' "
210 "-xvzpf %s/%s > /dev/null 2>&1",
216 dfatal("Command failed: %s\n", buf);
221 * Nominal bulk build sequence
226 for (scan = pkgs; scan; scan = scan->bnext) {
227 ddprintf(0, "SCANLEAVES %08x %s\n",
228 scan->flags, scan->portdir);
229 scan->flags |= PKGF_BUILDLOOP;
231 * NOTE: We must still find dependencies if PACKAGED
232 * to fill in the gaps, as some of them may
233 * need to be rebuilt.
235 if (scan->flags & (PKGF_SUCCESS | PKGF_FAILURE |
238 ddprintf(0, "%s: already built\n",
243 build_find_leaves(NULL, scan, &build_tail,
244 &ap, &haswork, 0, first);
245 ddprintf(0, "TOPLEVEL %s %08x\n",
248 scan->flags &= ~PKGF_BUILDLOOP;
249 build_clear_trav(scan);
253 startbuild(&build_list, &build_tail);
255 if (haswork == 0 && RunningWorkers) {
256 waitbuild(RunningWorkers, 1);
260 pthread_mutex_unlock(&WorkerMutex);
264 ddprintf(0, "BuildCount %d\n", BuildCount);
268 * Traverse the packages (pkg) depends on recursively until we find
269 * a leaf to build or report as unbuildable. Calculates and assigns a
270 * dependency count. Returns all parallel-buildable packages.
272 * (pkg) itself is only added to the list if it is immediately buildable.
276 build_find_leaves(pkg_t *parent, pkg_t *pkg, pkg_t ***build_tailp,
277 int *app, int *hasworkp, int level, int first)
286 * Already on build list, possibly in-progress, tell caller that
289 ddprintf(level, "sbuild_find_leaves %d %s %08x {\n",
290 level, pkg->portdir, pkg->flags);
291 if (pkg->flags & PKGF_BUILDLIST) {
292 ddprintf(level, "} (already on build list)\n");
293 *app |= PKGF_NOTREADY;
294 return (pkg->idep_count);
301 PKGLIST_FOREACH(link, &pkg->idepon_list) {
306 ddprintf(level, "check %s %08x\t", scan->portdir, scan->flags);
309 * When accounting for a successful build, just bump
310 * idep_count by one. scan->idep_count will heavily
311 * overlap packages that we count down multiple branches.
313 * We must still recurse through PACKAGED packages as
314 * some of their dependencies might be missing.
316 if (scan->flags & PKGF_SUCCESS) {
317 ddprintf(0, "SUCCESS - OK\n");
323 * ERROR includes FAILURE, which is set in numerous situations
324 * including when NOBUILD state is processed. So check for
325 * NOBUILD state first.
327 * An ERROR in a sub-package causes a NOBUILD in packages
330 if (scan->flags & PKGF_NOBUILD) {
331 ddprintf(0, "NOBUILD - OK "
332 "(propogate failure upward)\n");
333 *app |= PKGF_NOBUILD_S;
336 if (scan->flags & PKGF_ERROR) {
337 ddprintf(0, "ERROR - OK (propogate failure upward)\n");
338 *app |= PKGF_NOBUILD_S;
343 * If already on build-list this dependency is not ready.
345 if (scan->flags & PKGF_BUILDLIST) {
346 ddprintf(0, " [BUILDLIST]");
347 *app |= PKGF_NOTREADY;
351 * If not packaged this dependency is not ready for
354 if ((scan->flags & PKGF_PACKAGED) == 0) {
355 ddprintf(0, " [NOT_PACKAGED]");
356 *app |= PKGF_NOTREADY;
360 * Reduce search complexity, if we have already processed
361 * scan in the traversal it will either already be on the
362 * build list or it will not be buildable. Either way
363 * the parent is not buildable.
365 if (scan->flags & PKGF_BUILDTRAV) {
366 ddprintf(0, " [BUILDTRAV]\n");
367 *app |= PKGF_NOTREADY;
372 * Assert on dependency loop
375 if (scan->flags & PKGF_BUILDLOOP) {
376 dfatal("pkg dependency loop %s -> %s",
377 parent->portdir, scan->portdir);
379 scan->flags |= PKGF_BUILDLOOP;
381 ddprintf(0, " SUBRECURSION {\n");
382 idep_count += build_find_leaves(pkg, scan, build_tailp,
385 scan->flags &= ~PKGF_BUILDLOOP;
387 if (apsub & PKGF_NOBUILD) {
388 ddprintf(level, "} (sub-nobuild)\n");
389 } else if (apsub & PKGF_ERROR) {
390 ddprintf(level, "} (sub-error)\n");
391 } else if (apsub & PKGF_NOTREADY) {
392 ddprintf(level, "} (sub-notready)\n");
394 ddprintf(level, "} (sub-ok)\n");
398 pkg->idep_count = idep_count;
399 pkg->flags |= PKGF_BUILDTRAV;
402 * Incorporate scan results into pkg state.
404 if ((pkg->flags & PKGF_NOBUILD) == 0 && (*app & PKGF_NOBUILD)) {
406 } else if ((pkg->flags & PKGF_ERROR) == 0 && (*app & PKGF_ERROR)) {
409 pkg->flags |= *app & ~PKGF_NOTREADY;
412 * Clear PACKAGED bit if sub-dependencies aren't clean
414 if ((pkg->flags & PKGF_PACKAGED) &&
415 (pkg->flags & (PKGF_NOTREADY|PKGF_ERROR|PKGF_NOBUILD))) {
416 pkg->flags &= ~PKGF_PACKAGED;
417 ddassert(pkg->pkgfile);
418 asprintf(&buf, "%s/%s", RepositoryPath, pkg->pkgfile);
419 if (remove(buf) < 0) {
421 "[XXX] %s DELETE-PACKAGE %s (failed)\n",
425 "[XXX] %s DELETE-PACKAGE %s "
426 "(due to dependencies)\n",
435 * Handle propagated flags
437 if (pkg->flags & PKGF_ERROR) {
439 * This can only happen if the ERROR has already been
440 * processed and accounted for.
442 ddprintf(level, "} (ERROR - %s)\n", pkg->portdir);
443 } else if (*app & PKGF_NOTREADY) {
445 * We don't set PKGF_NOTREADY in the pkg, it is strictly
446 * a transient flag propagated via build_find_leaves().
448 * Just don't add the package to the list.
450 * NOTE: Even if NOBUILD is set (meaning we could list it
451 * and let startbuild() finish it up as a skip, we
452 * don't process it to the list because we want to
453 * process all the dependencies, so someone doing a
454 * manual build can get more complete information and
455 * does not have to iterate each failed dependency one
459 } else if (pkg->flags & PKGF_SUCCESS) {
460 ddprintf(level, "} (SUCCESS - %s)\n", pkg->portdir);
461 } else if (pkg->flags & PKGF_DUMMY) {
462 ddprintf(level, "} (DUMMY/META - SUCCESS)\n");
463 pkg->flags |= PKGF_SUCCESS;
466 dlog(DLOG_ALL, "[XXX] %s META-ALREADY-BUILT\n",
470 dlog(DLOG_SUCC, "[XXX] %s meta-node complete\n",
473 } else if (pkg->flags & PKGF_PACKAGED) {
475 * We can just mark the pkg successful. If this is
476 * the first pass, we count this as an initial pruning
477 * pass and reduce BuildTotal.
479 ddprintf(level, "} (PACKAGED - SUCCESS)\n");
480 pkg->flags |= PKGF_SUCCESS;
483 dlog(DLOG_ALL, "[XXX] %s ALREADY-BUILT\n",
489 * All dependencies are successful, queue new work
490 * and indicate not-ready to the parent (since our
491 * package has to be built).
493 * NOTE: The NOBUILD case propagates to here as well
494 * and is ultimately handled by startbuild().
497 if (pkg->flags & PKGF_NOBUILD)
498 ddprintf(level, "} (ADDLIST(NOBUILD) - %s)\n",
501 ddprintf(level, "} (ADDLIST - %s)\n", pkg->portdir);
502 pkg->flags |= PKGF_BUILDLIST;
504 *build_tailp = &pkg->build_next;
505 *app |= PKGF_NOTREADY;
513 build_clear_trav(pkg_t *pkg)
518 pkg->flags &= ~PKGF_BUILDTRAV;
519 PKGLIST_FOREACH(link, &pkg->idepon_list) {
521 if (scan && (scan->flags & PKGF_BUILDTRAV))
522 build_clear_trav(scan);
527 * Take a list of pkg ready to go, sort it, and assign it to worker
528 * slots. This routine blocks in waitbuild() until it can dispose of
531 * WorkerMutex is held by the caller.
535 startbuild(pkg_t **build_listp, pkg_t ***build_tailp)
546 static int IterateWorker;
551 if (*build_listp == NULL)
558 for (pkg = *build_listp; pkg; pkg = pkg->build_next)
560 idep_ary = calloc(count, sizeof(pkg_t *));
561 depi_ary = calloc(count, sizeof(pkg_t *));
564 for (pkg = *build_listp; pkg; pkg = pkg->build_next) {
565 idep_ary[count] = pkg;
566 depi_ary[count] = pkg;
571 * idep_ary - sorted by #of dependencies this pkg has.
572 * depi_ary - sorted by #of other packages that depend on this pkg.
574 qsort(idep_ary, count, sizeof(pkg_t *), qsort_idep);
575 qsort(depi_ary, count, sizeof(pkg_t *), qsort_depi);
581 * Half the workers build based on the highest depi count,
582 * the other half build based on the highest idep count.
584 * This is an attempt to get projects which many other projects
585 * depend on built first, but to also try to build large projects
586 * (which tend to have a lot of dependencies) earlier rather than
587 * later so the end of the bulk run doesn't inefficiently build
588 * the last few huge projects.
590 * Loop until we manage to assign slots to everyone. We do not
591 * wait for build completion.
593 * This is the point where we handle DUMMY packages (these are
594 * dummy unflavored packages which 'cover' all the flavors for
595 * a package). These are not real packages are marked SUCCESS
596 * at this time because their dependencies (the flavors) have all
599 while (idep_index != count || depi_index != count) {
604 * Find candidate to start sorted by depi or idep.
607 while (idep_index < count) {
608 ipkg = idep_ary[idep_index];
610 (PKGF_SUCCESS | PKGF_FAILURE |
611 PKGF_ERROR | PKGF_RUNNING)) == 0) {
619 while (depi_index < count) {
620 pkgi = depi_ary[depi_index];
622 (PKGF_SUCCESS | PKGF_FAILURE |
623 PKGF_ERROR | PKGF_RUNNING)) == 0) {
631 * ipkg and pkgi must either both be NULL, or both
634 if (ipkg == NULL && pkgi == NULL)
636 ddassert(ipkg && pkgi);
639 * Handle the NOBUILD case right here, there's no point
640 * queueing it anywhere.
642 if (ipkg->flags & PKGF_NOBUILD) {
645 ipkg->flags |= PKGF_FAILURE;
646 ipkg->flags &= ~PKGF_BUILDLIST;
650 reason = buildskipreason(NULL, ipkg);
651 dlog(DLOG_SKIP, "[XXX] %s skipped due to %s\n",
652 ipkg->portdir, reason);
656 if (pkgi->flags & PKGF_NOBUILD) {
659 pkgi->flags |= PKGF_FAILURE;
660 pkgi->flags &= ~PKGF_BUILDLIST;
664 reason = buildskipreason(NULL, pkgi);
665 dlog(DLOG_SKIP, "[XXX] %s skipped due to %s\n",
666 pkgi->portdir, reason);
672 * Block while no slots are available. waitbuild()
673 * will clean out any DONE states.
675 while (RunningWorkers >= DynamicMaxWorkers) {
676 waitbuild(RunningWorkers, 1);
680 * Find an available worker slot, there should be at
683 for (i = 0; i < MaxWorkers; ++i) {
684 n = IterateWorker % MaxWorkers;
685 work = &WorkerAry[n];
687 if (work->state == WORKER_DONE ||
688 work->state == WORKER_FAILED) {
689 workercomplete(work);
691 if (work->state == WORKER_NONE ||
692 work->state == WORKER_IDLE) {
693 if (n <= MaxWorkers / 2) {
694 startworker(pkgi, work);
696 startworker(ipkg, work);
703 ddassert(i != MaxWorkers);
708 * We disposed of the whole list
713 *build_tailp = build_listp;
716 typedef const pkg_t *pkg_tt;
719 qsort_idep(const void *pkg1_arg, const void *pkg2_arg)
721 const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg;
722 const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg;
724 return (pkg2->idep_count - pkg1->idep_count);
728 qsort_depi(const void *pkg1_arg, const void *pkg2_arg)
730 const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg;
731 const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg;
733 return (pkg2->depi_count - pkg1->depi_count);
737 * Frontend starts a pkg up on a worker
739 * WorkerMutex must be held.
742 startworker(pkg_t *pkg, worker_t *work)
744 switch(work->state) {
746 pthread_create(&work->td, NULL, childBuilderThread, work);
747 work->state = WORKER_IDLE;
750 dlog(DLOG_ALL, "[%03d] %s START\n", work->index, pkg->portdir);
752 pkg->flags |= PKGF_RUNNING;
754 pthread_cond_signal(&work->cond);
765 dfatal("startworker: Unexpected state %d for worker %d",
766 work->state, work->index);
772 cleanworker(worker_t *work)
774 work->state = WORKER_PENDING;
776 work->accum_error = 0;
777 work->start_time = time(NULL);
781 * Frontend finishes up a completed pkg on a worker.
783 * If the worker is in a FAILED state we clean the pkg out but (for now)
784 * leave it in its failed state so we can debug. At this point
785 * workercomplete() will be called every once in a while on the state
786 * and we have to deal with the NULL pkg.
788 * WorkerMutex must be held.
791 workercomplete(worker_t *work)
796 * Steady state FAILED case.
798 if (work->state == WORKER_FAILED) {
799 if (work->pkg == NULL)
804 * Process pkg out of the worker
807 if (pkg->flags & (PKGF_ERROR|PKGF_NOBUILD)) {
808 pkg->flags |= PKGF_FAILURE;
811 * This NOBUILD condition XXX can occur if the package is
812 * not allowed to be built.
814 if (pkg->flags & PKGF_NOBUILD) {
818 reason = buildskipreason(NULL, pkg);
819 dlog(DLOG_SKIP, "[%03d] %s skipped due to %s\n",
820 work->index, pkg->portdir, reason);
824 dlog(DLOG_FAIL, "[%03d] %s build-failed\n",
825 work->index, pkg->portdir);
833 t = time(NULL) - work->start_time;
839 "[%03d] %s\tbuild-succeded in %02d:%02d:%02d\n",
840 work->index, pkg->portdir, h, m, s);
841 pkg->flags |= PKGF_SUCCESS;
845 pkg->flags &= ~PKGF_BUILDLIST;
846 dlog(DLOG_ALL, "[%03d] %s FINISHED (%s)\n",
849 ((pkg->flags & PKGF_SUCCESS) ? "success" : "failure"));
850 pkg->flags &= ~PKGF_RUNNING;
854 if (work->state == WORKER_FAILED) {
855 dlog(DLOG_ALL, "[%03d] XXX/XXX WORKER IS IN A FAILED STATE\n",
857 } else if (work->flags & WORKERF_FREEZE) {
858 dlog(DLOG_ALL, "[%03d] XXX/XXX WORKER FROZEN BY REQUEST\n",
860 work->state = WORKER_FROZEN;
862 work->state = WORKER_IDLE;
867 * Wait for one or more workers to complete.
869 * WorkerMutex must be held.
872 waitbuild(int whilematch, int dynamicmax)
874 static time_t wblast_time;
883 while (RunningWorkers == whilematch) {
884 for (i = 0; i < MaxWorkers; ++i) {
885 work = &WorkerAry[i];
886 if (work->state == WORKER_DONE ||
887 work->state == WORKER_FAILED) {
888 workercomplete(work);
890 pthread_cond_signal(&work->cond);
896 if (RunningWorkers == whilematch) {
897 clock_gettime(CLOCK_REALTIME, &ts);
900 pthread_cond_timedwait(&WorkerCond, &WorkerMutex, &ts);
904 * Dynamically reduce MaxWorkers based on the load. When
905 * the load exceeds 2 x ncpus we reduce the number of workers
906 * up to 75% of MaxWorkers @ (5 x ncpus) load.
908 * Dynamically reduce MaxWorkers based on swap use, starting
909 * at 10% swap and up to 75% of MaxWorkers at 40% swap.
911 * NOTE! Generally speaking this allows more workers to be
912 * configured which helps in two ways. First it allows
913 * a higher build rate for smaller packages. Second
914 * it allows dsynth to ratchet-down the number of slots
915 * when large packages are forcing the load up.
917 * A high load doesn't hurt efficiency, but swap usage
918 * due to loading the tmpfs in lots of worker slots up
919 * with tons of pkg install's (pre-reqs for a build)
920 * does. Reducing the number of worker slots has a
921 * huge beneficial effect on reducing swap use / paging.
924 if (dynamicmax && (wblast_time == 0 ||
925 (unsigned)(t - wblast_time) >= 5)) {
926 double min_load = 1.5 * NumCores;
927 double max_load = 5.0 * NumCores;
928 double min_swap = 0.10;
929 double max_swap = 0.40;
938 getloadavg(dload, 3);
939 dswap = getswappct(&noswap);
941 if (dload[0] < min_load) {
943 } else if (dload[0] <= max_load) {
944 reduce1 = 75 * (dload[0] - min_load) /
945 (max_load - min_load);
947 reduce1 = MaxWorkers * 75 / 100;
950 if (dswap < min_swap) {
952 } else if (dswap <= max_swap) {
953 reduce2 = 75 * (dswap - min_swap) /
954 (max_swap - min_swap);
956 reduce2 = MaxWorkers * 75 / 100;
960 * Priority reduction, convert to DynamicMaxWorkers
962 if (reduce1 > reduce2)
966 reduce = MaxWorkers - reduce;
969 if (reduce > MaxWorkers)
975 if (reduce > DynamicMaxWorkers + 1) {
976 reduce = DynamicMaxWorkers + 1;
980 * Stop waiting if DynamicMaxWorkers is going
983 if (DynamicMaxWorkers < reduce)
989 if (DynamicMaxWorkers != reduce) {
991 "[XXX] Load=%-6.2f(-%-2d) "
992 "Swappct=%-3.2f(-%-2d) "
996 dswap * 100.0, reduce2,
999 DynamicMaxWorkers = reduce;
1007 * Worker pthread (WorkerAry)
1009 * This thread belongs to the dsynth master process and handled a worker slot.
1010 * (this_thread) -> WORKER fork/exec (WorkerPocess) -> (pty) -> sub-processes.
1013 childBuilderThread(void *arg)
1015 char *envary[1] = { NULL };
1016 worker_t *work = arg;
1021 volatile int dowait;
1026 pthread_mutex_lock(&WorkerMutex);
1027 while (work->terminate == 0) {
1030 switch(work->state) {
1033 case WORKER_PENDING:
1035 * Fork the management process for the pkg operation
1036 * on this worker slot.
1038 * This process will set up the environment, do the
1039 * mounts, will become the reaper, and will process
1040 * pipe commands and handle chroot operations.
1042 * NOTE: If SOCK_CLOEXEC is not supported WorkerMutex
1043 * is sufficient to interlock F_SETFD FD_CLOEXEC
1046 ddassert(work->pkg);
1047 if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC,
1048 PF_UNSPEC, work->fds)) {
1049 dfatal_errno("socketpair() during worker fork");
1051 snprintf(slotbuf, sizeof(slotbuf),
1053 snprintf(fdbuf, sizeof(fdbuf),
1055 snprintf(flagsbuf, sizeof(flagsbuf),
1056 "%d", DebugStopMode);
1062 * We pass the salve descriptor in fd 3 and close all
1063 * other descriptors for security.
1065 pthread_mutex_unlock(&WorkerMutex);
1068 close(work->fds[0]);
1069 dup2(work->fds[1], 3);
1071 fcntl(3, F_SETFD, 0);
1072 execle(DSynthExecPath, DSynthExecPath,
1073 "WORKER", slotbuf, fdbuf,
1074 work->pkg->portdir, work->pkg->pkgfile,
1077 write(2, "EXECLE FAILURE\n", 15);
1080 pthread_mutex_lock(&WorkerMutex);
1081 close(work->fds[1]);
1082 work->phase = PHASE_PENDING;
1085 work->state = WORKER_RUNNING;
1087 case WORKER_RUNNING:
1089 * Poll for status updates, if NULL is returned
1090 * and status is non-zero, the communications link
1091 * failed unexpectedly.
1094 pthread_mutex_unlock(&WorkerMutex);
1095 status = ipcreadmsg(work->fds[0], &wmsg);
1096 pthread_mutex_lock(&WorkerMutex);
1100 * Normal message, can include normal
1101 * termination which changes us over
1106 case WMSG_CMD_INSTALL_PKGS:
1107 wmsg.cmd = WMSG_RES_INSTALL_PKGS;
1108 wmsg.status = childInstallPkgDeps(work);
1109 pthread_mutex_unlock(&WorkerMutex);
1110 ipcwritemsg(work->fds[0], &wmsg);
1111 pthread_mutex_lock(&WorkerMutex);
1113 case WMSG_CMD_STATUS_UPDATE:
1114 work->phase = wmsg.phase;
1115 work->lines = wmsg.lines;
1117 case WMSG_CMD_SUCCESS:
1118 work->flags |= WORKERF_SUCCESS;
1120 case WMSG_CMD_FAILURE:
1121 work->flags |= WORKERF_FAILURE;
1123 case WMSG_CMD_FREEZEWORKER:
1124 work->flags |= WORKERF_FREEZE;
1132 close(work->fds[0]);
1133 pthread_mutex_unlock(&WorkerMutex);
1134 while (waitpid(work->pid, &status, 0) < 0 &&
1138 pthread_mutex_lock(&WorkerMutex);
1140 if (work->flags & WORKERF_SUCCESS) {
1141 pkg->flags |= PKGF_SUCCESS;
1142 work->state = WORKER_DONE;
1143 } else if (work->flags & WORKERF_FAILURE) {
1144 pkg->flags |= PKGF_FAILURE;
1145 work->state = WORKER_DONE;
1147 pkg->flags |= PKGF_FAILURE;
1148 work->state = WORKER_FAILED;
1150 work->flags |= WORKERF_STATUS_UPDATE;
1151 pthread_cond_signal(&WorkerCond);
1156 * pkg remains attached until frontend processes the
1157 * completion. The frontend will then set the state
1163 * A worker failure means that the worker did not
1164 * send us a WMSG_CMD_SUCCESS or WMSG_CMD_FAILURE
1165 * ipc before terminating.
1167 * We just sit in this state until the front-end
1168 * does something about it.
1172 dfatal("worker: Unexpected state %d for worker %d",
1173 work->state, work->index);
1179 * The dsynth frontend will poll us approximately once
1180 * a second (its variable).
1183 pthread_cond_wait(&work->cond, &WorkerMutex);
1187 * Scrap the comm socket if running, this should cause the worker
1188 * process to kill its sub-programs and cleanup.
1190 if (work->state == WORKER_RUNNING) {
1191 pthread_mutex_unlock(&WorkerMutex);
1192 close(work->fds[0]);
1193 while (waitpid(work->pid, &status, 0) < 0 &&
1195 pthread_mutex_lock(&WorkerMutex);
1201 work->state = WORKER_EXITING;
1202 pthread_cond_signal(&WorkerCond);
1203 pthread_mutex_unlock(&WorkerMutex);
1209 * Install all the binary packages (we have already built them) that
1210 * the current work package depends on, without duplicates, in a script
1211 * which will be run from within the specified work jail.
1213 * Locked by WorkerMutex (global)
1216 childInstallPkgDeps(worker_t *work)
1221 if (PKGLIST_EMPTY(&work->pkg->idepon_list))
1224 asprintf(&buf, "%s/tmp/dsynth_install_pkgs", work->basedir);
1225 fp = fopen(buf, "w");
1226 ddassert(fp != NULL);
1227 fprintf(fp, "#!/bin/sh\n");
1229 fchmod(fileno(fp), 0755);
1231 childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 0);
1232 childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 1);
1233 fprintf(fp, "\nexit 0\n");
1240 childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list, int undoit)
1245 PKGLIST_FOREACH(link, list) {
1249 if (pkg->dsynth_install_flg == 1) {
1250 pkg->dsynth_install_flg = 0;
1251 childInstallPkgDeps_recurse(fp,
1257 if (pkg->dsynth_install_flg) {
1258 if (DebugOpt && pkg->pkgfile) {
1259 fprintf(fp, "echo 'AlreadyHave %s'\n",
1265 childInstallPkgDeps_recurse(fp, &pkg->idepon_list, undoit);
1266 if (pkg->dsynth_install_flg)
1268 pkg->dsynth_install_flg = 1;
1271 * If this is a dummy node with no package, the originator
1272 * is requesting a flavored package. We select the default
1273 * flavor which we presume is the first one.
1275 if (pkg->pkgfile == NULL && (pkg->flags & PKGF_DUMMY)) {
1276 pkg_t *spkg = pkg->idepon_list.next->pkg;
1281 "echo 'DUMMY use %s (%p)'\n",
1282 pkg->portdir, pkg->pkgfile);
1285 "echo 'CANNOT FIND DEFAULT FLAVOR "
1292 * Generate package installation command
1295 fprintf(fp, "echo 'Installing /packages/All/%s'\n",
1297 fprintf(fp, "pkg install -q -y /packages/All/%s "
1301 fprintf(fp, "echo 'CANNOT FIND PKG FOR %s'\n",
1308 * Worker process interactions.
1310 * The worker process is responsible for managing the build of a single
1311 * package. It is exec'd by the master dsynth and only loads the
1314 * This process does not run in the chroot. It will become the reaper for
1315 * all sub-processes and it will enter the chroot to execute various phases.
1316 * It catches SIGINTR, SIGHUP, and SIGPIPE and will iterate, terminate, and
1317 * reap all sub-process upon kill or exit.
1319 * The command line forwarded to this function is:
1321 * WORKER slot# socketfd portdir/subdir
1327 * SSL_NO_VERIFY_PEER=1
1328 * USE_PACKAGE_DEPENDS_ONLY=1 (exec_phase_depends)
1330 * PORT_DBDIR=/options For ports options
1331 * PACKAGE_BUILDING=yes Don't build packages that aren't legally
1332 * buildable for a binary repo.
1333 * PKG_DBDIR=/var/db/pkg
1334 * PKG_CACHEDIR=/var/cache/pkg
1335 * PKG_CREATE_VERBOSE=yes Ensure periodic output during packaging
1336 * (custom environment)
1337 * PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
1338 * UNAME_s=DragonFly (example)
1339 * UNAME_v=DragonFly 5.7-SYNTH (example)
1340 * UNAME_p=x86_64 (example)
1341 * UNAME_m=x86_64 (example)
1342 * UNAME_r=5.7-SYNTH (example)
1343 * NO_DEPENDS=yes (exec_phase)
1344 * DISTDIR=/distfiles
1345 * WRKDIRPREFIX=/construction
1347 * MAKE_JOBS_NUMBER=n
1351 * /usr/local/sbin/pkg-static install /packages/All/<the pkg pkg>
1352 * /usr/local/sbin/pkg-static install /packages/All/<pkg>
1353 * (for all dependencies)
1355 * PHASES: make -C path FLAVOR=flavor <phase>
1373 * package e.g. /construction/lang/perl5.28/pkg/perl5-5.28.2.txz
1374 * install-mtree (skipped)
1376 * deinstall (skipped)
1380 WorkerProcess(int ac, char **av)
1389 int do_install_phase;
1402 dlog(DLOG_ALL, "WORKER PROCESS %d- bad arguments\n", getpid());
1405 slot = strtol(av[1], NULL, 0);
1406 fd = strtol(av[2], NULL, 0); /* master<->slave messaging */
1409 flavor = strchr(portdir, '@');
1412 DebugStopMode = strtol(av[5], NULL, 0);
1414 bzero(&wmsg, sizeof(wmsg));
1416 setproctitle("WORKER [%02d] STARTUP %s", slot, portdir);
1418 if (strcmp(portdir, "ports-mgmt/pkg") == 0)
1421 signal(SIGTERM, phaseTerminateSignal);
1422 signal(SIGINT, phaseTerminateSignal);
1423 signal(SIGHUP, phaseTerminateSignal);
1426 * Set up the environment
1428 setenv("TERM", "dumb", 1);
1429 setenv("USER", "root", 1);
1430 setenv("HOME", "/root", 1);
1431 setenv("LANG", "C", 1);
1432 setenv("SSL_NO_VERIFY_PEER", "1", 1);
1433 setenv("USE_PACKAGE_DEPENDS_ONLY", "1", 1);
1434 setenv("PORTSDIR", "/xports", 1);
1435 setenv("PORT_DBDIR", "/options", 1);
1436 setenv("PKG_DBDIR", "/var/db/pkg", 1);
1437 setenv("PKG_CACHEDIR", "/var/cache/pkg", 1);
1440 setenv("_PERL5_FROM_BIN", "5.28.2", 1);
1441 setenv("OPSYS", OperatingSystemName, 1);
1444 setenv("DFLYVERSION", "5.7.0", 1);
1445 setenv("OSVERSION", "9999999", 1);
1449 "/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin",
1452 setenv("UNAME_s", OperatingSystemName, 1);
1453 setenv("UNAME_v", VersionName, 1);
1454 setenv("UNAME_p", ArchitectureName, 1);
1455 setenv("UNAME_m", MachineName, 1);
1456 setenv("UNAME_r", ReleaseName, 1);
1458 setenv("NO_DEPENDS", "yes", 1);
1459 setenv("DISTDIR", "/distfiles", 1);
1460 setenv("WRKDIRPREFIX", "/construction", 1);
1461 setenv("BATCH", "yes", 1);
1464 * Special consideration
1466 * PACKAGE_BUILDING - Disallow packaging ports which do not allow
1467 * for binary distribution.
1469 * PKG_CREATE_VERBOSE - Ensure periodic output during the packaging
1470 * process to avoid a watchdog timeout.
1473 setenv("PACKAGE_BUILDING", "yes", 1);
1474 setenv("PKG_CREATE_VERBOSE", "yes", 1);
1476 asprintf(&buf, "%d", MaxJobs);
1477 setenv("MAKE_JOBS_NUMBER", buf, 1);
1481 setenv("FLAVOR", flavor, 1);
1484 * Load environment from profile
1486 for (benv = BuildEnv; benv; benv = benv->next) {
1487 if (DebugOpt >= 2) {
1488 dlog(DLOG_ALL, "[%03d] ENV %s=%s\n",
1489 slot, benv->label, benv->data);
1491 setenv(benv->label, benv->data, 1);
1497 if (procctl(P_PID, getpid(), PROC_REAP_ACQUIRE, NULL) < 0)
1498 dfatal_errno("procctl() - Cannot become reaper");
1501 * Initialize a worker structure
1505 bzero(&pkg, sizeof(pkg));
1506 pkg.portdir = portdir; /* sans flavor */
1507 pkg.pkgfile = pkgfile;
1508 if (strchr(portdir, '/'))
1509 len = strchr(portdir, '/') - portdir;
1516 asprintf(&pkg.logfile,
1517 "%s/%*.*s___%s%s%s.log",
1518 LogsPath, len, len, portdir,
1519 ((portdir[len] == '/') ? portdir + len + 1 : portdir + len),
1520 (flavor ? "@" : ""),
1521 (flavor ? flavor : ""));
1522 tmpfd = open(pkg.logfile, O_RDWR|O_CREAT|O_TRUNC, 0666);
1524 dlog(DLOG_ALL, "[%03d] %s LOGFILE %s\n",
1525 slot, pkg.portdir, pkg.logfile);
1528 dlog(DLOG_ALL, "[%03d] LOGFILE %s (create failed)\n",
1533 * Setup the work structure. Because this is an exec'd sub-process,
1534 * there is only one work structure.
1536 work = &WorkerAry[0];
1537 work->flavor = flavor;
1540 work->start_time = time(NULL);
1543 * Do mounts and start the phases
1546 setproctitle("WORKER [%02d] MOUNTS %s", slot, portdir);
1547 DoWorkerMounts(work);
1549 wmsg.cmd = WMSG_CMD_INSTALL_PKGS;
1550 ipcwritemsg(fd, &wmsg);
1551 status = ipcreadmsg(fd, &wmsg);
1552 if (status < 0 || wmsg.cmd != WMSG_RES_INSTALL_PKGS)
1553 dfatal("pkg installation handshake failed");
1554 do_install_phase = wmsg.status;
1556 wmsg.cmd = WMSG_CMD_STATUS_UPDATE;
1557 wmsg.phase = PHASE_INSTALL_PKGS;
1560 status = ipcwritemsg(fd, &wmsg);
1563 dophase(work, &wmsg,
1564 WDOG5, PHASE_PACKAGE, "package");
1566 if (do_install_phase) {
1567 dophase(work, &wmsg,
1568 WDOG4, PHASE_INSTALL_PKGS, "setup");
1570 dophase(work, &wmsg,
1571 WDOG2, PHASE_CHECK_SANITY, "check-sanity");
1572 dophase(work, &wmsg,
1573 WDOG2, PHASE_PKG_DEPENDS, "pkg-depends");
1574 dophase(work, &wmsg,
1575 WDOG7, PHASE_FETCH_DEPENDS, "fetch-depends");
1576 dophase(work, &wmsg,
1577 WDOG7, PHASE_FETCH, "fetch");
1578 dophase(work, &wmsg,
1579 WDOG2, PHASE_CHECKSUM, "checksum");
1580 dophase(work, &wmsg,
1581 WDOG3, PHASE_EXTRACT_DEPENDS, "extract-depends");
1582 dophase(work, &wmsg,
1583 WDOG3, PHASE_EXTRACT, "extract");
1584 dophase(work, &wmsg,
1585 WDOG2, PHASE_PATCH_DEPENDS, "patch-depends");
1586 dophase(work, &wmsg,
1587 WDOG2, PHASE_PATCH, "patch");
1588 dophase(work, &wmsg,
1589 WDOG5, PHASE_BUILD_DEPENDS, "build-depends");
1590 dophase(work, &wmsg,
1591 WDOG5, PHASE_LIB_DEPENDS, "lib-depends");
1592 dophase(work, &wmsg,
1593 WDOG3, PHASE_CONFIGURE, "configure");
1594 dophase(work, &wmsg,
1595 WDOG9, PHASE_BUILD, "build");
1596 dophase(work, &wmsg,
1597 WDOG5, PHASE_RUN_DEPENDS, "run-depends");
1598 dophase(work, &wmsg,
1599 WDOG5, PHASE_STAGE, "stage");
1601 dophase(work, &wmsg,
1602 WDOG5, PHASE_TEST, "test");
1604 dophase(work, &wmsg,
1605 WDOG1, PHASE_CHECK_PLIST, "check-plist");
1606 dophase(work, &wmsg,
1607 WDOG5, PHASE_PACKAGE, "package");
1609 dophase(work, &wmsg,
1610 WDOG5, PHASE_INSTALL_MTREE, "install-mtree");
1611 dophase(work, &wmsg,
1612 WDOG5, PHASE_INSTALL, "install");
1613 dophase(work, &wmsg,
1614 WDOG5, PHASE_DEINSTALL, "deinstall");
1618 if (MasterPtyFd >= 0) {
1623 setproctitle("WORKER [%02d] CLEANUP %s", slot, portdir);
1626 * Copy the package to the repo.
1628 if (work->accum_error == 0) {
1632 asprintf(&b1, "%s/construction/%s/pkg/%s",
1633 work->basedir, pkg.portdir, pkg.pkgfile);
1634 asprintf(&b2, "%s/%s", RepositoryPath, pkg.pkgfile);
1635 if (copyfile(b1, b2)) {
1636 ++work->accum_error;
1637 dlog(DLOG_ALL, "[%03d] %s Unable to copy %s to %s\n",
1638 work->index, pkg.portdir, b1, b2);
1645 * Unmount, unless we are in DebugStopMode.
1647 if (DebugStopMode == 0)
1648 DoWorkerUnmounts(work);
1651 * Send completion status to master dsynth worker thread.
1653 if (work->accum_error) {
1654 wmsg.cmd = WMSG_CMD_FAILURE;
1656 wmsg.cmd = WMSG_CMD_SUCCESS;
1658 ipcwritemsg(fd, &wmsg);
1659 if (DebugStopMode) {
1660 wmsg.cmd = WMSG_CMD_FREEZEWORKER;
1661 ipcwritemsg(fd, &wmsg);
1666 dophase(worker_t *work, wmsg_t *wmsg, int wdog, int phaseid, const char *phase)
1668 pkg_t *pkg = work->pkg;
1682 if (work->accum_error)
1684 setproctitle("WORKER [%02d] %8.8s %s",
1685 work->index, phase, pkg->portdir);
1686 wmsg->phase = phaseid;
1687 if (ipcwritemsg(work->fds[0], wmsg) < 0) {
1688 dlog(DLOG_ALL, "[%03d] %s Lost Communication with dsynth, "
1689 "aborting worker\n",
1690 work->index, pkg->portdir);
1691 ++work->accum_error;
1696 * Execute the port make command in chroot on a pty
1700 if (MasterPtyFd >= 0) {
1703 slavefd = open(ptsname(MasterPtyFd), O_RDWR);
1704 dassert_errno(slavefd >= 0, "Cannot open slave pty");
1712 pid = forkpty(&MasterPtyFd, NULL, NULL, NULL);
1717 if (chdir(work->basedir) < 0)
1718 dfatal_errno("chdir in phase initialization");
1719 if (chroot(work->basedir) < 0)
1720 dfatal_errno("chroot in phase initialization");
1723 * We have a choice here on how to handle stdin (fd 0).
1724 * We can leave it connected to the pty in which case
1725 * the build will just block if it tries to ask a
1726 * question (and the watchdog will kill it, eventually),
1727 * or we can try to EOF the pty, or we can attach /dev/null
1733 fd = open("/dev/null", O_RDWR);
1734 dassert_errno(fd >= 0, "cannot open /dev/null");
1742 * Execute the appropriate command.
1745 case PHASE_INSTALL_PKGS:
1746 snprintf(buf, sizeof(buf), "/tmp/dsynth_install_pkgs");
1747 execl(buf, buf, NULL);
1750 snprintf(buf, sizeof(buf), "/xports/%s", pkg->portdir);
1751 execl(MAKE_BINARY, MAKE_BINARY, "-C", buf, phase, NULL);
1756 fcntl(MasterPtyFd, F_SETFL, O_NONBLOCK);
1759 dlog(DLOG_ALL, "[%03d] %s Fork Failed\n",
1760 work->index, pkg->logfile);
1761 ++work->accum_error;
1767 fdlog = open(pkg->logfile, O_RDWR|O_CREAT|O_APPEND, 0644);
1769 dlog(DLOG_ALL, "[%03d] %s Cannot open logfile '%s': %s\n",
1770 work->index, pkg->portdir,
1771 pkg->logfile, strerror(errno));
1773 start_time = time(NULL);
1774 last_time = start_time;
1775 wdog_time = start_time;
1780 ms = mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time);
1781 if (ms == MPTY_FAILED) {
1783 "[%03d] %s lost pty in phase %s, terminating\n",
1784 work->index, pkg->portdir, phase);
1791 * Generally speaking update status once a second.
1792 * This also allows us to detect if the management
1793 * dsynth process has gone away.
1795 next_time = time(NULL);
1796 if (next_time != last_time) {
1802 * Send status update to the worker management thread
1803 * in the master dsynth process. Remember, *WE* are
1804 * the worker management process sub-fork.
1806 if (ipcwritemsg(work->fds[0], wmsg) < 0)
1808 last_time = next_time;
1813 getloadavg(dload, 3);
1814 dv = dload[2] / NumCores;
1815 if (dv < (double)NumCores) {
1818 if (dv > 4.0 * NumCores)
1819 dv = 4.0 * NumCores;
1820 wdog_scaled = wdog * dv / NumCores;
1826 if (next_time - wdog_time >= wdog_scaled * 60) {
1827 snprintf(buf, sizeof(buf),
1829 "WATCHDOG TIMEOUT FOR %s in %s "
1830 "after %d minutes\n"
1833 pkg->portdir, phase, wdog_scaled, pid);
1835 write(fdlog, buf, strlen(buf));
1837 "[%03d] WATCHDOG TIMEOUT FOR %s in %s "
1838 "after %d minutes (%d min scaled)\n",
1839 work->index, pkg->portdir, phase,
1842 ++work->accum_error;
1848 * Check process exit. Normally the pty will EOF
1849 * but if background processes remain we need to
1850 * check here to see if our primary exec is done,
1851 * so we can break out and reap those processes.
1853 * Generally reap any other processes we have inherited
1854 * while we are here.
1857 wpid = wait3(&status, WNOHANG, NULL);
1858 } while (wpid > 0 && wpid != pid);
1859 if (wpid == pid && WIFEXITED(status)) {
1865 next_time = time(NULL);
1867 setproctitle("WORKER [%02d] EXITREAP %s",
1868 work->index, pkg->portdir);
1871 * We usually get here due to a mpty EOF, but not always as there
1872 * could be persistent processes still holding the slave. Finish
1873 * up getting the exit status for the main process we are waiting
1874 * on and clean out any data left on the MasterPtyFd (as it could
1875 * be blocking the exit).
1877 while (wpid_reaped == 0) {
1878 (void)mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time);
1879 wpid = waitpid(pid, &status, WNOHANG);
1880 if (wpid == pid && WIFEXITED(status)) {
1884 if (wpid < 0 && errno != EINTR) {
1889 * Safety. The normal phase waits until the fork/exec'd
1890 * pid finishes, causing a pty EOF on exit (the slave
1891 * descriptor is closed by the kernel on exit so the
1892 * process should already have exited).
1894 * However, it is also possible to get here if the pty fails
1895 * for some reason. In this case, make sure that the process
1902 * Clean out anything left on the pty but don't wait around
1903 * because there could be background processes preventing the
1904 * slave side from closing.
1906 while (mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time) == MPTY_DATA)
1910 * Report on the exit condition. If the pid was somehow lost
1911 * (probably due to someone gdb'ing the process), assume an error.
1914 if (WEXITSTATUS(status)) {
1915 dlog(DLOG_ALL, "[%03d] %s Build phase '%s' "
1917 work->index, pkg->portdir, phase,
1918 WEXITSTATUS(status));
1919 ++work->accum_error;
1922 dlog(DLOG_ALL, "[%03d] %s Build phase '%s' failed - lost pid\n",
1923 work->index, pkg->portdir, phase);
1924 ++work->accum_error;
1928 * Kill any processes still running (sometimes processes end up in
1929 * the background during a dports build), and clean up any other
1930 * children that we have inherited.
1943 last_time = next_time - start_time;
1945 m = last_time / 60 % 60;
1946 h = last_time / 3600;
1948 fp = fdopen(fdlog, "a");
1950 dlog(DLOG_ALL, "[%03d] %s Cannot fdopen fdlog: %s %d\n",
1951 work->index, pkg->portdir,
1952 strerror(errno), fstat(fdlog, &st));
1957 fprintf(fp, "--------\n");
1958 if (work->accum_error) {
1959 fprintf(fp, "PHASE %s FAILED %02d:%02d:%02d\n",
1962 fprintf(fp, "PHASE %s SUCCEEDED %02d:%02d:%02d\n",
1965 last_time = next_time - work->start_time;
1967 m = last_time / 60 % 60;
1968 h = last_time / 3600;
1969 if (phaseid == PHASE_PACKAGE) {
1970 fprintf(fp, "TOTAL TIME %02d:%02d:%02d\n", h, m, s);
1972 fprintf(fp, "--------\n");
1983 struct reaper_status rs;
1986 while (procctl(P_PID, getpid(), PROC_REAP_STATUS, &rs) == 0) {
1987 if ((rs.flags & PROC_REAP_ACQUIRE) == 0)
1989 if (rs.pid_head < 0)
1991 if (kill(rs.pid_head, SIGKILL) == 0) {
1992 while (waitpid(rs.pid_head, &status, 0) < 0)
1996 while (wait3(&status, 0, NULL) > 0)
2001 phaseTerminateSignal(int sig __unused)
2003 if (MasterPtyFd >= 0)
2006 kill(SigPid, SIGKILL);
2009 DoWorkerUnmounts(SigWork);
2015 buildskipreason(pkglink_t *parent, pkg_t *pkg)
2019 char *reason = NULL;
2026 PKGLIST_FOREACH(link, &pkg->idepon_list) {
2030 if ((scan->flags & (PKGF_ERROR | PKGF_NOBUILD)) == 0)
2032 if (scan->flags & PKGF_NOBUILD) {
2034 stack.next = parent;
2035 ptr = buildskipreason(&stack, scan);
2036 len = strlen(scan->portdir) + strlen(ptr) + 8;
2037 reason = realloc(reason, tot + len);
2038 snprintf(reason + tot, len, "%s->%s",
2039 scan->portdir, ptr);
2042 len = strlen(scan->portdir) + 8;
2043 reason = realloc(reason, tot + len);
2044 snprintf(reason + tot, len, "%s", scan->portdir);
2048 * Don't try to print the entire graph
2052 tot += strlen(reason + tot);
2053 reason[tot++] = ' ';
2060 * The master ptyfd is in non-blocking mode. Drain up to 1024 bytes
2061 * and update wmsg->lines and *wdog_timep as appropriate.
2063 * This function will poll, stalling up to 1 second.
2066 mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg, time_t *wdog_timep)
2073 pfd.events = POLLIN;
2076 poll(&pfd, 1, 1000);
2078 r = read(ptyfd, buf, sizeof(buf));
2080 *wdog_timep = time(NULL);
2081 if (r > 0 && fdlog >= 0)
2082 write(fdlog, buf, r);
2090 if (errno != EINTR && errno != EAGAIN)
2093 } else if (r == 0) {
2101 * Copy a (package) file from (src) to (dst), use an intermediate file and
2102 * rename to ensure that interruption does not leave us with a corrupt
2105 * This is called by the WORKER process.
2107 * (dsynth management thread -> WORKER process -> sub-processes)
2109 #define COPYBLKSIZE 32768
2112 copyfile(char *src, char *dst)
2121 asprintf(&tmp, "%s.new", dst);
2122 buf = malloc(COPYBLKSIZE);
2124 fd1 = open(src, O_RDONLY|O_CLOEXEC);
2125 fd2 = open(tmp, O_RDWR|O_CREAT|O_TRUNC|O_CLOEXEC, 0644);
2126 while ((r = read(fd1, buf, COPYBLKSIZE)) > 0) {
2127 if (write(fd2, buf, r) != r)
2137 if (rename(tmp, dst)) {