dphase - Stabilization
[dragonfly.git] / usr.bin / dsynth / build.c
CommitLineData
8e25f19b
MD
1/*
2 * Copyright (c) 2019 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
6 *
7 * This code uses concepts and configuration based on 'synth', by
8 * John R. Marino <draco@marino.st>, which was written in ada.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 *
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in
18 * the documentation and/or other materials provided with the
19 * distribution.
20 * 3. Neither the name of The DragonFly Project nor the names of its
21 * contributors may be used to endorse or promote products derived
22 * from this software without specific, prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * SUCH DAMAGE.
36 */
37#include "dsynth.h"
38
39worker_t WorkerAry[MAXWORKERS];
40int BuildInitialized;
41int RunningWorkers;
f7f25838 42int DynamicMaxWorkers;
8d9409b8 43int FailedWorkers;
8e25f19b
MD
44pthread_mutex_t WorkerMutex;
45pthread_cond_t WorkerCond;
46
47static int build_find_leaves(pkg_t *parent, pkg_t *pkg, pkg_t ***build_tailp,
48 int *app, int *hasworkp, int level, int first);
49static void build_clear_trav(pkg_t *pkg);
50static void startbuild(pkg_t **build_listp, pkg_t ***build_tailp);
51static int qsort_depi(const void *pkg1, const void *pkg2);
52static int qsort_idep(const void *pkg1, const void *pkg2);
53static void startworker(pkg_t *pkg, worker_t *work);
54static void cleanworker(worker_t *work);
3699ee09 55static void waitbuild(int whilematch, int dynamicmax);
8e25f19b
MD
56static void workercomplete(worker_t *work);
57static void *childBuilderThread(void *arg);
58static int childInstallPkgDeps(worker_t *work);
59static void childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list, int undoit);
60static void dophase(worker_t *work, wmsg_t *wmsg,
61 int wdog, int phaseid, const char *phase);
62static void phaseReapAll(void);
63static void phaseTerminateSignal(int sig);
3699ee09 64static char *buildskipreason(pkglink_t *parent, pkg_t *pkg);
7880dcf7
MD
65static int mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg,
66 time_t *wdog_timep);
8e25f19b
MD
67static int copyfile(char *src, char *dst);
68
69static worker_t *SigWork;
1d6e00cd 70static int MasterPtyFd = -1;
8e25f19b
MD
71static pid_t SigPid;
72
7880dcf7
MD
73#define MPTY_FAILED -2
74#define MPTY_AGAIN -1
75#define MPTY_EOF 0
76#define MPTY_DATA 1
77
8e25f19b
MD
78int BuildTotal;
79int BuildCount;
80int BuildSkipCount;
81int BuildFailCount;
82int BuildSuccessCount;
83
84/*
85 * Initialize the WorkerAry[]
86 */
87void
88DoInitBuild(int slot_override)
89{
90 worker_t *work;
91 struct stat st;
92 int i;
93
94 ddassert(slot_override < 0 || MaxWorkers == 1);
95
96 bzero(WorkerAry, MaxWorkers * sizeof(worker_t));
97 pthread_mutex_init(&WorkerMutex, NULL);
98
99 for (i = 0; i < MaxWorkers; ++i) {
100 work = &WorkerAry[i];
101 work->index = (slot_override >= 0) ? slot_override : i;
102 work->state = WORKER_NONE;
103 asprintf(&work->basedir, "%s/SL%02d", BuildBase, work->index);
104 pthread_cond_init(&work->cond, NULL);
105 }
106 BuildCount = 0;
107
108 /*
109 * Create required sub-directories. The base directories must already
110 * exist as a dsynth configuration safety.
111 */
112 if (stat(RepositoryPath, &st) < 0) {
113 if (mkdir(RepositoryPath, 0755) < 0)
114 dfatal("Cannot mkdir %s\n", RepositoryPath);
115 }
116
117 BuildInitialized = 1;
f7f25838
MD
118
119 /*
120 * slow-start (increases at a rate of 1 per 5 seconds)
121 */
122 if (SlowStartOpt > MaxWorkers)
123 DynamicMaxWorkers = MaxWorkers;
124 else if (SlowStartOpt > 0)
125 DynamicMaxWorkers = SlowStartOpt;
126 else
127 DynamicMaxWorkers = MaxWorkers;
8e25f19b
MD
128}
129
130/*
131 * Called by the frontend to clean-up any hanging mounts.
132 */
133void
134DoCleanBuild(void)
135{
136 int i;
137
138 ddassert(BuildInitialized);
139
140 for (i = 0; i < MaxWorkers; ++i) {
141 DoWorkerUnmounts(&WorkerAry[i]);
142 }
143 dlogreset();
144}
145
146void
147DoBuild(pkg_t *pkgs)
148{
149 pkg_t *build_list = NULL;
150 pkg_t **build_tail = &build_list;
151 pkg_t *scan;
152 int haswork = 1;
153 int first = 1;
154
155 for (scan = pkgs; scan; scan = scan->bnext)
156 ++BuildTotal;
157
158 /*
159 * The pkg and pkg-static binaries are needed. If already present
160 * then assume that the template is also valid, otherwise build
161 * both.
162 */
163 scan = GetPkgPkg(pkgs);
164
165 /*
1645cafe
MD
166 * Create our template. The template will be missing pkg
167 * and pkg-static.
8e25f19b 168 */
1645cafe 169 if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0) {
1d6e00cd
MD
170 DoCreateTemplate(1); /* force a fresh template */
171 } else {
172 DoCreateTemplate(0);
1645cafe
MD
173 }
174
175 /*
176 * This will clear the screen and set-up our gui, so sleep
177 * a little first in case the user wants to see what was
178 * printed before.
179 */
180 sleep(2);
8e25f19b
MD
181 pthread_mutex_lock(&WorkerMutex);
182 GuiInit();
183 GuiReset();
184
185 if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0) {
186 char *buf;
187 int rc;
188
8e25f19b
MD
189 /*
190 * Build pkg/pkg-static.
191 */
192 build_list = scan;
193 build_tail = &scan->build_next;
194 startbuild(&build_list, &build_tail);
3699ee09
MD
195 while (RunningWorkers == 1)
196 waitbuild(1, 0);
8e25f19b
MD
197
198 if (scan->flags & PKGF_NOBUILD)
199 dfatal("Unable to build 'pkg'");
200 if (scan->flags & PKGF_ERROR)
201 dfatal("Error building 'pkg'");
202 if ((scan->flags & PKGF_SUCCESS) == 0)
203 dfatal("Error building 'pkg'");
204
205 /*
206 * Install pkg/pkg-static into the template
207 */
208 asprintf(&buf,
209 "cd %s/Template; "
210 "tar --exclude '+*' --exclude '*/man/*' "
1645cafe 211 "-xvzpf %s/%s > /dev/null 2>&1",
8e25f19b
MD
212 BuildBase,
213 RepositoryPath,
214 scan->pkgfile);
215 rc = system(buf);
216 if (rc)
217 dfatal("Command failed: %s\n", buf);
218 free(buf);
219 }
220
221 /*
222 * Nominal bulk build sequence
223 */
224 while (haswork) {
225 haswork = 0;
226 fflush(stdout);
227 for (scan = pkgs; scan; scan = scan->bnext) {
228 ddprintf(0, "SCANLEAVES %08x %s\n",
229 scan->flags, scan->portdir);
230 scan->flags |= PKGF_BUILDLOOP;
231 /*
232 * NOTE: We must still find dependencies if PACKAGED
233 * to fill in the gaps, as some of them may
234 * need to be rebuilt.
235 */
236 if (scan->flags & (PKGF_SUCCESS | PKGF_FAILURE |
3699ee09 237 PKGF_ERROR)) {
8e25f19b
MD
238#if 0
239 ddprintf(0, "%s: already built\n",
240 scan->portdir);
241#endif
242 } else {
243 int ap = 0;
244 build_find_leaves(NULL, scan, &build_tail,
245 &ap, &haswork, 0, first);
246 ddprintf(0, "TOPLEVEL %s %08x\n",
247 scan->portdir, ap);
248 }
249 scan->flags &= ~PKGF_BUILDLOOP;
250 build_clear_trav(scan);
251 }
252 first = 0;
253 fflush(stdout);
254 startbuild(&build_list, &build_tail);
255
256 if (haswork == 0 && RunningWorkers) {
3699ee09 257 waitbuild(RunningWorkers, 1);
8e25f19b
MD
258 haswork = 1;
259 }
260 }
261 pthread_mutex_unlock(&WorkerMutex);
262
3699ee09
MD
263 GuiDone();
264
8e25f19b
MD
265 ddprintf(0, "BuildCount %d\n", BuildCount);
266}
267
268/*
269 * Traverse the packages (pkg) depends on recursively until we find
270 * a leaf to build or report as unbuildable. Calculates and assigns a
271 * dependency count. Returns all parallel-buildable packages.
272 *
273 * (pkg) itself is only added to the list if it is immediately buildable.
274 */
275static
276int
277build_find_leaves(pkg_t *parent, pkg_t *pkg, pkg_t ***build_tailp,
278 int *app, int *hasworkp, int level, int first)
279{
280 pkglink_t *link;
281 pkg_t *scan;
282 int idep_count = 0;
283 int apsub;
284 char *buf;
285
286 /*
287 * Already on build list, possibly in-progress, tell caller that
288 * it is not ready.
289 */
290 ddprintf(level, "sbuild_find_leaves %d %s %08x {\n",
291 level, pkg->portdir, pkg->flags);
292 if (pkg->flags & PKGF_BUILDLIST) {
293 ddprintf(level, "} (already on build list)\n");
294 *app |= PKGF_NOTREADY;
295 return (pkg->idep_count);
296 }
297
298 /*
299 * Check dependencies
300 */
301 ++level;
302 PKGLIST_FOREACH(link, &pkg->idepon_list) {
303 scan = link->pkg;
304
305 if (scan == NULL)
306 continue;
307 ddprintf(level, "check %s %08x\t", scan->portdir, scan->flags);
308
309 /*
310 * When accounting for a successful build, just bump
311 * idep_count by one. scan->idep_count will heavily
312 * overlap packages that we count down multiple branches.
313 *
314 * We must still recurse through PACKAGED packages as
315 * some of their dependencies might be missing.
316 */
317 if (scan->flags & PKGF_SUCCESS) {
318 ddprintf(0, "SUCCESS - OK\n");
319 ++idep_count;
320 continue;
321 }
9c4c701f
MD
322
323 /*
324 * ERROR includes FAILURE, which is set in numerous situations
325 * including when NOBUILD state is processed. So check for
326 * NOBUILD state first.
327 *
328 * An ERROR in a sub-package causes a NOBUILD in packages
329 * that depend on it.
330 */
8e25f19b
MD
331 if (scan->flags & PKGF_NOBUILD) {
332 ddprintf(0, "NOBUILD - OK "
333 "(propogate failure upward)\n");
334 *app |= PKGF_NOBUILD_S;
335 continue;
336 }
9c4c701f
MD
337 if (scan->flags & PKGF_ERROR) {
338 ddprintf(0, "ERROR - OK (propogate failure upward)\n");
339 *app |= PKGF_NOBUILD_S;
340 continue;
341 }
8e25f19b
MD
342
343 /*
344 * If already on build-list this dependency is not ready.
345 */
346 if (scan->flags & PKGF_BUILDLIST) {
347 ddprintf(0, " [BUILDLIST]");
348 *app |= PKGF_NOTREADY;
349 }
350
351 /*
352 * If not packaged this dependency is not ready for
353 * the caller.
354 */
355 if ((scan->flags & PKGF_PACKAGED) == 0) {
356 ddprintf(0, " [NOT_PACKAGED]");
357 *app |= PKGF_NOTREADY;
358 }
359
360 /*
361 * Reduce search complexity, if we have already processed
362 * scan in the traversal it will either already be on the
363 * build list or it will not be buildable. Either way
364 * the parent is not buildable.
365 */
366 if (scan->flags & PKGF_BUILDTRAV) {
367 ddprintf(0, " [BUILDTRAV]\n");
368 *app |= PKGF_NOTREADY;
369 continue;
370 }
371
372 /*
373 * Assert on dependency loop
374 */
375 ++idep_count;
376 if (scan->flags & PKGF_BUILDLOOP) {
377 dfatal("pkg dependency loop %s -> %s",
378 parent->portdir, scan->portdir);
379 }
380 scan->flags |= PKGF_BUILDLOOP;
381 apsub = 0;
382 ddprintf(0, " SUBRECURSION {\n");
383 idep_count += build_find_leaves(pkg, scan, build_tailp,
384 &apsub, hasworkp,
385 level + 1, first);
386 scan->flags &= ~PKGF_BUILDLOOP;
387 *app |= apsub;
388 if (apsub & PKGF_NOBUILD) {
389 ddprintf(level, "} (sub-nobuild)\n");
390 } else if (apsub & PKGF_ERROR) {
391 ddprintf(level, "} (sub-error)\n");
392 } else if (apsub & PKGF_NOTREADY) {
393 ddprintf(level, "} (sub-notready)\n");
394 } else {
395 ddprintf(level, "} (sub-ok)\n");
396 }
397 }
398 --level;
399 pkg->idep_count = idep_count;
400 pkg->flags |= PKGF_BUILDTRAV;
401
402 /*
403 * Incorporate scan results into pkg state.
404 */
405 if ((pkg->flags & PKGF_NOBUILD) == 0 && (*app & PKGF_NOBUILD)) {
406 *hasworkp = 1;
407 } else if ((pkg->flags & PKGF_ERROR) == 0 && (*app & PKGF_ERROR)) {
408 *hasworkp = 1;
409 }
410 pkg->flags |= *app & ~PKGF_NOTREADY;
411
412 /*
413 * Clear PACKAGED bit if sub-dependencies aren't clean
414 */
415 if ((pkg->flags & PKGF_PACKAGED) &&
416 (pkg->flags & (PKGF_NOTREADY|PKGF_ERROR|PKGF_NOBUILD))) {
417 pkg->flags &= ~PKGF_PACKAGED;
418 ddassert(pkg->pkgfile);
419 asprintf(&buf, "%s/%s", RepositoryPath, pkg->pkgfile);
420 if (remove(buf) < 0) {
421 dlog(DLOG_ALL,
422 "[XXX] %s DELETE-PACKAGE %s (failed)\n",
423 pkg->portdir, buf);
424 } else {
425 dlog(DLOG_ALL,
426 "[XXX] %s DELETE-PACKAGE %s "
427 "(due to dependencies)\n",
428 pkg->portdir, buf);
429 }
430 free(buf);
431 buf = NULL;
432 *hasworkp = 1;
433 }
434
435 /*
436 * Handle propagated flags
437 */
438 if (pkg->flags & PKGF_ERROR) {
9c4c701f
MD
439 /*
440 * This can only happen if the ERROR has already been
441 * processed and accounted for.
442 */
8e25f19b 443 ddprintf(level, "} (ERROR - %s)\n", pkg->portdir);
8e25f19b
MD
444 } else if (*app & PKGF_NOTREADY) {
445 /*
446 * We don't set PKGF_NOTREADY in the pkg, it is strictly
447 * a transient flag propagated via build_find_leaves().
448 *
449 * Just don't add the package to the list.
9c4c701f
MD
450 *
451 * NOTE: Even if NOBUILD is set (meaning we could list it
452 * and let startbuild() finish it up as a skip, we
453 * don't process it to the list because we want to
454 * process all the dependencies, so someone doing a
455 * manual build can get more complete information and
456 * does not have to iterate each failed dependency one
457 * at a time.
8e25f19b
MD
458 */
459 ;
460 } else if (pkg->flags & PKGF_SUCCESS) {
461 ddprintf(level, "} (SUCCESS - %s)\n", pkg->portdir);
462 } else if (pkg->flags & PKGF_DUMMY) {
463 ddprintf(level, "} (DUMMY/META - SUCCESS)\n");
464 pkg->flags |= PKGF_SUCCESS;
465 *hasworkp = 1;
466 if (first) {
467 dlog(DLOG_ALL, "[XXX] %s META-ALREADY-BUILT\n",
468 pkg->portdir);
469 --BuildTotal;
470 } else {
471 dlog(DLOG_SUCC, "[XXX] %s meta-node complete\n",
472 pkg->portdir);
473 }
474 } else if (pkg->flags & PKGF_PACKAGED) {
475 /*
476 * We can just mark the pkg successful. If this is
477 * the first pass, we count this as an initial pruning
478 * pass and reduce BuildTotal.
479 */
480 ddprintf(level, "} (PACKAGED - SUCCESS)\n");
481 pkg->flags |= PKGF_SUCCESS;
482 *hasworkp = 1;
483 if (first) {
484 dlog(DLOG_ALL, "[XXX] %s ALREADY-BUILT\n",
485 pkg->portdir);
486 --BuildTotal;
487 }
488 } else {
489 /*
490 * All dependencies are successful, queue new work
491 * and indicate not-ready to the parent (since our
492 * package has to be built).
9c4c701f
MD
493 *
494 * NOTE: The NOBUILD case propagates to here as well
495 * and is ultimately handled by startbuild().
8e25f19b
MD
496 */
497 *hasworkp = 1;
9c4c701f
MD
498 if (pkg->flags & PKGF_NOBUILD)
499 ddprintf(level, "} (ADDLIST(NOBUILD) - %s)\n",
500 pkg->portdir);
501 else
502 ddprintf(level, "} (ADDLIST - %s)\n", pkg->portdir);
8e25f19b
MD
503 pkg->flags |= PKGF_BUILDLIST;
504 **build_tailp = pkg;
505 *build_tailp = &pkg->build_next;
506 *app |= PKGF_NOTREADY;
507 }
508
509 return idep_count;
510}
511
512static
513void
514build_clear_trav(pkg_t *pkg)
515{
516 pkglink_t *link;
517 pkg_t *scan;
518
519 pkg->flags &= ~PKGF_BUILDTRAV;
520 PKGLIST_FOREACH(link, &pkg->idepon_list) {
521 scan = link->pkg;
522 if (scan && (scan->flags & PKGF_BUILDTRAV))
523 build_clear_trav(scan);
524 }
525}
526
527/*
528 * Take a list of pkg ready to go, sort it, and assign it to worker
529 * slots. This routine blocks in waitbuild() until it can dispose of
530 * the entire list.
531 *
532 * WorkerMutex is held by the caller.
533 */
534static
535void
536startbuild(pkg_t **build_listp, pkg_t ***build_tailp)
537{
538 pkg_t *pkg;
539 pkg_t **idep_ary;
540 pkg_t **depi_ary;
541 int count;
542 int idep_index;
543 int depi_index;
544 int i;
545 int n;
546 worker_t *work;
547 static int IterateWorker;
548
549 /*
550 * Nothing to do
551 */
552 if (*build_listp == NULL)
553 return;
554
555 /*
556 * Sort
557 */
558 count = 0;
559 for (pkg = *build_listp; pkg; pkg = pkg->build_next)
560 ++count;
561 idep_ary = calloc(count, sizeof(pkg_t *));
562 depi_ary = calloc(count, sizeof(pkg_t *));
563
564 count = 0;
565 for (pkg = *build_listp; pkg; pkg = pkg->build_next) {
566 idep_ary[count] = pkg;
567 depi_ary[count] = pkg;
568 ++count;
569 }
570
571 /*
572 * idep_ary - sorted by #of dependencies this pkg has.
573 * depi_ary - sorted by #of other packages that depend on this pkg.
574 */
575 qsort(idep_ary, count, sizeof(pkg_t *), qsort_idep);
576 qsort(depi_ary, count, sizeof(pkg_t *), qsort_depi);
577
578 idep_index = 0;
579 depi_index = 0;
580
581 /*
582 * Half the workers build based on the highest depi count,
583 * the other half build based on the highest idep count.
584 *
585 * This is an attempt to get projects which many other projects
586 * depend on built first, but to also try to build large projects
587 * (which tend to have a lot of dependencies) earlier rather than
588 * later so the end of the bulk run doesn't inefficiently build
589 * the last few huge projects.
590 *
591 * Loop until we manage to assign slots to everyone. We do not
592 * wait for build completion.
593 *
594 * This is the point where we handle DUMMY packages (these are
595 * dummy unflavored packages which 'cover' all the flavors for
596 * a package). These are not real packages are marked SUCCESS
597 * at this time because their dependencies (the flavors) have all
598 * been built.
599 */
600 while (idep_index != count || depi_index != count) {
601 pkg_t *pkgi;
602 pkg_t *ipkg;
603
604 /*
605 * Find candidate to start sorted by depi or idep.
606 */
607 ipkg = NULL;
608 while (idep_index < count) {
609 ipkg = idep_ary[idep_index];
610 if ((ipkg->flags &
3699ee09
MD
611 (PKGF_SUCCESS | PKGF_FAILURE |
612 PKGF_ERROR | PKGF_RUNNING)) == 0) {
8e25f19b
MD
613 break;
614 }
615 ipkg = NULL;
616 ++idep_index;
617 }
618
619 pkgi = NULL;
620 while (depi_index < count) {
621 pkgi = depi_ary[depi_index];
622 if ((pkgi->flags &
3699ee09
MD
623 (PKGF_SUCCESS | PKGF_FAILURE |
624 PKGF_ERROR | PKGF_RUNNING)) == 0) {
8e25f19b
MD
625 break;
626 }
627 pkgi = NULL;
628 ++depi_index;
629 }
630
f7f25838
MD
631 /*
632 * ipkg and pkgi must either both be NULL, or both
633 * be non-NULL.
634 */
8e25f19b
MD
635 if (ipkg == NULL && pkgi == NULL)
636 break;
637 ddassert(ipkg && pkgi);
638
9c4c701f
MD
639 /*
640 * Handle the NOBUILD case right here, there's no point
641 * queueing it anywhere.
642 */
643 if (ipkg->flags & PKGF_NOBUILD) {
644 char *reason;
645
646 ipkg->flags |= PKGF_FAILURE;
647 ipkg->flags &= ~PKGF_BUILDLIST;
648
649 ++BuildSkipCount;
650 ++BuildCount;
3699ee09 651 reason = buildskipreason(NULL, ipkg);
1d6e00cd 652 dlog(DLOG_SKIP, "[XXX] %s skipped due to %s\n",
9c4c701f
MD
653 ipkg->portdir, reason);
654 free(reason);
655 continue;
656 }
657 if (pkgi->flags & PKGF_NOBUILD) {
658 char *reason;
659
660 pkgi->flags |= PKGF_FAILURE;
661 pkgi->flags &= ~PKGF_BUILDLIST;
662
663 ++BuildSkipCount;
664 ++BuildCount;
3699ee09 665 reason = buildskipreason(NULL, pkgi);
1d6e00cd 666 dlog(DLOG_SKIP, "[XXX] %s skipped due to %s\n",
9c4c701f
MD
667 pkgi->portdir, reason);
668 free(reason);
669 continue;
670 }
671
8e25f19b
MD
672 /*
673 * Block while no slots are available. waitbuild()
674 * will clean out any DONE states.
675 */
8d9409b8
MD
676 while (RunningWorkers >= DynamicMaxWorkers ||
677 RunningWorkers >= MaxWorkers - FailedWorkers) {
3699ee09 678 waitbuild(RunningWorkers, 1);
8e25f19b
MD
679 }
680
681 /*
682 * Find an available worker slot, there should be at
683 * least one.
684 */
685 for (i = 0; i < MaxWorkers; ++i) {
686 n = IterateWorker % MaxWorkers;
687 work = &WorkerAry[n];
688
689 if (work->state == WORKER_DONE ||
690 work->state == WORKER_FAILED) {
691 workercomplete(work);
692 }
693 if (work->state == WORKER_NONE ||
694 work->state == WORKER_IDLE) {
695 if (n <= MaxWorkers / 2) {
696 startworker(pkgi, work);
697 } else {
698 startworker(ipkg, work);
699 }
700 /*GuiUpdate(work);*/
701 break;
702 }
703 ++IterateWorker;
704 }
705 ddassert(i != MaxWorkers);
706 }
707 GuiSync();
708
709 /*
710 * We disposed of the whole list
711 */
712 free(idep_ary);
713 free(depi_ary);
714 *build_listp = NULL;
715 *build_tailp = build_listp;
716}
717
718typedef const pkg_t *pkg_tt;
719
720static int
721qsort_idep(const void *pkg1_arg, const void *pkg2_arg)
722{
723 const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg;
724 const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg;
725
726 return (pkg2->idep_count - pkg1->idep_count);
727}
728
729static int
730qsort_depi(const void *pkg1_arg, const void *pkg2_arg)
731{
732 const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg;
733 const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg;
734
735 return (pkg2->depi_count - pkg1->depi_count);
736}
737
738/*
739 * Frontend starts a pkg up on a worker
740 *
741 * WorkerMutex must be held.
742 */
743static void
744startworker(pkg_t *pkg, worker_t *work)
745{
746 switch(work->state) {
747 case WORKER_NONE:
748 pthread_create(&work->td, NULL, childBuilderThread, work);
749 work->state = WORKER_IDLE;
750 /* fall through */
751 case WORKER_IDLE:
752 dlog(DLOG_ALL, "[%03d] %s START\n", work->index, pkg->portdir);
753 cleanworker(work);
754 pkg->flags |= PKGF_RUNNING;
755 work->pkg = pkg;
756 pthread_cond_signal(&work->cond);
757 ++RunningWorkers;
758 /*GuiUpdate(work);*/
759 break;
760 case WORKER_PENDING:
761 case WORKER_RUNNING:
762 case WORKER_DONE:
763 case WORKER_FAILED:
8ec23ca1 764 case WORKER_FROZEN:
8e25f19b
MD
765 case WORKER_EXITING:
766 default:
767 dfatal("startworker: Unexpected state %d for worker %d",
768 work->state, work->index);
769 break;
770 }
771}
772
773static void
774cleanworker(worker_t *work)
775{
776 work->state = WORKER_PENDING;
777 work->flags = 0;
778 work->accum_error = 0;
779 work->start_time = time(NULL);
780}
781
782/*
783 * Frontend finishes up a completed pkg on a worker.
784 *
785 * If the worker is in a FAILED state we clean the pkg out but (for now)
786 * leave it in its failed state so we can debug. At this point
787 * workercomplete() will be called every once in a while on the state
788 * and we have to deal with the NULL pkg.
789 *
790 * WorkerMutex must be held.
791 */
792static void
793workercomplete(worker_t *work)
794{
795 pkg_t *pkg;
796
797 /*
798 * Steady state FAILED case.
799 */
800 if (work->state == WORKER_FAILED) {
801 if (work->pkg == NULL)
802 return;
803 }
804
805 /*
806 * Process pkg out of the worker
807 */
808 pkg = work->pkg;
809 if (pkg->flags & (PKGF_ERROR|PKGF_NOBUILD)) {
810 pkg->flags |= PKGF_FAILURE;
9c4c701f
MD
811
812 /*
813 * This NOBUILD condition XXX can occur if the package is
814 * not allowed to be built.
815 */
8e25f19b
MD
816 if (pkg->flags & PKGF_NOBUILD) {
817 char *reason;
818
819 ++BuildSkipCount;
3699ee09 820 reason = buildskipreason(NULL, pkg);
1d6e00cd 821 dlog(DLOG_SKIP, "[%03d] %s skipped due to %s\n",
8e25f19b
MD
822 work->index, pkg->portdir, reason);
823 free(reason);
824 } else {
825 ++BuildFailCount;
826 dlog(DLOG_FAIL, "[%03d] %s build-failed\n",
827 work->index, pkg->portdir);
828 }
829 } else {
830 time_t t;
831 int h;
832 int m;
833 int s;
834
835 t = time(NULL) - work->start_time;
836 s = t % 60;
837 m = t / 60 % 60;
838 h = t / 3600;
839
840 dlog(DLOG_SUCC,
841 "[%03d] %s\tbuild-succeded in %02d:%02d:%02d\n",
842 work->index, pkg->portdir, h, m, s);
843 pkg->flags |= PKGF_SUCCESS;
844 ++BuildSuccessCount;
845 }
846 ++BuildCount;
847 pkg->flags &= ~PKGF_BUILDLIST;
848 dlog(DLOG_ALL, "[%03d] %s FINISHED (%s)\n",
849 work->index,
850 pkg->portdir,
851 ((pkg->flags & PKGF_SUCCESS) ? "success" : "failure"));
852 pkg->flags &= ~PKGF_RUNNING;
853 work->pkg = NULL;
854 --RunningWorkers;
855
856 if (work->state == WORKER_FAILED) {
857 dlog(DLOG_ALL, "[%03d] XXX/XXX WORKER IS IN A FAILED STATE\n",
858 work->index);
8d9409b8 859 ++FailedWorkers;
8ec23ca1
MD
860 } else if (work->flags & WORKERF_FREEZE) {
861 dlog(DLOG_ALL, "[%03d] XXX/XXX WORKER FROZEN BY REQUEST\n",
862 work->index);
863 work->state = WORKER_FROZEN;
8e25f19b
MD
864 } else {
865 work->state = WORKER_IDLE;
866 }
867}
868
869/*
870 * Wait for one or more workers to complete.
871 *
872 * WorkerMutex must be held.
873 */
874static void
3699ee09 875waitbuild(int whilematch, int dynamicmax)
8e25f19b 876{
1645cafe 877 static time_t wblast_time;
8e25f19b
MD
878 struct timespec ts;
879 worker_t *work;
1645cafe 880 time_t t;
8e25f19b
MD
881 int i;
882
883 if (whilematch == 0)
884 whilematch = 1;
885
886 while (RunningWorkers == whilematch) {
887 for (i = 0; i < MaxWorkers; ++i) {
888 work = &WorkerAry[i];
889 if (work->state == WORKER_DONE ||
890 work->state == WORKER_FAILED) {
891 workercomplete(work);
892 } else {
893 pthread_cond_signal(&work->cond);
894 }
895 GuiUpdate(work);
896 }
897 GuiUpdateTop();
898 GuiSync();
899 if (RunningWorkers == whilematch) {
900 clock_gettime(CLOCK_REALTIME, &ts);
901 ts.tv_sec += 1;
902 ts.tv_nsec = 0;
903 pthread_cond_timedwait(&WorkerCond, &WorkerMutex, &ts);
904 }
1645cafe
MD
905
906 /*
907 * Dynamically reduce MaxWorkers based on the load. When
908 * the load exceeds 2 x ncpus we reduce the number of workers
9c4c701f 909 * up to 75% of MaxWorkers @ (5 x ncpus) load.
f7f25838
MD
910 *
911 * Dynamically reduce MaxWorkers based on swap use, starting
912 * at 10% swap and up to 75% of MaxWorkers at 40% swap.
913 *
914 * NOTE! Generally speaking this allows more workers to be
915 * configured which helps in two ways. First it allows
916 * a higher build rate for smaller packages. Second
917 * it allows dsynth to ratchet-down the number of slots
918 * when large packages are forcing the load up.
919 *
920 * A high load doesn't hurt efficiency, but swap usage
921 * due to loading the tmpfs in lots of worker slots up
922 * with tons of pkg install's (pre-reqs for a build)
923 * does. Reducing the number of worker slots has a
924 * huge beneficial effect on reducing swap use / paging.
1645cafe
MD
925 */
926 t = time(NULL);
3699ee09
MD
927 if (dynamicmax && (wblast_time == 0 ||
928 (unsigned)(t - wblast_time) >= 5)) {
929 double min_load = 1.5 * NumCores;
930 double max_load = 5.0 * NumCores;
931 double min_swap = 0.10;
932 double max_swap = 0.40;
1645cafe 933 double dload[3];
3699ee09
MD
934 double dswap;
935 int reduce1;
936 int reduce2;
937 int reduce;
938 int noswap;
1645cafe
MD
939
940 wblast_time = t;
941 getloadavg(dload, 3);
3699ee09 942 dswap = getswappct(&noswap);
1645cafe 943
3699ee09
MD
944 if (dload[0] < min_load) {
945 reduce1 = 0;
946 } else if (dload[0] <= max_load) {
947 reduce1 = 75 * (dload[0] - min_load) /
948 (max_load - min_load);
949 } else {
950 reduce1 = MaxWorkers * 75 / 100;
951 }
952
953 if (dswap < min_swap) {
954 reduce2 = 0;
955 } else if (dswap <= max_swap) {
956 reduce2 = 75 * (dswap - min_swap) /
957 (max_swap - min_swap);
958 } else {
959 reduce2 = MaxWorkers * 75 / 100;
960 }
961
962 /*
963 * Priority reduction, convert to DynamicMaxWorkers
964 */
965 if (reduce1 > reduce2)
966 reduce = reduce1;
967 else
968 reduce = reduce2;
969 reduce = MaxWorkers - reduce;
970 if (reduce < 4)
971 reduce = 4;
972 if (reduce > MaxWorkers)
973 reduce = MaxWorkers;
974
975 /*
976 * Handle slow-start
977 */
978 if (reduce > DynamicMaxWorkers + 1) {
979 reduce = DynamicMaxWorkers + 1;
980 }
981
982 /*
983 * Stop waiting if DynamicMaxWorkers is going
984 * to increase.
985 */
986 if (DynamicMaxWorkers < reduce)
987 whilematch = -1;
988
989 /*
990 * And adjust
991 */
992 if (DynamicMaxWorkers != reduce) {
64824e6c 993 dlog(DLOG_ALL,
3699ee09
MD
994 "[XXX] Load=%-6.2f(-%-2d) "
995 "Swappct=%-3.2f(-%-2d) "
996 "Adjust MaxWorkers "
997 "%2d->%-2d\n",
998 dload[0], reduce1,
999 dswap * 100.0, reduce2,
1000 DynamicMaxWorkers,
1001 reduce);
1002 DynamicMaxWorkers = reduce;
1645cafe
MD
1003 }
1004 }
8e25f19b
MD
1005 }
1006}
1007
1008
1009/*
1010 * Worker pthread (WorkerAry)
1011 *
1012 * This thread belongs to the dsynth master process and handled a worker slot.
1013 * (this_thread) -> WORKER fork/exec (WorkerPocess) -> (pty) -> sub-processes.
1014 */
1015static void *
1016childBuilderThread(void *arg)
1017{
1018 char *envary[1] = { NULL };
1019 worker_t *work = arg;
1020 wmsg_t wmsg;
1021 pkg_t *pkg;
1022 pid_t pid;
1023 int status;
1024 volatile int dowait;
1025 char slotbuf[8];
1026 char fdbuf[8];
8ec23ca1 1027 char flagsbuf[16];
8e25f19b
MD
1028
1029 pthread_mutex_lock(&WorkerMutex);
1030 while (work->terminate == 0) {
1031 dowait = 1;
1032
1033 switch(work->state) {
1034 case WORKER_IDLE:
1035 break;
1036 case WORKER_PENDING:
1037 /*
1038 * Fork the management process for the pkg operation
1039 * on this worker slot.
1040 *
1041 * This process will set up the environment, do the
1042 * mounts, will become the reaper, and will process
1043 * pipe commands and handle chroot operations.
1044 *
1045 * NOTE: If SOCK_CLOEXEC is not supported WorkerMutex
1046 * is sufficient to interlock F_SETFD FD_CLOEXEC
1047 * operations.
1048 */
1049 ddassert(work->pkg);
1050 if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC,
1051 PF_UNSPEC, work->fds)) {
1052 dfatal_errno("socketpair() during worker fork");
1053 }
8ec23ca1
MD
1054 snprintf(slotbuf, sizeof(slotbuf),
1055 "%d", work->index);
1056 snprintf(fdbuf, sizeof(fdbuf),
1057 "3");
1058 snprintf(flagsbuf, sizeof(flagsbuf),
1059 "%d", DebugStopMode);
8e25f19b
MD
1060
1061 /*
1062 * fds[0] - master
1063 * fds[1] - slave
1064 *
1065 * We pass the salve descriptor in fd 3 and close all
1066 * other descriptors for security.
1067 */
1068 pthread_mutex_unlock(&WorkerMutex);
1069 pid = vfork();
1070 if (pid == 0) {
1071 close(work->fds[0]);
1072 dup2(work->fds[1], 3);
1073 closefrom(4);
1074 fcntl(3, F_SETFD, 0);
1075 execle(DSynthExecPath, DSynthExecPath,
1076 "WORKER", slotbuf, fdbuf,
1077 work->pkg->portdir, work->pkg->pkgfile,
8ec23ca1 1078 flagsbuf,
8e25f19b
MD
1079 NULL, envary);
1080 write(2, "EXECLE FAILURE\n", 15);
1081 _exit(1);
1082 }
1083 pthread_mutex_lock(&WorkerMutex);
1084 close(work->fds[1]);
1085 work->phase = PHASE_PENDING;
1086 work->lines = 0;
1087 work->pid = pid;
1088 work->state = WORKER_RUNNING;
1089 /* fall through */
1090 case WORKER_RUNNING:
1091 /*
1092 * Poll for status updates, if NULL is returned
1093 * and status is non-zero, the communications link
1094 * failed unexpectedly.
1095 */
1096 pkg = work->pkg;
1097 pthread_mutex_unlock(&WorkerMutex);
1098 status = ipcreadmsg(work->fds[0], &wmsg);
1099 pthread_mutex_lock(&WorkerMutex);
1100
1101 if (status == 0) {
1102 /*
1103 * Normal message, can include normal
1104 * termination which changes us over
1105 * to another state.
1106 */
1107 dowait = 0;
1108 switch(wmsg.cmd) {
1109 case WMSG_CMD_INSTALL_PKGS:
1110 wmsg.cmd = WMSG_RES_INSTALL_PKGS;
1111 wmsg.status = childInstallPkgDeps(work);
1112 pthread_mutex_unlock(&WorkerMutex);
1113 ipcwritemsg(work->fds[0], &wmsg);
1114 pthread_mutex_lock(&WorkerMutex);
1115 break;
1116 case WMSG_CMD_STATUS_UPDATE:
1117 work->phase = wmsg.phase;
1118 work->lines = wmsg.lines;
1119 break;
1120 case WMSG_CMD_SUCCESS:
1121 work->flags |= WORKERF_SUCCESS;
1122 break;
1123 case WMSG_CMD_FAILURE:
1124 work->flags |= WORKERF_FAILURE;
1125 break;
8ec23ca1
MD
1126 case WMSG_CMD_FREEZEWORKER:
1127 work->flags |= WORKERF_FREEZE;
1128 break;
8e25f19b
MD
1129 default:
1130 break;
1131 }
1132 GuiUpdate(work);
1133 GuiSync();
1134 } else {
1135 close(work->fds[0]);
1136 pthread_mutex_unlock(&WorkerMutex);
1137 while (waitpid(work->pid, &status, 0) < 0 &&
1138 errno == EINTR) {
1139 ;
1140 }
1141 pthread_mutex_lock(&WorkerMutex);
1142
1143 if (work->flags & WORKERF_SUCCESS) {
1144 pkg->flags |= PKGF_SUCCESS;
1145 work->state = WORKER_DONE;
1146 } else if (work->flags & WORKERF_FAILURE) {
1147 pkg->flags |= PKGF_FAILURE;
1148 work->state = WORKER_DONE;
1149 } else {
1150 pkg->flags |= PKGF_FAILURE;
1151 work->state = WORKER_FAILED;
1152 }
1153 work->flags |= WORKERF_STATUS_UPDATE;
1154 pthread_cond_signal(&WorkerCond);
1155 }
1156 break;
1157 case WORKER_DONE:
1158 /*
1159 * pkg remains attached until frontend processes the
1160 * completion. The frontend will then set the state
1161 * back to idle.
1162 */
1163 break;
1164 case WORKER_FAILED:
1165 /*
1166 * A worker failure means that the worker did not
1167 * send us a WMSG_CMD_SUCCESS or WMSG_CMD_FAILURE
1168 * ipc before terminating.
1169 *
1170 * We just sit in this state until the front-end
1171 * does something about it.
1172 */
1173 break;
1174 default:
1175 dfatal("worker: Unexpected state %d for worker %d",
1176 work->state, work->index);
1177 /* NOT REACHED */
1178 break;
1179 }
1180
1181 /*
1182 * The dsynth frontend will poll us approximately once
1183 * a second (its variable).
1184 */
1185 if (dowait)
1186 pthread_cond_wait(&work->cond, &WorkerMutex);
1187 }
1188
1189 /*
1190 * Scrap the comm socket if running, this should cause the worker
1191 * process to kill its sub-programs and cleanup.
1192 */
1193 if (work->state == WORKER_RUNNING) {
1194 pthread_mutex_unlock(&WorkerMutex);
1195 close(work->fds[0]);
1196 while (waitpid(work->pid, &status, 0) < 0 &&
1197 errno == EINTR);
1198 pthread_mutex_lock(&WorkerMutex);
1199 }
1200
1201 /*
1202 * Final handshake
1203 */
1204 work->state = WORKER_EXITING;
1205 pthread_cond_signal(&WorkerCond);
1206 pthread_mutex_unlock(&WorkerMutex);
1207
1208 return NULL;
1209}
1210
1211/*
1212 * Install all the binary packages (we have already built them) that
1213 * the current work package depends on, without duplicates, in a script
1214 * which will be run from within the specified work jail.
1215 *
1216 * Locked by WorkerMutex (global)
1217 */
1218static int
1219childInstallPkgDeps(worker_t *work)
1220{
1221 char *buf;
1222 FILE *fp;
1223
1224 if (PKGLIST_EMPTY(&work->pkg->idepon_list))
1225 return 0;
1226
1227 asprintf(&buf, "%s/tmp/dsynth_install_pkgs", work->basedir);
1228 fp = fopen(buf, "w");
1229 ddassert(fp != NULL);
1230 fprintf(fp, "#!/bin/sh\n");
1231 fprintf(fp, "#\n");
1232 fchmod(fileno(fp), 0755);
1233
1234 childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 0);
1235 childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 1);
1236 fprintf(fp, "\nexit 0\n");
1237 fclose(fp);
8d9409b8 1238 free(buf);
8e25f19b
MD
1239
1240 return 1;
1241}
1242
1243static void
1244childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list, int undoit)
1245{
1246 pkglink_t *link;
1247 pkg_t *pkg;
1248
1249 PKGLIST_FOREACH(link, list) {
1250 pkg = link->pkg;
1251
1252 if (undoit) {
1253 if (pkg->dsynth_install_flg == 1) {
1254 pkg->dsynth_install_flg = 0;
1255 childInstallPkgDeps_recurse(fp,
1256 &pkg->idepon_list,
1257 undoit);
1258 }
1259 continue;
1260 }
1261 if (pkg->dsynth_install_flg) {
1262 if (DebugOpt && pkg->pkgfile) {
1263 fprintf(fp, "echo 'AlreadyHave %s'\n",
1264 pkg->pkgfile);
1265 }
1266 continue;
1267 }
1268
1269 childInstallPkgDeps_recurse(fp, &pkg->idepon_list, undoit);
1270 if (pkg->dsynth_install_flg)
1271 continue;
1272 pkg->dsynth_install_flg = 1;
1273
1274 /*
1275 * If this is a dummy node with no package, the originator
1276 * is requesting a flavored package. We select the default
1277 * flavor which we presume is the first one.
1278 */
1279 if (pkg->pkgfile == NULL && (pkg->flags & PKGF_DUMMY)) {
1280 pkg_t *spkg = pkg->idepon_list.next->pkg;
1281
1282 if (spkg) {
1283 pkg = spkg;
1284 fprintf(fp,
1285 "echo 'DUMMY use %s (%p)'\n",
1286 pkg->portdir, pkg->pkgfile);
1287 } else {
1288 fprintf(fp,
1289 "echo 'CANNOT FIND DEFAULT FLAVOR "
1290 "FOR %s'\n",
1291 pkg->portdir);
1292 }
1293 }
1294
1295 /*
1296 * Generate package installation command
1297 */
1298 if (pkg->pkgfile) {
1299 fprintf(fp, "echo 'Installing /packages/All/%s'\n",
1300 pkg->pkgfile);
1301 fprintf(fp, "pkg install -q -y /packages/All/%s "
1302 "|| exit 1\n",
1303 pkg->pkgfile);
1304 } else {
1305 fprintf(fp, "echo 'CANNOT FIND PKG FOR %s'\n",
1306 pkg->portdir);
1307 }
1308 }
1309}
1310
1311/*
1312 * Worker process interactions.
1313 *
1314 * The worker process is responsible for managing the build of a single
1315 * package. It is exec'd by the master dsynth and only loads the
1316 * configuration.
1317 *
1318 * This process does not run in the chroot. It will become the reaper for
1319 * all sub-processes and it will enter the chroot to execute various phases.
1320 * It catches SIGINTR, SIGHUP, and SIGPIPE and will iterate, terminate, and
1321 * reap all sub-process upon kill or exit.
1322 *
1323 * The command line forwarded to this function is:
1324 *
1325 * WORKER slot# socketfd portdir/subdir
1326 *
1327 * TERM=dumb
1328 * USER=root
1329 * HOME=/root
1330 * LANG=C
1331 * SSL_NO_VERIFY_PEER=1
1332 * USE_PACKAGE_DEPENDS_ONLY=1 (exec_phase_depends)
1333 * PORTSDIR=/xports
64824e6c
MD
1334 * PORT_DBDIR=/options For ports options
1335 * PACKAGE_BUILDING=yes Don't build packages that aren't legally
1336 * buildable for a binary repo.
8e25f19b
MD
1337 * PKG_DBDIR=/var/db/pkg
1338 * PKG_CACHEDIR=/var/cache/pkg
64824e6c 1339 * PKG_CREATE_VERBOSE=yes Ensure periodic output during packaging
8e25f19b
MD
1340 * (custom environment)
1341 * PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
1342 * UNAME_s=DragonFly (example)
1343 * UNAME_v=DragonFly 5.7-SYNTH (example)
1344 * UNAME_p=x86_64 (example)
1345 * UNAME_m=x86_64 (example)
1346 * UNAME_r=5.7-SYNTH (example)
1347 * NO_DEPENDS=yes (exec_phase)
1348 * DISTDIR=/distfiles
1349 * WRKDIRPREFIX=/construction
1350 * BATCH=yes
1351 * MAKE_JOBS_NUMBER=n
1352 *
1353 * SETUP:
1354 * ldconfig -R
1355 * /usr/local/sbin/pkg-static install /packages/All/<the pkg pkg>
1356 * /usr/local/sbin/pkg-static install /packages/All/<pkg>
1357 * (for all dependencies)
1358 *
1359 * PHASES: make -C path FLAVOR=flavor <phase>
1360 * check-sanity
1361 * pkg-depends
1362 * fetch-depends
1363 * fetch
1364 * checksum
1365 * extract-depends
1366 * extract
1367 * patch-depends
1368 * patch
1369 * build-depends
1370 * lib-depends
1371 * configure
1372 * build
1373 * run-depends
1374 * stage
f7f25838 1375 * test (skipped)
8e25f19b
MD
1376 * check-plist
1377 * package e.g. /construction/lang/perl5.28/pkg/perl5-5.28.2.txz
f7f25838
MD
1378 * install-mtree (skipped)
1379 * install (skipped)
1380 * deinstall (skipped)
8e25f19b 1381 */
1d6e00cd 1382
8e25f19b
MD
1383void
1384WorkerProcess(int ac, char **av)
1385{
1386 wmsg_t wmsg;
1387 int fd;
1388 int slot;
1389 int tmpfd;
1390 int pkgpkg = 0;
8ec23ca1 1391 int status;
8e25f19b
MD
1392 int len;
1393 int do_install_phase;
1394 char *portdir;
1395 char *pkgfile;
1396 char *flavor;
1397 char *buf;
1398 worker_t *work;
1399 pkg_t pkg;
1400 buildenv_t *benv;
1401
1402 /*
1403 * Parse arguments
1404 */
8ec23ca1 1405 if (ac != 6) {
1645cafe 1406 dlog(DLOG_ALL, "WORKER PROCESS %d- bad arguments\n", getpid());
8e25f19b
MD
1407 exit(1);
1408 }
1409 slot = strtol(av[1], NULL, 0);
1410 fd = strtol(av[2], NULL, 0); /* master<->slave messaging */
1411 portdir = av[3];
1412 pkgfile = av[4];
1413 flavor = strchr(portdir, '@');
1414 if (flavor)
1415 *flavor++ = 0;
8ec23ca1
MD
1416 DebugStopMode = strtol(av[5], NULL, 0);
1417
8e25f19b
MD
1418 bzero(&wmsg, sizeof(wmsg));
1419
1420 setproctitle("WORKER [%02d] STARTUP %s", slot, portdir);
1421
1422 if (strcmp(portdir, "ports-mgmt/pkg") == 0)
1423 pkgpkg = 1;
1424
1425 signal(SIGTERM, phaseTerminateSignal);
1426 signal(SIGINT, phaseTerminateSignal);
1427 signal(SIGHUP, phaseTerminateSignal);
1428
1429 /*
1430 * Set up the environment
1431 */
1432 setenv("TERM", "dumb", 1);
1433 setenv("USER", "root", 1);
1434 setenv("HOME", "/root", 1);
1435 setenv("LANG", "C", 1);
1436 setenv("SSL_NO_VERIFY_PEER", "1", 1);
1437 setenv("USE_PACKAGE_DEPENDS_ONLY", "1", 1);
1438 setenv("PORTSDIR", "/xports", 1);
64824e6c 1439 setenv("PORT_DBDIR", "/options", 1);
8e25f19b
MD
1440 setenv("PKG_DBDIR", "/var/db/pkg", 1);
1441 setenv("PKG_CACHEDIR", "/var/cache/pkg", 1);
667fb2cb
MD
1442 setenv("PKG_SUFX", USE_PKG_SUFX, 1);
1443
8e25f19b 1444
8ec23ca1
MD
1445#if 0
1446 setenv("_PERL5_FROM_BIN", "5.28.2", 1);
1447 setenv("OPSYS", OperatingSystemName, 1);
1448#endif
1449#if 0
1450 setenv("DFLYVERSION", "5.7.0", 1);
1451 setenv("OSVERSION", "9999999", 1);
1452#endif
1453
8e25f19b
MD
1454 setenv("PATH",
1455 "/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin",
1456 1);
1457
1458 setenv("UNAME_s", OperatingSystemName, 1);
1459 setenv("UNAME_v", VersionName, 1);
1460 setenv("UNAME_p", ArchitectureName, 1);
1461 setenv("UNAME_m", MachineName, 1);
1462 setenv("UNAME_r", ReleaseName, 1);
1463
1464 setenv("NO_DEPENDS", "yes", 1);
1465 setenv("DISTDIR", "/distfiles", 1);
1466 setenv("WRKDIRPREFIX", "/construction", 1);
1467 setenv("BATCH", "yes", 1);
1468
64824e6c
MD
1469 /*
1470 * Special consideration
1471 *
1472 * PACKAGE_BUILDING - Disallow packaging ports which do not allow
1473 * for binary distribution.
1474 *
1475 * PKG_CREATE_VERBOSE - Ensure periodic output during the packaging
1476 * process to avoid a watchdog timeout.
1477 *
1478 */
1479 setenv("PACKAGE_BUILDING", "yes", 1);
1480 setenv("PKG_CREATE_VERBOSE", "yes", 1);
1481
8e25f19b
MD
1482 asprintf(&buf, "%d", MaxJobs);
1483 setenv("MAKE_JOBS_NUMBER", buf, 1);
1484 free(buf);
1485
1486 if (flavor)
1487 setenv("FLAVOR", flavor, 1);
1488
1489 /*
1490 * Load environment from profile
1491 */
1492 for (benv = BuildEnv; benv; benv = benv->next) {
1493 if (DebugOpt >= 2) {
1494 dlog(DLOG_ALL, "[%03d] ENV %s=%s\n",
1495 slot, benv->label, benv->data);
1496 }
1497 setenv(benv->label, benv->data, 1);
1498 }
1499
1500 /*
1501 * Become the reaper
1502 */
1503 if (procctl(P_PID, getpid(), PROC_REAP_ACQUIRE, NULL) < 0)
1504 dfatal_errno("procctl() - Cannot become reaper");
1505
1506 /*
1507 * Initialize a worker structure
1508 */
1509 DoInitBuild(slot);
1510
1511 bzero(&pkg, sizeof(pkg));
1512 pkg.portdir = portdir; /* sans flavor */
1513 pkg.pkgfile = pkgfile;
1514 if (strchr(portdir, '/'))
1515 len = strchr(portdir, '/') - portdir;
1516 else
1517 len = 0;
1518
1519 /*
1520 * Setup the logfile
1521 */
1522 asprintf(&pkg.logfile,
1523 "%s/%*.*s___%s%s%s.log",
1524 LogsPath, len, len, portdir,
1525 ((portdir[len] == '/') ? portdir + len + 1 : portdir + len),
1526 (flavor ? "@" : ""),
1527 (flavor ? flavor : ""));
1528 tmpfd = open(pkg.logfile, O_RDWR|O_CREAT|O_TRUNC, 0666);
1529 if (tmpfd >= 0) {
1530 dlog(DLOG_ALL, "[%03d] %s LOGFILE %s\n",
1531 slot, pkg.portdir, pkg.logfile);
1532 close(tmpfd);
1533 } else {
1534 dlog(DLOG_ALL, "[%03d] LOGFILE %s (create failed)\n",
1535 slot, pkg.logfile);
1536 }
1537
1538 /*
1539 * Setup the work structure. Because this is an exec'd sub-process,
1540 * there is only one work structure.
1541 */
1542 work = &WorkerAry[0];
1543 work->flavor = flavor;
1544 work->fds[0] = fd;
1545 work->pkg = &pkg;
1546 work->start_time = time(NULL);
1547
1548 /*
1549 * Do mounts and start the phases
1550 */
1551 SigWork = work;
1552 setproctitle("WORKER [%02d] MOUNTS %s", slot, portdir);
1553 DoWorkerMounts(work);
1554
1555 wmsg.cmd = WMSG_CMD_INSTALL_PKGS;
1556 ipcwritemsg(fd, &wmsg);
1557 status = ipcreadmsg(fd, &wmsg);
1558 if (status < 0 || wmsg.cmd != WMSG_RES_INSTALL_PKGS)
1559 dfatal("pkg installation handshake failed");
1560 do_install_phase = wmsg.status;
1561
1562 wmsg.cmd = WMSG_CMD_STATUS_UPDATE;
1563 wmsg.phase = PHASE_INSTALL_PKGS;
1564 wmsg.lines = 0;
1565
1566 status = ipcwritemsg(fd, &wmsg);
1567
1568 if (pkgpkg) {
1569 dophase(work, &wmsg,
1570 WDOG5, PHASE_PACKAGE, "package");
1571 } else {
1572 if (do_install_phase) {
1573 dophase(work, &wmsg,
1574 WDOG4, PHASE_INSTALL_PKGS, "setup");
1575 }
1576 dophase(work, &wmsg,
1577 WDOG2, PHASE_CHECK_SANITY, "check-sanity");
1578 dophase(work, &wmsg,
1579 WDOG2, PHASE_PKG_DEPENDS, "pkg-depends");
1580 dophase(work, &wmsg,
1581 WDOG7, PHASE_FETCH_DEPENDS, "fetch-depends");
1582 dophase(work, &wmsg,
1583 WDOG7, PHASE_FETCH, "fetch");
1584 dophase(work, &wmsg,
1585 WDOG2, PHASE_CHECKSUM, "checksum");
1586 dophase(work, &wmsg,
1587 WDOG3, PHASE_EXTRACT_DEPENDS, "extract-depends");
1588 dophase(work, &wmsg,
1589 WDOG3, PHASE_EXTRACT, "extract");
1590 dophase(work, &wmsg,
1591 WDOG2, PHASE_PATCH_DEPENDS, "patch-depends");
1592 dophase(work, &wmsg,
1593 WDOG2, PHASE_PATCH, "patch");
1594 dophase(work, &wmsg,
1595 WDOG5, PHASE_BUILD_DEPENDS, "build-depends");
1596 dophase(work, &wmsg,
1597 WDOG5, PHASE_LIB_DEPENDS, "lib-depends");
1598 dophase(work, &wmsg,
1599 WDOG3, PHASE_CONFIGURE, "configure");
1600 dophase(work, &wmsg,
1601 WDOG9, PHASE_BUILD, "build");
1602 dophase(work, &wmsg,
1603 WDOG5, PHASE_RUN_DEPENDS, "run-depends");
1604 dophase(work, &wmsg,
1605 WDOG5, PHASE_STAGE, "stage");
1606#if 0
1607 dophase(work, &wmsg,
1608 WDOG5, PHASE_TEST, "test");
1609#endif
1610 dophase(work, &wmsg,
1611 WDOG1, PHASE_CHECK_PLIST, "check-plist");
1612 dophase(work, &wmsg,
1613 WDOG5, PHASE_PACKAGE, "package");
1614#if 0
1615 dophase(work, &wmsg,
1616 WDOG5, PHASE_INSTALL_MTREE, "install-mtree");
1617 dophase(work, &wmsg,
1618 WDOG5, PHASE_INSTALL, "install");
1619 dophase(work, &wmsg,
1620 WDOG5, PHASE_DEINSTALL, "deinstall");
1621#endif
1622 }
1623
1d6e00cd
MD
1624 if (MasterPtyFd >= 0) {
1625 close(MasterPtyFd);
1626 MasterPtyFd = -1;
1627 }
1628
8e25f19b
MD
1629 setproctitle("WORKER [%02d] CLEANUP %s", slot, portdir);
1630
1631 /*
1632 * Copy the package to the repo.
1633 */
1634 if (work->accum_error == 0) {
1635 char *b1;
1636 char *b2;
1637
1638 asprintf(&b1, "%s/construction/%s/pkg/%s",
1639 work->basedir, pkg.portdir, pkg.pkgfile);
1640 asprintf(&b2, "%s/%s", RepositoryPath, pkg.pkgfile);
1641 if (copyfile(b1, b2)) {
1642 ++work->accum_error;
1643 dlog(DLOG_ALL, "[%03d] %s Unable to copy %s to %s\n",
1644 work->index, pkg.portdir, b1, b2);
1645 }
1646 free(b1);
1647 free(b2);
1648 }
1649
8ec23ca1
MD
1650 /*
1651 * Unmount, unless we are in DebugStopMode.
1652 */
1653 if (DebugStopMode == 0)
1654 DoWorkerUnmounts(work);
8e25f19b 1655
8ec23ca1
MD
1656 /*
1657 * Send completion status to master dsynth worker thread.
1658 */
8e25f19b
MD
1659 if (work->accum_error) {
1660 wmsg.cmd = WMSG_CMD_FAILURE;
1661 } else {
1662 wmsg.cmd = WMSG_CMD_SUCCESS;
1663 }
8ec23ca1
MD
1664 ipcwritemsg(fd, &wmsg);
1665 if (DebugStopMode) {
1666 wmsg.cmd = WMSG_CMD_FREEZEWORKER;
1667 ipcwritemsg(fd, &wmsg);
1668 }
8e25f19b
MD
1669}
1670
1671static void
1672dophase(worker_t *work, wmsg_t *wmsg, int wdog, int phaseid, const char *phase)
1673{
1674 pkg_t *pkg = work->pkg;
1675 char buf[1024];
1676 pid_t pid;
8e25f19b 1677 int status;
7880dcf7
MD
1678 int ms;
1679 pid_t wpid;
1680 int wpid_reaped;
8e25f19b
MD
1681 int fdlog;
1682 time_t start_time;
1683 time_t last_time;
1684 time_t next_time;
1685 time_t wdog_time;
1686 FILE *fp;
1687
1688 if (work->accum_error)
1689 return;
1690 setproctitle("WORKER [%02d] %8.8s %s",
1691 work->index, phase, pkg->portdir);
1692 wmsg->phase = phaseid;
1693 if (ipcwritemsg(work->fds[0], wmsg) < 0) {
1694 dlog(DLOG_ALL, "[%03d] %s Lost Communication with dsynth, "
1695 "aborting worker\n",
1696 work->index, pkg->portdir);
1697 ++work->accum_error;
1698 return;
1699 }
1700
1701 /*
1702 * Execute the port make command in chroot on a pty
1703 */
1704 fflush(stdout);
1705 fflush(stderr);
1d6e00cd
MD
1706 if (MasterPtyFd >= 0) {
1707 int slavefd;
1708
1709 slavefd = open(ptsname(MasterPtyFd), O_RDWR);
1710 dassert_errno(slavefd >= 0, "Cannot open slave pty");
1711 pid = fork();
1712 if (pid == 0) {
1713 login_tty(slavefd);
1714 } else {
1715 close(slavefd);
1716 }
1717 } else {
1718 pid = forkpty(&MasterPtyFd, NULL, NULL, NULL);
1719 }
1720
1721 if (pid == 0) {
8e25f19b
MD
1722 closefrom(3);
1723 if (chdir(work->basedir) < 0)
1724 dfatal_errno("chdir in phase initialization");
1725 if (chroot(work->basedir) < 0)
1726 dfatal_errno("chroot in phase initialization");
1727
9c4c701f
MD
1728 /*
1729 * We have a choice here on how to handle stdin (fd 0).
1730 * We can leave it connected to the pty in which case
1731 * the build will just block if it tries to ask a
1732 * question (and the watchdog will kill it, eventually),
1733 * or we can try to EOF the pty, or we can attach /dev/null
1734 * to descriptor 0.
1735 */
1736 if (NullStdinOpt) {
1737 int fd;
1738
1739 fd = open("/dev/null", O_RDWR);
1740 dassert_errno(fd >= 0, "cannot open /dev/null");
1741 if (fd != 0) {
1742 dup2(fd, 0);
1743 close(fd);
1744 }
1745 }
1746
8e25f19b
MD
1747 /*
1748 * Execute the appropriate command.
1749 */
1750 switch(phaseid) {
1751 case PHASE_INSTALL_PKGS:
1752 snprintf(buf, sizeof(buf), "/tmp/dsynth_install_pkgs");
1753 execl(buf, buf, NULL);
1754 break;
1755 default:
1756 snprintf(buf, sizeof(buf), "/xports/%s", pkg->portdir);
1757 execl(MAKE_BINARY, MAKE_BINARY, "-C", buf, phase, NULL);
1758 break;
1759 }
1760 _exit(1);
1761 }
7880dcf7 1762 fcntl(MasterPtyFd, F_SETFL, O_NONBLOCK);
8e25f19b 1763
1d6e00cd 1764 if (pid < 0) {
6a3a20b1
MD
1765 dlog(DLOG_ALL, "[%03d] %s Fork Failed: %s\n",
1766 work->index, pkg->logfile, strerror(errno));
1d6e00cd
MD
1767 ++work->accum_error;
1768 return;
1769 }
1770
8e25f19b
MD
1771 SigPid = pid;
1772
1773 fdlog = open(pkg->logfile, O_RDWR|O_CREAT|O_APPEND, 0644);
1774 if (fdlog < 0) {
1775 dlog(DLOG_ALL, "[%03d] %s Cannot open logfile '%s': %s\n",
1776 work->index, pkg->portdir,
1777 pkg->logfile, strerror(errno));
1778 }
1779 start_time = time(NULL);
1780 last_time = start_time;
1781 wdog_time = start_time;
7880dcf7 1782 wpid_reaped = 0;
8e25f19b
MD
1783
1784 status = 0;
1785 for (;;) {
7880dcf7
MD
1786 ms = mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time);
1787 if (ms == MPTY_FAILED) {
1788 dlog(DLOG_ALL,
1789 "[%03d] %s lost pty in phase %s, terminating\n",
1790 work->index, pkg->portdir, phase);
1791 break;
8e25f19b 1792 }
7880dcf7
MD
1793 if (ms == MPTY_EOF)
1794 break;
8e25f19b
MD
1795
1796 /*
1797 * Generally speaking update status once a second.
1798 * This also allows us to detect if the management
1799 * dsynth process has gone away.
1800 */
1801 next_time = time(NULL);
1802 if (next_time != last_time) {
1803 double dload[3];
1804 double dv;
1805 int wdog_scaled;
1806
1807 /*
1808 * Send status update to the worker management thread
1809 * in the master dsynth process. Remember, *WE* are
1810 * the worker management process sub-fork.
1811 */
1812 if (ipcwritemsg(work->fds[0], wmsg) < 0)
1813 break;
1814 last_time = next_time;
1815
1816 /*
1817 * Watchdog scaling
1818 */
1819 getloadavg(dload, 3);
1820 dv = dload[2] / NumCores;
1821 if (dv < (double)NumCores) {
1822 wdog_scaled = wdog;
1823 } else {
1824 if (dv > 4.0 * NumCores)
1825 dv = 4.0 * NumCores;
1826 wdog_scaled = wdog * dv / NumCores;
1827 }
1828
1829 /*
1830 * Watchdog
1831 */
1832 if (next_time - wdog_time >= wdog_scaled * 60) {
1833 snprintf(buf, sizeof(buf),
1834 "\n--------\n"
1835 "WATCHDOG TIMEOUT FOR %s in %s "
1836 "after %d minutes\n"
1837 "Killing pid %d\n"
1838 "--------\n",
1839 pkg->portdir, phase, wdog_scaled, pid);
1840 if (fdlog >= 0)
1841 write(fdlog, buf, strlen(buf));
1842 dlog(DLOG_ALL,
1843 "[%03d] WATCHDOG TIMEOUT FOR %s in %s "
1844 "after %d minutes (%d min scaled)\n",
1845 work->index, pkg->portdir, phase,
1846 wdog, wdog_scaled);
1847 kill(pid, SIGKILL);
1848 ++work->accum_error;
1849 break;
1850 }
1851 }
1852
1853 /*
1854 * Check process exit. Normally the pty will EOF
1855 * but if background processes remain we need to
1856 * check here to see if our primary exec is done,
1857 * so we can break out and reap those processes.
e86766ee
MD
1858 *
1859 * Generally reap any other processes we have inherited
1860 * while we are here.
8e25f19b 1861 */
e86766ee
MD
1862 do {
1863 wpid = wait3(&status, WNOHANG, NULL);
1864 } while (wpid > 0 && wpid != pid);
7880dcf7
MD
1865 if (wpid == pid && WIFEXITED(status)) {
1866 wpid_reaped = 1;
8e25f19b 1867 break;
7880dcf7 1868 }
8e25f19b 1869 }
8e25f19b
MD
1870
1871 next_time = time(NULL);
1872
1873 setproctitle("WORKER [%02d] EXITREAP %s",
1874 work->index, pkg->portdir);
7880dcf7
MD
1875
1876 /*
1877 * We usually get here due to a mpty EOF, but not always as there
1878 * could be persistent processes still holding the slave. Finish
1879 * up getting the exit status for the main process we are waiting
1880 * on and clean out any data left on the MasterPtyFd (as it could
1881 * be blocking the exit).
1882 */
1883 while (wpid_reaped == 0) {
1884 (void)mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time);
1885 wpid = waitpid(pid, &status, WNOHANG);
1886 if (wpid == pid && WIFEXITED(status)) {
1887 wpid_reaped = 1;
1888 break;
1889 }
1890 if (wpid < 0 && errno != EINTR) {
1891 break;
8e25f19b
MD
1892 }
1893
7880dcf7
MD
1894 /*
1895 * Safety. The normal phase waits until the fork/exec'd
1896 * pid finishes, causing a pty EOF on exit (the slave
1897 * descriptor is closed by the kernel on exit so the
1898 * process should already have exited).
1899 *
1900 * However, it is also possible to get here if the pty fails
1901 * for some reason. In this case, make sure that the process
1902 * is killed.
1903 */
1904 kill(pid, SIGKILL);
8e25f19b
MD
1905 }
1906
7880dcf7
MD
1907 /*
1908 * Clean out anything left on the pty but don't wait around
1909 * because there could be background processes preventing the
1910 * slave side from closing.
1911 */
1912 while (mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time) == MPTY_DATA)
1913 ;
1914
1915 /*
1916 * Report on the exit condition. If the pid was somehow lost
1917 * (probably due to someone gdb'ing the process), assume an error.
1918 */
1919 if (wpid_reaped) {
1920 if (WEXITSTATUS(status)) {
1921 dlog(DLOG_ALL, "[%03d] %s Build phase '%s' "
1922 "failed exit %d\n",
1923 work->index, pkg->portdir, phase,
1924 WEXITSTATUS(status));
1925 ++work->accum_error;
1926 }
1927 } else {
1928 dlog(DLOG_ALL, "[%03d] %s Build phase '%s' failed - lost pid\n",
8e25f19b
MD
1929 work->index, pkg->portdir, phase);
1930 ++work->accum_error;
1931 }
1932
1933 /*
1934 * Kill any processes still running (sometimes processes end up in
1935 * the background during a dports build), and clean up any other
1936 * children that we have inherited.
1937 */
1938 phaseReapAll();
1939
1940 /*
1941 * Update log
1942 */
1943 if (fdlog >= 0) {
1944 struct stat st;
1945 int h;
1946 int m;
1947 int s;
1948
1949 last_time = next_time - start_time;
1950 s = last_time % 60;
1951 m = last_time / 60 % 60;
1952 h = last_time / 3600;
1953
1954 fp = fdopen(fdlog, "a");
1955 if (fp == NULL) {
1956 dlog(DLOG_ALL, "[%03d] %s Cannot fdopen fdlog: %s %d\n",
1957 work->index, pkg->portdir,
1958 strerror(errno), fstat(fdlog, &st));
1959 close(fdlog);
1960 goto skip;
1961 }
1962
1963 fprintf(fp, "--------\n");
1964 if (work->accum_error) {
1965 fprintf(fp, "PHASE %s FAILED %02d:%02d:%02d\n",
1966 phase, h, m, s);
1967 } else {
1968 fprintf(fp, "PHASE %s SUCCEEDED %02d:%02d:%02d\n",
1969 phase, h, m, s);
1970 }
1971 last_time = next_time - work->start_time;
1972 s = last_time % 60;
1973 m = last_time / 60 % 60;
1974 h = last_time / 3600;
1975 if (phaseid == PHASE_PACKAGE) {
1976 fprintf(fp, "TOTAL TIME %02d:%02d:%02d\n", h, m, s);
1977 }
1978 fprintf(fp, "--------\n");
1979 fclose(fp);
1980skip:
1981 ;
1982 }
1983
1984}
1985
1986static void
1987phaseReapAll(void)
1988{
1989 struct reaper_status rs;
1990 int status;
1991
1992 while (procctl(P_PID, getpid(), PROC_REAP_STATUS, &rs) == 0) {
1993 if ((rs.flags & PROC_REAP_ACQUIRE) == 0)
1994 break;
1995 if (rs.pid_head < 0)
1996 break;
1997 if (kill(rs.pid_head, SIGKILL) == 0) {
1998 while (waitpid(rs.pid_head, &status, 0) < 0)
1999 ;
2000 }
2001 }
2002 while (wait3(&status, 0, NULL) > 0)
2003 ;
2004}
2005
2006static void
2007phaseTerminateSignal(int sig __unused)
2008{
1d6e00cd
MD
2009 if (MasterPtyFd >= 0)
2010 close(MasterPtyFd);
8e25f19b
MD
2011 if (SigPid > 1)
2012 kill(SigPid, SIGKILL);
2013 phaseReapAll();
2014 if (SigWork)
2015 DoWorkerUnmounts(SigWork);
2016 exit(1);
2017}
2018
2019static
2020char *
3699ee09 2021buildskipreason(pkglink_t *parent, pkg_t *pkg)
8e25f19b
MD
2022{
2023 pkglink_t *link;
2024 pkg_t *scan;
2025 char *reason = NULL;
3699ee09 2026 char *ptr;
8e25f19b
MD
2027 size_t tot;
2028 size_t len;
3699ee09 2029 pkglink_t stack;
8e25f19b
MD
2030
2031 tot = 0;
2032 PKGLIST_FOREACH(link, &pkg->idepon_list) {
2033 scan = link->pkg;
2034 if (scan == NULL)
2035 continue;
2036 if ((scan->flags & (PKGF_ERROR | PKGF_NOBUILD)) == 0)
2037 continue;
3699ee09
MD
2038 if (scan->flags & PKGF_NOBUILD) {
2039 stack.pkg = scan;
2040 stack.next = parent;
2041 ptr = buildskipreason(&stack, scan);
2042 len = strlen(scan->portdir) + strlen(ptr) + 8;
2043 reason = realloc(reason, tot + len);
2044 snprintf(reason + tot, len, "%s->%s",
2045 scan->portdir, ptr);
2046 free(ptr);
2047 } else {
2048 len = strlen(scan->portdir) + 8;
2049 reason = realloc(reason, tot + len);
2050 snprintf(reason + tot, len, "%s", scan->portdir);
2051 }
2052
2053 /*
2054 * Don't try to print the entire graph
2055 */
2056 if (parent)
2057 break;
8e25f19b 2058 tot += strlen(reason + tot);
3699ee09
MD
2059 reason[tot++] = ' ';
2060 reason[tot] = 0;
8e25f19b
MD
2061 }
2062 return (reason);
2063}
2064
7880dcf7
MD
2065/*
2066 * The master ptyfd is in non-blocking mode. Drain up to 1024 bytes
2067 * and update wmsg->lines and *wdog_timep as appropriate.
2068 *
2069 * This function will poll, stalling up to 1 second.
2070 */
2071static int
2072mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg, time_t *wdog_timep)
2073{
2074 struct pollfd pfd;
2075 char buf[1024];
2076 ssize_t r;
2077
2078 pfd.fd = ptyfd;
2079 pfd.events = POLLIN;
2080 pfd.revents = 0;
2081
2082 poll(&pfd, 1, 1000);
2083 if (pfd.revents) {
2084 r = read(ptyfd, buf, sizeof(buf));
2085 if (r > 0) {
2086 *wdog_timep = time(NULL);
2087 if (r > 0 && fdlog >= 0)
2088 write(fdlog, buf, r);
2089 while (--r >= 0) {
2090 if (buf[r] == '\n')
2091 ++wmsg->lines;
2092 }
2093 return MPTY_DATA;
2094 return 1;
2095 } else if (r < 0) {
2096 if (errno != EINTR && errno != EAGAIN)
2097 return MPTY_FAILED;
2098 return MPTY_AGAIN;
2099 } else if (r == 0) {
2100 return MPTY_EOF;
2101 }
2102 }
2103 return MPTY_AGAIN;
2104}
2105
8e25f19b
MD
2106/*
2107 * Copy a (package) file from (src) to (dst), use an intermediate file and
2108 * rename to ensure that interruption does not leave us with a corrupt
2109 * package file.
2110 *
2111 * This is called by the WORKER process.
2112 *
2113 * (dsynth management thread -> WORKER process -> sub-processes)
2114 */
2115#define COPYBLKSIZE 32768
2116
2117static int
2118copyfile(char *src, char *dst)
2119{
2120 char *tmp;
2121 char *buf;
2122 int fd1;
2123 int fd2;
2124 int error = 0;
2125 ssize_t r;
2126
2127 asprintf(&tmp, "%s.new", dst);
2128 buf = malloc(COPYBLKSIZE);
2129
2130 fd1 = open(src, O_RDONLY|O_CLOEXEC);
2131 fd2 = open(tmp, O_RDWR|O_CREAT|O_TRUNC|O_CLOEXEC, 0644);
2132 while ((r = read(fd1, buf, COPYBLKSIZE)) > 0) {
2133 if (write(fd2, buf, r) != r)
2134 error = 1;
2135 }
f7f25838
MD
2136 if (r < 0)
2137 error = 1;
8e25f19b
MD
2138 close(fd1);
2139 close(fd2);
2140 if (error) {
2141 remove(tmp);
2142 } else {
2143 if (rename(tmp, dst)) {
2144 error = 1;
2145 remove(tmp);
2146 }
2147 }
2148
2149 free(buf);
2150 free(tmp);
2151
2152 return error;
2153}