12ce3208d4b5b906e26c85c47427b970d074232c
[dragonfly.git] / usr.bin / dsynth / build.c
1 /*
2  * Copyright (c) 2019 The DragonFly Project.  All rights reserved.
3  *
4  * This code is derived from software contributed to The DragonFly Project
5  * by Matthew Dillon <dillon@backplane.com>
6  *
7  * This code uses concepts and configuration based on 'synth', by
8  * John R. Marino <draco@marino.st>, which was written in ada.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  *
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in
18  *    the documentation and/or other materials provided with the
19  *    distribution.
20  * 3. Neither the name of The DragonFly Project nor the names of its
21  *    contributors may be used to endorse or promote products derived
22  *    from this software without specific, prior written permission.
23  *
24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
27  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
28  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
29  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
30  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
31  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
32  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
33  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
34  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35  * SUCH DAMAGE.
36  */
37 #include "dsynth.h"
38
39 worker_t WorkerAry[MAXWORKERS];
40 int BuildInitialized;
41 int RunningWorkers;
42 int DynamicMaxWorkers;
43 int FailedWorkers;
44 pthread_mutex_t WorkerMutex;
45 pthread_cond_t WorkerCond;
46
47 static int build_find_leaves(pkg_t *parent, pkg_t *pkg, pkg_t ***build_tailp,
48                         int *app, int *hasworkp, int level, int first);
49 static void build_clear_trav(pkg_t *pkg);
50 static void startbuild(pkg_t **build_listp, pkg_t ***build_tailp);
51 static int qsort_depi(const void *pkg1, const void *pkg2);
52 static int qsort_idep(const void *pkg1, const void *pkg2);
53 static void startworker(pkg_t *pkg, worker_t *work);
54 static void cleanworker(worker_t *work);
55 static void waitbuild(int whilematch, int dynamicmax);
56 static void workercomplete(worker_t *work);
57 static void *childBuilderThread(void *arg);
58 static int childInstallPkgDeps(worker_t *work);
59 static void childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list, int undoit);
60 static void dophase(worker_t *work, wmsg_t *wmsg,
61                         int wdog, int phaseid, const char *phase);
62 static void phaseReapAll(void);
63 static void phaseTerminateSignal(int sig);
64 static char *buildskipreason(pkglink_t *parent, pkg_t *pkg);
65 static int mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg,
66                         time_t *wdog_timep);
67 static int copyfile(char *src, char *dst);
68
69 static worker_t *SigWork;
70 static int MasterPtyFd = -1;
71 static pid_t SigPid;
72
73 #define MPTY_FAILED     -2
74 #define MPTY_AGAIN      -1
75 #define MPTY_EOF        0
76 #define MPTY_DATA       1
77
78 int BuildTotal;
79 int BuildCount;
80 int BuildSkipCount;
81 int BuildFailCount;
82 int BuildSuccessCount;
83
84 /*
85  * Initialize the WorkerAry[]
86  */
87 void
88 DoInitBuild(int slot_override)
89 {
90         worker_t *work;
91         struct stat st;
92         int i;
93
94         ddassert(slot_override < 0 || MaxWorkers == 1);
95
96         bzero(WorkerAry, MaxWorkers * sizeof(worker_t));
97         pthread_mutex_init(&WorkerMutex, NULL);
98
99         for (i = 0; i < MaxWorkers; ++i) {
100                 work = &WorkerAry[i];
101                 work->index = (slot_override >= 0) ? slot_override : i;
102                 work->state = WORKER_NONE;
103                 asprintf(&work->basedir, "%s/SL%02d", BuildBase, work->index);
104                 pthread_cond_init(&work->cond, NULL);
105         }
106         BuildCount = 0;
107
108         /*
109          * Create required sub-directories. The base directories must already
110          * exist as a dsynth configuration safety.
111          */
112         if (stat(RepositoryPath, &st) < 0) {
113                 if (mkdir(RepositoryPath, 0755) < 0)
114                         dfatal("Cannot mkdir %s\n", RepositoryPath);
115         }
116
117         BuildInitialized = 1;
118
119         /*
120          * slow-start (increases at a rate of 1 per 5 seconds)
121          */
122         if (SlowStartOpt > MaxWorkers)
123                 DynamicMaxWorkers = MaxWorkers;
124         else if (SlowStartOpt > 0)
125                 DynamicMaxWorkers = SlowStartOpt;
126         else
127                 DynamicMaxWorkers = MaxWorkers;
128 }
129
130 /*
131  * Called by the frontend to clean-up any hanging mounts.
132  */
133 void
134 DoCleanBuild(void)
135 {
136         int i;
137
138         ddassert(BuildInitialized);
139
140         for (i = 0; i < MaxWorkers; ++i) {
141                 DoWorkerUnmounts(&WorkerAry[i]);
142         }
143         dlogreset();
144 }
145
146 void
147 DoBuild(pkg_t *pkgs)
148 {
149         pkg_t *build_list = NULL;
150         pkg_t **build_tail = &build_list;
151         pkg_t *scan;
152         int haswork = 1;
153         int first = 1;
154
155         for (scan = pkgs; scan; scan = scan->bnext)
156                 ++BuildTotal;
157
158         /*
159          * The pkg and pkg-static binaries are needed.  If already present
160          * then assume that the template is also valid, otherwise build
161          * both.
162          */
163         scan = GetPkgPkg(pkgs);
164
165         /*
166          * Create our template.  The template will be missing pkg
167          * and pkg-static.
168          */
169         if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0) {
170                 DoCreateTemplate(1);    /* force a fresh template */
171         } else {
172                 DoCreateTemplate(0);
173         }
174
175         /*
176          * This will clear the screen and set-up our gui, so sleep
177          * a little first in case the user wants to see what was
178          * printed before.
179          */
180         sleep(2);
181         pthread_mutex_lock(&WorkerMutex);
182         GuiInit();
183         GuiReset();
184
185         if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0) {
186                 char *buf;
187                 int rc;
188
189                 /*
190                  * Build pkg/pkg-static.
191                  */
192                 build_list = scan;
193                 build_tail = &scan->build_next;
194                 startbuild(&build_list, &build_tail);
195                 while (RunningWorkers == 1)
196                         waitbuild(1, 0);
197
198                 if (scan->flags & PKGF_NOBUILD)
199                         dfatal("Unable to build 'pkg'");
200                 if (scan->flags & PKGF_ERROR)
201                         dfatal("Error building 'pkg'");
202                 if ((scan->flags & PKGF_SUCCESS) == 0)
203                         dfatal("Error building 'pkg'");
204
205                 /*
206                  * Install pkg/pkg-static into the template
207                  */
208                 asprintf(&buf,
209                          "cd %s/Template; "
210                          "tar --exclude '+*' --exclude '*/man/*' "
211                          "-xvzpf %s/%s > /dev/null 2>&1",
212                          BuildBase,
213                          RepositoryPath,
214                          scan->pkgfile);
215                 rc = system(buf);
216                 if (rc)
217                         dfatal("Command failed: %s\n", buf);
218                 free(buf);
219         }
220
221         /*
222          * Nominal bulk build sequence
223          */
224         while (haswork) {
225                 haswork = 0;
226                 fflush(stdout);
227                 for (scan = pkgs; scan; scan = scan->bnext) {
228                         ddprintf(0, "SCANLEAVES %08x %s\n",
229                                  scan->flags, scan->portdir);
230                         scan->flags |= PKGF_BUILDLOOP;
231                         /*
232                          * NOTE: We must still find dependencies if PACKAGED
233                          *       to fill in the gaps, as some of them may
234                          *       need to be rebuilt.
235                          */
236                         if (scan->flags & (PKGF_SUCCESS | PKGF_FAILURE |
237                                            PKGF_ERROR)) {
238 #if 0
239                                 ddprintf(0, "%s: already built\n",
240                                          scan->portdir);
241 #endif
242                         } else {
243                                 int ap = 0;
244                                 build_find_leaves(NULL, scan, &build_tail,
245                                                   &ap, &haswork, 0, first);
246                                 ddprintf(0, "TOPLEVEL %s %08x\n",
247                                          scan->portdir, ap);
248                         }
249                         scan->flags &= ~PKGF_BUILDLOOP;
250                         build_clear_trav(scan);
251                 }
252                 first = 0;
253                 fflush(stdout);
254                 startbuild(&build_list, &build_tail);
255
256                 if (haswork == 0 && RunningWorkers) {
257                         waitbuild(RunningWorkers, 1);
258                         haswork = 1;
259                 }
260         }
261         pthread_mutex_unlock(&WorkerMutex);
262
263         GuiDone();
264
265         ddprintf(0, "BuildCount %d\n", BuildCount);
266 }
267
268 /*
269  * Traverse the packages (pkg) depends on recursively until we find
270  * a leaf to build or report as unbuildable.  Calculates and assigns a
271  * dependency count.  Returns all parallel-buildable packages.
272  *
273  * (pkg) itself is only added to the list if it is immediately buildable.
274  */
275 static
276 int
277 build_find_leaves(pkg_t *parent, pkg_t *pkg, pkg_t ***build_tailp,
278                   int *app, int *hasworkp, int level, int first)
279 {
280         pkglink_t *link;
281         pkg_t *scan;
282         int idep_count = 0;
283         int apsub;
284         char *buf;
285
286         /*
287          * Already on build list, possibly in-progress, tell caller that
288          * it is not ready.
289          */
290         ddprintf(level, "sbuild_find_leaves %d %s %08x {\n",
291                  level, pkg->portdir, pkg->flags);
292         if (pkg->flags & PKGF_BUILDLIST) {
293                 ddprintf(level, "} (already on build list)\n");
294                 *app |= PKGF_NOTREADY;
295                 return (pkg->idep_count);
296         }
297
298         /*
299          * Check dependencies
300          */
301         ++level;
302         PKGLIST_FOREACH(link, &pkg->idepon_list) {
303                 scan = link->pkg;
304
305                 if (scan == NULL)
306                         continue;
307                 ddprintf(level, "check %s %08x\t", scan->portdir, scan->flags);
308
309                 /*
310                  * When accounting for a successful build, just bump
311                  * idep_count by one.  scan->idep_count will heavily
312                  * overlap packages that we count down multiple branches.
313                  *
314                  * We must still recurse through PACKAGED packages as
315                  * some of their dependencies might be missing.
316                  */
317                 if (scan->flags & PKGF_SUCCESS) {
318                         ddprintf(0, "SUCCESS - OK\n");
319                         ++idep_count;
320                         continue;
321                 }
322
323                 /*
324                  * ERROR includes FAILURE, which is set in numerous situations
325                  * including when NOBUILD state is processed.  So check for
326                  * NOBUILD state first.
327                  *
328                  * An ERROR in a sub-package causes a NOBUILD in packages
329                  * that depend on it.
330                  */
331                 if (scan->flags & PKGF_NOBUILD) {
332                         ddprintf(0, "NOBUILD - OK "
333                                     "(propogate failure upward)\n");
334                         *app |= PKGF_NOBUILD_S;
335                         continue;
336                 }
337                 if (scan->flags & PKGF_ERROR) {
338                         ddprintf(0, "ERROR - OK (propogate failure upward)\n");
339                         *app |= PKGF_NOBUILD_S;
340                         continue;
341                 }
342
343                 /*
344                  * If already on build-list this dependency is not ready.
345                  */
346                 if (scan->flags & PKGF_BUILDLIST) {
347                         ddprintf(0, " [BUILDLIST]");
348                         *app |= PKGF_NOTREADY;
349                 }
350
351                 /*
352                  * If not packaged this dependency is not ready for
353                  * the caller.
354                  */
355                 if ((scan->flags & PKGF_PACKAGED) == 0) {
356                         ddprintf(0, " [NOT_PACKAGED]");
357                         *app |= PKGF_NOTREADY;
358                 }
359
360                 /*
361                  * Reduce search complexity, if we have already processed
362                  * scan in the traversal it will either already be on the
363                  * build list or it will not be buildable.  Either way
364                  * the parent is not buildable.
365                  */
366                 if (scan->flags & PKGF_BUILDTRAV) {
367                         ddprintf(0, " [BUILDTRAV]\n");
368                         *app |= PKGF_NOTREADY;
369                         continue;
370                 }
371
372                 /*
373                  * Assert on dependency loop
374                  */
375                 ++idep_count;
376                 if (scan->flags & PKGF_BUILDLOOP) {
377                         dfatal("pkg dependency loop %s -> %s",
378                                 parent->portdir, scan->portdir);
379                 }
380                 scan->flags |= PKGF_BUILDLOOP;
381                 apsub = 0;
382                 ddprintf(0, " SUBRECURSION {\n");
383                 idep_count += build_find_leaves(pkg, scan, build_tailp,
384                                                 &apsub, hasworkp,
385                                                 level + 1, first);
386                 scan->flags &= ~PKGF_BUILDLOOP;
387                 *app |= apsub;
388                 if (apsub & PKGF_NOBUILD) {
389                         ddprintf(level, "} (sub-nobuild)\n");
390                 } else if (apsub & PKGF_ERROR) {
391                         ddprintf(level, "} (sub-error)\n");
392                 } else if (apsub & PKGF_NOTREADY) {
393                         ddprintf(level, "} (sub-notready)\n");
394                 } else {
395                         ddprintf(level, "} (sub-ok)\n");
396                 }
397         }
398         --level;
399         pkg->idep_count = idep_count;
400         pkg->flags |= PKGF_BUILDTRAV;
401
402         /*
403          * Incorporate scan results into pkg state.
404          */
405         if ((pkg->flags & PKGF_NOBUILD) == 0 && (*app & PKGF_NOBUILD)) {
406                 *hasworkp = 1;
407         } else if ((pkg->flags & PKGF_ERROR) == 0 && (*app & PKGF_ERROR)) {
408                 *hasworkp = 1;
409         }
410         pkg->flags |= *app & ~PKGF_NOTREADY;
411
412         /*
413          * Clear PACKAGED bit if sub-dependencies aren't clean
414          */
415         if ((pkg->flags & PKGF_PACKAGED) &&
416             (pkg->flags & (PKGF_NOTREADY|PKGF_ERROR|PKGF_NOBUILD))) {
417                 pkg->flags &= ~PKGF_PACKAGED;
418                 ddassert(pkg->pkgfile);
419                 asprintf(&buf, "%s/%s", RepositoryPath, pkg->pkgfile);
420                 if (remove(buf) < 0) {
421                         dlog(DLOG_ALL,
422                              "[XXX] %s DELETE-PACKAGE %s (failed)\n",
423                              pkg->portdir, buf);
424                 } else {
425                         dlog(DLOG_ALL,
426                              "[XXX] %s DELETE-PACKAGE %s "
427                              "(due to dependencies)\n",
428                              pkg->portdir, buf);
429                 }
430                 free(buf);
431                 buf = NULL;
432                 *hasworkp = 1;
433         }
434
435         /*
436          * Handle propagated flags
437          */
438         if (pkg->flags & PKGF_ERROR) {
439                 /*
440                  * This can only happen if the ERROR has already been
441                  * processed and accounted for.
442                  */
443                 ddprintf(level, "} (ERROR - %s)\n", pkg->portdir);
444         } else if (*app & PKGF_NOTREADY) {
445                 /*
446                  * We don't set PKGF_NOTREADY in the pkg, it is strictly
447                  * a transient flag propagated via build_find_leaves().
448                  *
449                  * Just don't add the package to the list.
450                  *
451                  * NOTE: Even if NOBUILD is set (meaning we could list it
452                  *       and let startbuild() finish it up as a skip, we
453                  *       don't process it to the list because we want to
454                  *       process all the dependencies, so someone doing a
455                  *       manual build can get more complete information and
456                  *       does not have to iterate each failed dependency one
457                  *       at a time.
458                  */
459                 ;
460         } else if (pkg->flags & PKGF_SUCCESS) {
461                 ddprintf(level, "} (SUCCESS - %s)\n", pkg->portdir);
462         } else if (pkg->flags & PKGF_DUMMY) {
463                 ddprintf(level, "} (DUMMY/META - SUCCESS)\n");
464                 pkg->flags |= PKGF_SUCCESS;
465                 *hasworkp = 1;
466                 if (first) {
467                         dlog(DLOG_ALL, "[XXX] %s META-ALREADY-BUILT\n",
468                              pkg->portdir);
469                         --BuildTotal;
470                 } else {
471                         dlog(DLOG_SUCC, "[XXX] %s meta-node complete\n",
472                              pkg->portdir);
473                 }
474         } else if (pkg->flags & PKGF_PACKAGED) {
475                 /*
476                  * We can just mark the pkg successful.  If this is
477                  * the first pass, we count this as an initial pruning
478                  * pass and reduce BuildTotal.
479                  */
480                 ddprintf(level, "} (PACKAGED - SUCCESS)\n");
481                 pkg->flags |= PKGF_SUCCESS;
482                 *hasworkp = 1;
483                 if (first) {
484                         dlog(DLOG_ALL, "[XXX] %s ALREADY-BUILT\n",
485                              pkg->portdir);
486                         --BuildTotal;
487                 }
488         } else {
489                 /*
490                  * All dependencies are successful, queue new work
491                  * and indicate not-ready to the parent (since our
492                  * package has to be built).
493                  *
494                  * NOTE: The NOBUILD case propagates to here as well
495                  *       and is ultimately handled by startbuild().
496                  */
497                 *hasworkp = 1;
498                 if (pkg->flags & PKGF_NOBUILD)
499                         ddprintf(level, "} (ADDLIST(NOBUILD) - %s)\n",
500                                  pkg->portdir);
501                 else
502                         ddprintf(level, "} (ADDLIST - %s)\n", pkg->portdir);
503                 pkg->flags |= PKGF_BUILDLIST;
504                 **build_tailp = pkg;
505                 *build_tailp = &pkg->build_next;
506                 *app |= PKGF_NOTREADY;
507         }
508
509         return idep_count;
510 }
511
512 static
513 void
514 build_clear_trav(pkg_t *pkg)
515 {
516         pkglink_t *link;
517         pkg_t *scan;
518
519         pkg->flags &= ~PKGF_BUILDTRAV;
520         PKGLIST_FOREACH(link, &pkg->idepon_list) {
521                 scan = link->pkg;
522                 if (scan && (scan->flags & PKGF_BUILDTRAV))
523                         build_clear_trav(scan);
524         }
525 }
526
527 /*
528  * Take a list of pkg ready to go, sort it, and assign it to worker
529  * slots.  This routine blocks in waitbuild() until it can dispose of
530  * the entire list.
531  *
532  * WorkerMutex is held by the caller.
533  */
534 static
535 void
536 startbuild(pkg_t **build_listp, pkg_t ***build_tailp)
537 {
538         pkg_t *pkg;
539         pkg_t **idep_ary;
540         pkg_t **depi_ary;
541         int count;
542         int idep_index;
543         int depi_index;
544         int i;
545         int n;
546         worker_t *work;
547         static int IterateWorker;
548
549         /*
550          * Nothing to do
551          */
552         if (*build_listp == NULL)
553                 return;
554
555         /*
556          * Sort
557          */
558         count = 0;
559         for (pkg = *build_listp; pkg; pkg = pkg->build_next)
560                 ++count;
561         idep_ary = calloc(count, sizeof(pkg_t *));
562         depi_ary = calloc(count, sizeof(pkg_t *));
563
564         count = 0;
565         for (pkg = *build_listp; pkg; pkg = pkg->build_next) {
566                 idep_ary[count] = pkg;
567                 depi_ary[count] = pkg;
568                 ++count;
569         }
570
571         /*
572          * idep_ary - sorted by #of dependencies this pkg has.
573          * depi_ary - sorted by #of other packages that depend on this pkg.
574          */
575         qsort(idep_ary, count, sizeof(pkg_t *), qsort_idep);
576         qsort(depi_ary, count, sizeof(pkg_t *), qsort_depi);
577
578         idep_index = 0;
579         depi_index = 0;
580
581         /*
582          * Half the workers build based on the highest depi count,
583          * the other half build based on the highest idep count.
584          *
585          * This is an attempt to get projects which many other projects
586          * depend on built first, but to also try to build large projects
587          * (which tend to have a lot of dependencies) earlier rather than
588          * later so the end of the bulk run doesn't inefficiently build
589          * the last few huge projects.
590          *
591          * Loop until we manage to assign slots to everyone.  We do not
592          * wait for build completion.
593          *
594          * This is the point where we handle DUMMY packages (these are
595          * dummy unflavored packages which 'cover' all the flavors for
596          * a package).  These are not real packages are marked SUCCESS
597          * at this time because their dependencies (the flavors) have all
598          * been built.
599          */
600         while (idep_index != count || depi_index != count) {
601                 pkg_t *pkgi;
602                 pkg_t *ipkg;
603
604                 /*
605                  * Find candidate to start sorted by depi or idep.
606                  */
607                 ipkg = NULL;
608                 while (idep_index < count) {
609                         ipkg = idep_ary[idep_index];
610                         if ((ipkg->flags &
611                              (PKGF_SUCCESS | PKGF_FAILURE |
612                               PKGF_ERROR | PKGF_RUNNING)) == 0) {
613                                 break;
614                         }
615                         ipkg = NULL;
616                         ++idep_index;
617                 }
618
619                 pkgi = NULL;
620                 while (depi_index < count) {
621                         pkgi = depi_ary[depi_index];
622                         if ((pkgi->flags &
623                              (PKGF_SUCCESS | PKGF_FAILURE |
624                               PKGF_ERROR | PKGF_RUNNING)) == 0) {
625                                 break;
626                         }
627                         pkgi = NULL;
628                         ++depi_index;
629                 }
630
631                 /*
632                  * ipkg and pkgi must either both be NULL, or both
633                  * be non-NULL.
634                  */
635                 if (ipkg == NULL && pkgi == NULL)
636                         break;
637                 ddassert(ipkg && pkgi);
638
639                 /*
640                  * Handle the NOBUILD case right here, there's no point
641                  * queueing it anywhere.
642                  */
643                 if (ipkg->flags & PKGF_NOBUILD) {
644                         char *reason;
645
646                         ipkg->flags |= PKGF_FAILURE;
647                         ipkg->flags &= ~PKGF_BUILDLIST;
648
649                         ++BuildSkipCount;
650                         ++BuildCount;
651                         reason = buildskipreason(NULL, ipkg);
652                         dlog(DLOG_SKIP, "[XXX] %s skipped due to %s\n",
653                              ipkg->portdir, reason);
654                         free(reason);
655                         continue;
656                 }
657                 if (pkgi->flags & PKGF_NOBUILD) {
658                         char *reason;
659
660                         pkgi->flags |= PKGF_FAILURE;
661                         pkgi->flags &= ~PKGF_BUILDLIST;
662
663                         ++BuildSkipCount;
664                         ++BuildCount;
665                         reason = buildskipreason(NULL, pkgi);
666                         dlog(DLOG_SKIP, "[XXX] %s skipped due to %s\n",
667                              pkgi->portdir, reason);
668                         free(reason);
669                         continue;
670                 }
671
672                 /*
673                  * Block while no slots are available.  waitbuild()
674                  * will clean out any DONE states.
675                  */
676                 while (RunningWorkers >= DynamicMaxWorkers ||
677                        RunningWorkers >= MaxWorkers - FailedWorkers) {
678                         waitbuild(RunningWorkers, 1);
679                 }
680
681                 /*
682                  * Find an available worker slot, there should be at
683                  * least one.
684                  */
685                 for (i = 0; i < MaxWorkers; ++i) {
686                         n = IterateWorker % MaxWorkers;
687                         work = &WorkerAry[n];
688
689                         if (work->state == WORKER_DONE ||
690                             work->state == WORKER_FAILED) {
691                                 workercomplete(work);
692                         }
693                         if (work->state == WORKER_NONE ||
694                             work->state == WORKER_IDLE) {
695                                 if (n <= MaxWorkers / 2) {
696                                         startworker(pkgi, work);
697                                 } else {
698                                         startworker(ipkg, work);
699                                 }
700                                 /*GuiUpdate(work);*/
701                                 break;
702                         }
703                         ++IterateWorker;
704                 }
705                 ddassert(i != MaxWorkers);
706         }
707         GuiSync();
708
709         /*
710          * We disposed of the whole list
711          */
712         free(idep_ary);
713         free(depi_ary);
714         *build_listp = NULL;
715         *build_tailp = build_listp;
716 }
717
718 typedef const pkg_t *pkg_tt;
719
720 static int
721 qsort_idep(const void *pkg1_arg, const void *pkg2_arg)
722 {
723         const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg;
724         const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg;
725
726         return (pkg2->idep_count - pkg1->idep_count);
727 }
728
729 static int
730 qsort_depi(const void *pkg1_arg, const void *pkg2_arg)
731 {
732         const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg;
733         const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg;
734
735         return (pkg2->depi_count - pkg1->depi_count);
736 }
737
738 /*
739  * Frontend starts a pkg up on a worker
740  *
741  * WorkerMutex must be held.
742  */
743 static void
744 startworker(pkg_t *pkg, worker_t *work)
745 {
746         switch(work->state) {
747         case WORKER_NONE:
748                 pthread_create(&work->td, NULL, childBuilderThread, work);
749                 work->state = WORKER_IDLE;
750                 /* fall through */
751         case WORKER_IDLE:
752                 dlog(DLOG_ALL, "[%03d] %s START\n", work->index, pkg->portdir);
753                 cleanworker(work);
754                 pkg->flags |= PKGF_RUNNING;
755                 work->pkg = pkg;
756                 pthread_cond_signal(&work->cond);
757                 ++RunningWorkers;
758                 /*GuiUpdate(work);*/
759                 break;
760         case WORKER_PENDING:
761         case WORKER_RUNNING:
762         case WORKER_DONE:
763         case WORKER_FAILED:
764         case WORKER_FROZEN:
765         case WORKER_EXITING:
766         default:
767                 dfatal("startworker: Unexpected state %d for worker %d",
768                        work->state, work->index);
769                 break;
770         }
771 }
772
773 static void
774 cleanworker(worker_t *work)
775 {
776         work->state = WORKER_PENDING;
777         work->flags = 0;
778         work->accum_error = 0;
779         work->start_time = time(NULL);
780 }
781
782 /*
783  * Frontend finishes up a completed pkg on a worker.
784  *
785  * If the worker is in a FAILED state we clean the pkg out but (for now)
786  * leave it in its failed state so we can debug.  At this point
787  * workercomplete() will be called every once in a while on the state
788  * and we have to deal with the NULL pkg.
789  *
790  * WorkerMutex must be held.
791  */
792 static void
793 workercomplete(worker_t *work)
794 {
795         pkg_t *pkg;
796
797         /*
798          * Steady state FAILED case.
799          */
800         if (work->state == WORKER_FAILED) {
801                 if (work->pkg == NULL)
802                         return;
803         }
804
805         /*
806          * Process pkg out of the worker
807          */
808         pkg = work->pkg;
809         if (pkg->flags & (PKGF_ERROR|PKGF_NOBUILD)) {
810                 pkg->flags |= PKGF_FAILURE;
811
812                 /*
813                  * This NOBUILD condition XXX can occur if the package is
814                  * not allowed to be built.
815                  */
816                 if (pkg->flags & PKGF_NOBUILD) {
817                         char *reason;
818
819                         ++BuildSkipCount;
820                         reason = buildskipreason(NULL, pkg);
821                         dlog(DLOG_SKIP, "[%03d] %s skipped due to %s\n",
822                              work->index, pkg->portdir, reason);
823                         free(reason);
824                 } else {
825                         ++BuildFailCount;
826                         dlog(DLOG_FAIL, "[%03d] %s build-failed\n",
827                              work->index, pkg->portdir);
828                 }
829         } else {
830                 time_t t;
831                 int h;
832                 int m;
833                 int s;
834
835                 t = time(NULL) - work->start_time;
836                 s = t % 60;
837                 m = t / 60 % 60;
838                 h = t / 3600;
839
840                 dlog(DLOG_SUCC,
841                      "[%03d] %s\tbuild-succeded in %02d:%02d:%02d\n",
842                      work->index, pkg->portdir, h, m, s);
843                 pkg->flags |= PKGF_SUCCESS;
844                 ++BuildSuccessCount;
845         }
846         ++BuildCount;
847         pkg->flags &= ~PKGF_BUILDLIST;
848         dlog(DLOG_ALL, "[%03d] %s FINISHED (%s)\n",
849              work->index,
850              pkg->portdir,
851              ((pkg->flags & PKGF_SUCCESS) ? "success" : "failure"));
852         pkg->flags &= ~PKGF_RUNNING;
853         work->pkg = NULL;
854         --RunningWorkers;
855
856         if (work->state == WORKER_FAILED) {
857                 dlog(DLOG_ALL, "[%03d] XXX/XXX WORKER IS IN A FAILED STATE\n",
858                      work->index);
859                 ++FailedWorkers;
860         } else if (work->flags & WORKERF_FREEZE) {
861                 dlog(DLOG_ALL, "[%03d] XXX/XXX WORKER FROZEN BY REQUEST\n",
862                      work->index);
863                 work->state = WORKER_FROZEN;
864         } else {
865                 work->state = WORKER_IDLE;
866         }
867 }
868
869 /*
870  * Wait for one or more workers to complete.
871  *
872  * WorkerMutex must be held.
873  */
874 static void
875 waitbuild(int whilematch, int dynamicmax)
876 {
877         static time_t wblast_time;
878         struct timespec ts;
879         worker_t *work;
880         time_t t;
881         int i;
882
883         if (whilematch == 0)
884                 whilematch = 1;
885
886         while (RunningWorkers == whilematch) {
887                 for (i = 0; i < MaxWorkers; ++i) {
888                         work = &WorkerAry[i];
889                         if (work->state == WORKER_DONE ||
890                             work->state == WORKER_FAILED) {
891                                 workercomplete(work);
892                         } else {
893                                 pthread_cond_signal(&work->cond);
894                         }
895                         GuiUpdate(work);
896                 }
897                 GuiUpdateTop();
898                 GuiSync();
899                 if (RunningWorkers == whilematch) {
900                         clock_gettime(CLOCK_REALTIME, &ts);
901                         ts.tv_sec += 1;
902                         ts.tv_nsec = 0;
903                         pthread_cond_timedwait(&WorkerCond, &WorkerMutex, &ts);
904                 }
905
906                 /*
907                  * Dynamically reduce MaxWorkers based on the load.  When
908                  * the load exceeds 2 x ncpus we reduce the number of workers
909                  * up to 75% of MaxWorkers @ (5 x ncpus) load.
910                  *
911                  * Dynamically reduce MaxWorkers based on swap use, starting
912                  * at 10% swap and up to 75% of MaxWorkers at 40% swap.
913                  *
914                  * NOTE! Generally speaking this allows more workers to be
915                  *       configured which helps in two ways.  First it allows
916                  *       a higher build rate for smaller packages.  Second
917                  *       it allows dsynth to ratchet-down the number of slots
918                  *       when large packages are forcing the load up.
919                  *
920                  *       A high load doesn't hurt efficiency, but swap usage
921                  *       due to loading the tmpfs in lots of worker slots up
922                  *       with tons of pkg install's (pre-reqs for a build)
923                  *       does.  Reducing the number of worker slots has a
924                  *       huge beneficial effect on reducing swap use / paging.
925                  */
926                 t = time(NULL);
927                 if (dynamicmax && (wblast_time == 0 ||
928                                    (unsigned)(t - wblast_time) >= 5)) {
929                         double min_load = 1.5 * NumCores;
930                         double max_load = 5.0 * NumCores;
931                         double min_swap = 0.10;
932                         double max_swap = 0.40;
933                         double dload[3];
934                         double dswap;
935                         int reduce1;
936                         int reduce2;
937                         int reduce;
938                         int noswap;
939
940                         wblast_time = t;
941                         getloadavg(dload, 3);
942                         dswap = getswappct(&noswap);
943
944                         if (dload[0] < min_load) {
945                                 reduce1 = 0;
946                         } else if (dload[0] <= max_load) {
947                                 reduce1 = 75 * (dload[0] - min_load) /
948                                           (max_load - min_load);
949                         } else {
950                                 reduce1 = MaxWorkers * 75 / 100;
951                         }
952
953                         if (dswap < min_swap) {
954                                 reduce2 = 0;
955                         } else if (dswap <= max_swap) {
956                                 reduce2 = 75 * (dswap - min_swap) /
957                                           (max_swap - min_swap);
958                         } else {
959                                 reduce2 = MaxWorkers * 75 / 100;
960                         }
961
962                         /*
963                          * Priority reduction, convert to DynamicMaxWorkers
964                          */
965                         if (reduce1 > reduce2)
966                                 reduce = reduce1;
967                         else
968                                 reduce = reduce2;
969                         reduce = MaxWorkers - reduce;
970                         if (reduce < 4)
971                                 reduce = 4;
972                         if (reduce > MaxWorkers)
973                                 reduce = MaxWorkers;
974
975                         /*
976                          * Handle slow-start
977                          */
978                         if (reduce > DynamicMaxWorkers + 1) {
979                                 reduce = DynamicMaxWorkers + 1;
980                         }
981
982                         /*
983                          * Stop waiting if DynamicMaxWorkers is going
984                          * to increase.
985                          */
986                         if (DynamicMaxWorkers < reduce)
987                                 whilematch = -1;
988
989                         /*
990                          * And adjust
991                          */
992                         if (DynamicMaxWorkers != reduce) {
993                                 dlog(DLOG_ALL,
994                                      "[XXX] Load=%-6.2f(-%-2d) "
995                                      "Swappct=%-3.2f(-%-2d) "
996                                      "Adjust MaxWorkers "
997                                      "%2d->%-2d\n",
998                                      dload[0], reduce1,
999                                      dswap * 100.0, reduce2,
1000                                      DynamicMaxWorkers,
1001                                      reduce);
1002                                 DynamicMaxWorkers = reduce;
1003                         }
1004                 }
1005         }
1006 }
1007
1008
1009 /*
1010  * Worker pthread (WorkerAry)
1011  *
1012  * This thread belongs to the dsynth master process and handled a worker slot.
1013  * (this_thread) -> WORKER fork/exec (WorkerPocess) -> (pty) -> sub-processes.
1014  */
1015 static void *
1016 childBuilderThread(void *arg)
1017 {
1018         char *envary[1] = { NULL };
1019         worker_t *work = arg;
1020         wmsg_t wmsg;
1021         pkg_t *pkg;
1022         pid_t pid;
1023         int status;
1024         volatile int dowait;
1025         char slotbuf[8];
1026         char fdbuf[8];
1027         char flagsbuf[16];
1028
1029         pthread_mutex_lock(&WorkerMutex);
1030         while (work->terminate == 0) {
1031                 dowait = 1;
1032
1033                 switch(work->state) {
1034                 case WORKER_IDLE:
1035                         break;
1036                 case WORKER_PENDING:
1037                         /*
1038                          * Fork the management process for the pkg operation
1039                          * on this worker slot.
1040                          *
1041                          * This process will set up the environment, do the
1042                          * mounts, will become the reaper, and will process
1043                          * pipe commands and handle chroot operations.
1044                          *
1045                          * NOTE: If SOCK_CLOEXEC is not supported WorkerMutex
1046                          *       is sufficient to interlock F_SETFD FD_CLOEXEC
1047                          *       operations.
1048                          */
1049                         ddassert(work->pkg);
1050                         if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC,
1051                                        PF_UNSPEC, work->fds)) {
1052                                 dfatal_errno("socketpair() during worker fork");
1053                         }
1054                         snprintf(slotbuf, sizeof(slotbuf),
1055                                  "%d", work->index);
1056                         snprintf(fdbuf, sizeof(fdbuf),
1057                                  "3");
1058                         snprintf(flagsbuf, sizeof(flagsbuf),
1059                                  "%d", DebugStopMode);
1060
1061                         /*
1062                          * fds[0] - master
1063                          * fds[1] - slave
1064                          *
1065                          * We pass the salve descriptor in fd 3 and close all
1066                          * other descriptors for security.
1067                          */
1068                         pthread_mutex_unlock(&WorkerMutex);
1069                         pid = vfork();
1070                         if (pid == 0) {
1071                                 close(work->fds[0]);
1072                                 dup2(work->fds[1], 3);
1073                                 closefrom(4);
1074                                 fcntl(3, F_SETFD, 0);
1075                                 execle(DSynthExecPath, DSynthExecPath,
1076                                        "WORKER", slotbuf, fdbuf,
1077                                        work->pkg->portdir, work->pkg->pkgfile,
1078                                        flagsbuf,
1079                                        NULL, envary);
1080                                 write(2, "EXECLE FAILURE\n", 15);
1081                                 _exit(1);
1082                         }
1083                         pthread_mutex_lock(&WorkerMutex);
1084                         close(work->fds[1]);
1085                         work->phase = PHASE_PENDING;
1086                         work->lines = 0;
1087                         work->pid = pid;
1088                         work->state = WORKER_RUNNING;
1089                         /* fall through */
1090                 case WORKER_RUNNING:
1091                         /*
1092                          * Poll for status updates, if NULL is returned
1093                          * and status is non-zero, the communications link
1094                          * failed unexpectedly.
1095                          */
1096                         pkg = work->pkg;
1097                         pthread_mutex_unlock(&WorkerMutex);
1098                         status = ipcreadmsg(work->fds[0], &wmsg);
1099                         pthread_mutex_lock(&WorkerMutex);
1100
1101                         if (status == 0) {
1102                                 /*
1103                                  * Normal message, can include normal
1104                                  * termination which changes us over
1105                                  * to another state.
1106                                  */
1107                                 dowait = 0;
1108                                 switch(wmsg.cmd) {
1109                                 case WMSG_CMD_INSTALL_PKGS:
1110                                         wmsg.cmd = WMSG_RES_INSTALL_PKGS;
1111                                         wmsg.status = childInstallPkgDeps(work);
1112                                         pthread_mutex_unlock(&WorkerMutex);
1113                                         ipcwritemsg(work->fds[0], &wmsg);
1114                                         pthread_mutex_lock(&WorkerMutex);
1115                                         break;
1116                                 case WMSG_CMD_STATUS_UPDATE:
1117                                         work->phase = wmsg.phase;
1118                                         work->lines = wmsg.lines;
1119                                         break;
1120                                 case WMSG_CMD_SUCCESS:
1121                                         work->flags |= WORKERF_SUCCESS;
1122                                         break;
1123                                 case WMSG_CMD_FAILURE:
1124                                         work->flags |= WORKERF_FAILURE;
1125                                         break;
1126                                 case WMSG_CMD_FREEZEWORKER:
1127                                         work->flags |= WORKERF_FREEZE;
1128                                         break;
1129                                 default:
1130                                         break;
1131                                 }
1132                                 GuiUpdate(work);
1133                                 GuiSync();
1134                         } else {
1135                                 close(work->fds[0]);
1136                                 pthread_mutex_unlock(&WorkerMutex);
1137                                 while (waitpid(work->pid, &status, 0) < 0 &&
1138                                        errno == EINTR) {
1139                                         ;
1140                                 }
1141                                 pthread_mutex_lock(&WorkerMutex);
1142
1143                                 if (work->flags & WORKERF_SUCCESS) {
1144                                         pkg->flags |= PKGF_SUCCESS;
1145                                         work->state = WORKER_DONE;
1146                                 } else if (work->flags & WORKERF_FAILURE) {
1147                                         pkg->flags |= PKGF_FAILURE;
1148                                         work->state = WORKER_DONE;
1149                                 } else {
1150                                         pkg->flags |= PKGF_FAILURE;
1151                                         work->state = WORKER_FAILED;
1152                                 }
1153                                 work->flags |= WORKERF_STATUS_UPDATE;
1154                                 pthread_cond_signal(&WorkerCond);
1155                         }
1156                         break;
1157                 case WORKER_DONE:
1158                         /*
1159                          * pkg remains attached until frontend processes the
1160                          * completion.  The frontend will then set the state
1161                          * back to idle.
1162                          */
1163                         break;
1164                 case WORKER_FAILED:
1165                         /*
1166                          * A worker failure means that the worker did not
1167                          * send us a WMSG_CMD_SUCCESS or WMSG_CMD_FAILURE
1168                          * ipc before terminating.
1169                          *
1170                          * We just sit in this state until the front-end
1171                          * does something about it.
1172                          */
1173                         break;
1174                 default:
1175                         dfatal("worker: Unexpected state %d for worker %d",
1176                                work->state, work->index);
1177                         /* NOT REACHED */
1178                         break;
1179                 }
1180
1181                 /*
1182                  * The dsynth frontend will poll us approximately once
1183                  * a second (its variable).
1184                  */
1185                 if (dowait)
1186                         pthread_cond_wait(&work->cond, &WorkerMutex);
1187         }
1188
1189         /*
1190          * Scrap the comm socket if running, this should cause the worker
1191          * process to kill its sub-programs and cleanup.
1192          */
1193         if (work->state == WORKER_RUNNING) {
1194                 pthread_mutex_unlock(&WorkerMutex);
1195                 close(work->fds[0]);
1196                 while (waitpid(work->pid, &status, 0) < 0 &&
1197                        errno == EINTR);
1198                 pthread_mutex_lock(&WorkerMutex);
1199         }
1200
1201         /*
1202          * Final handshake
1203          */
1204         work->state = WORKER_EXITING;
1205         pthread_cond_signal(&WorkerCond);
1206         pthread_mutex_unlock(&WorkerMutex);
1207
1208         return NULL;
1209 }
1210
1211 /*
1212  * Install all the binary packages (we have already built them) that
1213  * the current work package depends on, without duplicates, in a script
1214  * which will be run from within the specified work jail.
1215  *
1216  * Locked by WorkerMutex (global)
1217  */
1218 static int
1219 childInstallPkgDeps(worker_t *work)
1220 {
1221         char *buf;
1222         FILE *fp;
1223
1224         if (PKGLIST_EMPTY(&work->pkg->idepon_list))
1225                 return 0;
1226
1227         asprintf(&buf, "%s/tmp/dsynth_install_pkgs", work->basedir);
1228         fp = fopen(buf, "w");
1229         ddassert(fp != NULL);
1230         fprintf(fp, "#!/bin/sh\n");
1231         fprintf(fp, "#\n");
1232         fchmod(fileno(fp), 0755);
1233
1234         childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 0);
1235         childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 1);
1236         fprintf(fp, "\nexit 0\n");
1237         fclose(fp);
1238         free(buf);
1239
1240         return 1;
1241 }
1242
1243 static void
1244 childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list, int undoit)
1245 {
1246         pkglink_t *link;
1247         pkg_t *pkg;
1248
1249         PKGLIST_FOREACH(link, list) {
1250                 pkg = link->pkg;
1251
1252                 if (undoit) {
1253                         if (pkg->dsynth_install_flg == 1) {
1254                                 pkg->dsynth_install_flg = 0;
1255                                 childInstallPkgDeps_recurse(fp,
1256                                                             &pkg->idepon_list,
1257                                                             undoit);
1258                         }
1259                         continue;
1260                 }
1261                 if (pkg->dsynth_install_flg) {
1262                         if (DebugOpt && pkg->pkgfile) {
1263                                 fprintf(fp, "echo 'AlreadyHave %s'\n",
1264                                         pkg->pkgfile);
1265                         }
1266                         continue;
1267                 }
1268
1269                 childInstallPkgDeps_recurse(fp, &pkg->idepon_list, undoit);
1270                 if (pkg->dsynth_install_flg)
1271                         continue;
1272                 pkg->dsynth_install_flg = 1;
1273
1274                 /*
1275                  * If this is a dummy node with no package, the originator
1276                  * is requesting a flavored package.  We select the default
1277                  * flavor which we presume is the first one.
1278                  */
1279                 if (pkg->pkgfile == NULL && (pkg->flags & PKGF_DUMMY)) {
1280                         pkg_t *spkg = pkg->idepon_list.next->pkg;
1281
1282                         if (spkg) {
1283                                 pkg = spkg;
1284                                 fprintf(fp,
1285                                         "echo 'DUMMY use %s (%p)'\n",
1286                                         pkg->portdir, pkg->pkgfile);
1287                         } else {
1288                                 fprintf(fp,
1289                                         "echo 'CANNOT FIND DEFAULT FLAVOR "
1290                                         "FOR %s'\n",
1291                                         pkg->portdir);
1292                         }
1293                 }
1294
1295                 /*
1296                  * Generate package installation command
1297                  */
1298                 if (pkg->pkgfile) {
1299                         fprintf(fp, "echo 'Installing /packages/All/%s'\n",
1300                                 pkg->pkgfile);
1301                         fprintf(fp, "pkg install -q -y /packages/All/%s "
1302                                 "|| exit 1\n",
1303                                 pkg->pkgfile);
1304                 } else {
1305                         fprintf(fp, "echo 'CANNOT FIND PKG FOR %s'\n",
1306                                 pkg->portdir);
1307                 }
1308         }
1309 }
1310
1311 /*
1312  * Worker process interactions.
1313  *
1314  * The worker process is responsible for managing the build of a single
1315  * package.  It is exec'd by the master dsynth and only loads the
1316  * configuration.
1317  *
1318  * This process does not run in the chroot.  It will become the reaper for
1319  * all sub-processes and it will enter the chroot to execute various phases.
1320  * It catches SIGINTR, SIGHUP, and SIGPIPE and will iterate, terminate, and
1321  * reap all sub-process upon kill or exit.
1322  *
1323  * The command line forwarded to this function is:
1324  *
1325  *      WORKER slot# socketfd portdir/subdir
1326  *
1327  * TERM=dumb
1328  * USER=root
1329  * HOME=/root
1330  * LANG=C
1331  * SSL_NO_VERIFY_PEER=1
1332  * USE_PACKAGE_DEPENDS_ONLY=1   (exec_phase_depends)
1333  * PORTSDIR=/xports
1334  * PORT_DBDIR=/options          For ports options
1335  * PACKAGE_BUILDING=yes         Don't build packages that aren't legally
1336  *                              buildable for a binary repo.
1337  * PKG_DBDIR=/var/db/pkg
1338  * PKG_CACHEDIR=/var/cache/pkg
1339  * PKG_CREATE_VERBOSE=yes       Ensure periodic output during packaging
1340  * (custom environment)
1341  * PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
1342  * UNAME_s=DragonFly            (example)
1343  * UNAME_v=DragonFly 5.7-SYNTH  (example)
1344  * UNAME_p=x86_64               (example)
1345  * UNAME_m=x86_64               (example)
1346  * UNAME_r=5.7-SYNTH            (example)
1347  * NO_DEPENDS=yes               (exec_phase)
1348  * DISTDIR=/distfiles
1349  * WRKDIRPREFIX=/construction
1350  * BATCH=yes
1351  * MAKE_JOBS_NUMBER=n
1352  *
1353  * SETUP:
1354  *      ldconfig -R
1355  *      /usr/local/sbin/pkg-static install /packages/All/<the pkg pkg>
1356  *      /usr/local/sbin/pkg-static install /packages/All/<pkg>
1357  *                      (for all dependencies)
1358  *
1359  * PHASES:              make -C path FLAVOR=flavor <phase>
1360  *      check-sanity
1361  *      pkg-depends
1362  *      fetch-depends
1363  *      fetch
1364  *      checksum
1365  *      extract-depends
1366  *      extract
1367  *      patch-depends
1368  *      patch
1369  *      build-depends
1370  *      lib-depends
1371  *      configure
1372  *      build
1373  *      run-depends
1374  *      stage
1375  *      test            (skipped)
1376  *      check-plist
1377  *      package          e.g. /construction/lang/perl5.28/pkg/perl5-5.28.2.txz
1378  *      install-mtree   (skipped)
1379  *      install         (skipped)
1380  *      deinstall       (skipped)
1381  */
1382
1383 void
1384 WorkerProcess(int ac, char **av)
1385 {
1386         wmsg_t wmsg;
1387         int fd;
1388         int slot;
1389         int tmpfd;
1390         int pkgpkg = 0;
1391         int status;
1392         int len;
1393         int do_install_phase;
1394         char *portdir;
1395         char *pkgfile;
1396         char *flavor;
1397         char *buf;
1398         worker_t *work;
1399         pkg_t pkg;
1400         buildenv_t *benv;
1401
1402         /*
1403          * Parse arguments
1404          */
1405         if (ac != 6) {
1406                 dlog(DLOG_ALL, "WORKER PROCESS %d- bad arguments\n", getpid());
1407                 exit(1);
1408         }
1409         slot = strtol(av[1], NULL, 0);
1410         fd = strtol(av[2], NULL, 0);    /* master<->slave messaging */
1411         portdir = av[3];
1412         pkgfile = av[4];
1413         flavor = strchr(portdir, '@');
1414         if (flavor)
1415                 *flavor++ = 0;
1416         DebugStopMode = strtol(av[5], NULL, 0);
1417
1418         bzero(&wmsg, sizeof(wmsg));
1419
1420         setproctitle("WORKER [%02d] STARTUP  %s", slot, portdir);
1421
1422         if (strcmp(portdir, "ports-mgmt/pkg") == 0)
1423                 pkgpkg = 1;
1424
1425         signal(SIGTERM, phaseTerminateSignal);
1426         signal(SIGINT, phaseTerminateSignal);
1427         signal(SIGHUP, phaseTerminateSignal);
1428
1429         /*
1430          * Set up the environment
1431          */
1432         setenv("TERM", "dumb", 1);
1433         setenv("USER", "root", 1);
1434         setenv("HOME", "/root", 1);
1435         setenv("LANG", "C", 1);
1436         setenv("SSL_NO_VERIFY_PEER", "1", 1);
1437         setenv("USE_PACKAGE_DEPENDS_ONLY", "1", 1);
1438         setenv("PORTSDIR", "/xports", 1);
1439         setenv("PORT_DBDIR", "/options", 1);
1440         setenv("PKG_DBDIR", "/var/db/pkg", 1);
1441         setenv("PKG_CACHEDIR", "/var/cache/pkg", 1);
1442         setenv("PKG_SUFX", USE_PKG_SUFX, 1);
1443
1444
1445 #if 0
1446         setenv("_PERL5_FROM_BIN", "5.28.2", 1);
1447         setenv("OPSYS", OperatingSystemName, 1);
1448 #endif
1449 #if 0
1450         setenv("DFLYVERSION", "5.7.0", 1);
1451         setenv("OSVERSION", "9999999", 1);
1452 #endif
1453
1454         setenv("PATH",
1455                "/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin",
1456                1);
1457
1458         setenv("UNAME_s", OperatingSystemName, 1);
1459         setenv("UNAME_v", VersionName, 1);
1460         setenv("UNAME_p", ArchitectureName, 1);
1461         setenv("UNAME_m", MachineName, 1);
1462         setenv("UNAME_r", ReleaseName, 1);
1463
1464         setenv("NO_DEPENDS", "yes", 1);
1465         setenv("DISTDIR", "/distfiles", 1);
1466         setenv("WRKDIRPREFIX", "/construction", 1);
1467         setenv("BATCH", "yes", 1);
1468
1469         /*
1470          * Special consideration
1471          *
1472          * PACKAGE_BUILDING     - Disallow packaging ports which do not allow
1473          *                        for binary distribution.
1474          *
1475          * PKG_CREATE_VERBOSE   - Ensure periodic output during the packaging
1476          *                        process to avoid a watchdog timeout.
1477          *
1478          */
1479         setenv("PACKAGE_BUILDING", "yes", 1);
1480         setenv("PKG_CREATE_VERBOSE", "yes", 1);
1481
1482         asprintf(&buf, "%d", MaxJobs);
1483         setenv("MAKE_JOBS_NUMBER", buf, 1);
1484         free(buf);
1485
1486         if (flavor)
1487                 setenv("FLAVOR", flavor, 1);
1488
1489         /*
1490          * Load environment from profile
1491          */
1492         for (benv = BuildEnv; benv; benv = benv->next) {
1493                 if (DebugOpt >= 2) {
1494                         dlog(DLOG_ALL, "[%03d] ENV %s=%s\n",
1495                              slot, benv->label, benv->data);
1496                 }
1497                 setenv(benv->label, benv->data, 1);
1498         }
1499
1500         /*
1501          * Become the reaper
1502          */
1503         if (procctl(P_PID, getpid(), PROC_REAP_ACQUIRE, NULL) < 0)
1504                 dfatal_errno("procctl() - Cannot become reaper");
1505
1506         /*
1507          * Initialize a worker structure
1508          */
1509         DoInitBuild(slot);
1510
1511         bzero(&pkg, sizeof(pkg));
1512         pkg.portdir = portdir;          /* sans flavor */
1513         pkg.pkgfile = pkgfile;
1514         if (strchr(portdir, '/'))
1515                 len = strchr(portdir, '/') - portdir;
1516         else
1517                 len = 0;
1518
1519         /*
1520          * Setup the logfile
1521          */
1522         asprintf(&pkg.logfile,
1523                  "%s/%*.*s___%s%s%s.log",
1524                  LogsPath, len, len, portdir,
1525                  ((portdir[len] == '/') ? portdir + len + 1 : portdir + len),
1526                  (flavor ? "@" : ""),
1527                  (flavor ? flavor : ""));
1528         tmpfd = open(pkg.logfile, O_RDWR|O_CREAT|O_TRUNC, 0666);
1529         if (tmpfd >= 0) {
1530                 dlog(DLOG_ALL, "[%03d] %s LOGFILE %s\n",
1531                      slot, pkg.portdir, pkg.logfile);
1532                 close(tmpfd);
1533         } else {
1534                 dlog(DLOG_ALL, "[%03d] LOGFILE %s (create failed)\n",
1535                      slot, pkg.logfile);
1536         }
1537
1538         /*
1539          * Setup the work structure.  Because this is an exec'd sub-process,
1540          * there is only one work structure.
1541          */
1542         work = &WorkerAry[0];
1543         work->flavor = flavor;
1544         work->fds[0] = fd;
1545         work->pkg = &pkg;
1546         work->start_time = time(NULL);
1547
1548         /*
1549          * Do mounts and start the phases
1550          */
1551         SigWork = work;
1552         setproctitle("WORKER [%02d] MOUNTS   %s", slot, portdir);
1553         DoWorkerMounts(work);
1554
1555         wmsg.cmd = WMSG_CMD_INSTALL_PKGS;
1556         ipcwritemsg(fd, &wmsg);
1557         status = ipcreadmsg(fd, &wmsg);
1558         if (status < 0 || wmsg.cmd != WMSG_RES_INSTALL_PKGS)
1559                 dfatal("pkg installation handshake failed");
1560         do_install_phase = wmsg.status;
1561
1562         wmsg.cmd = WMSG_CMD_STATUS_UPDATE;
1563         wmsg.phase = PHASE_INSTALL_PKGS;
1564         wmsg.lines = 0;
1565
1566         status = ipcwritemsg(fd, &wmsg);
1567
1568         if (pkgpkg) {
1569                 dophase(work, &wmsg,
1570                         WDOG5, PHASE_PACKAGE, "package");
1571         } else {
1572                 if (do_install_phase) {
1573                         dophase(work, &wmsg,
1574                                 WDOG4, PHASE_INSTALL_PKGS, "setup");
1575                 }
1576                 dophase(work, &wmsg,
1577                         WDOG2, PHASE_CHECK_SANITY, "check-sanity");
1578                 dophase(work, &wmsg,
1579                         WDOG2, PHASE_PKG_DEPENDS, "pkg-depends");
1580                 dophase(work, &wmsg,
1581                         WDOG7, PHASE_FETCH_DEPENDS, "fetch-depends");
1582                 dophase(work, &wmsg,
1583                         WDOG7, PHASE_FETCH, "fetch");
1584                 dophase(work, &wmsg,
1585                         WDOG2, PHASE_CHECKSUM, "checksum");
1586                 dophase(work, &wmsg,
1587                         WDOG3, PHASE_EXTRACT_DEPENDS, "extract-depends");
1588                 dophase(work, &wmsg,
1589                         WDOG3, PHASE_EXTRACT, "extract");
1590                 dophase(work, &wmsg,
1591                         WDOG2, PHASE_PATCH_DEPENDS, "patch-depends");
1592                 dophase(work, &wmsg,
1593                         WDOG2, PHASE_PATCH, "patch");
1594                 dophase(work, &wmsg,
1595                         WDOG5, PHASE_BUILD_DEPENDS, "build-depends");
1596                 dophase(work, &wmsg,
1597                         WDOG5, PHASE_LIB_DEPENDS, "lib-depends");
1598                 dophase(work, &wmsg,
1599                         WDOG3, PHASE_CONFIGURE, "configure");
1600                 dophase(work, &wmsg,
1601                         WDOG9, PHASE_BUILD, "build");
1602                 dophase(work, &wmsg,
1603                         WDOG5, PHASE_RUN_DEPENDS, "run-depends");
1604                 dophase(work, &wmsg,
1605                         WDOG5, PHASE_STAGE, "stage");
1606 #if 0
1607                 dophase(work, &wmsg,
1608                         WDOG5, PHASE_TEST, "test");
1609 #endif
1610                 dophase(work, &wmsg,
1611                         WDOG1, PHASE_CHECK_PLIST, "check-plist");
1612                 dophase(work, &wmsg,
1613                         WDOG5, PHASE_PACKAGE, "package");
1614 #if 0
1615                 dophase(work, &wmsg,
1616                         WDOG5, PHASE_INSTALL_MTREE, "install-mtree");
1617                 dophase(work, &wmsg,
1618                         WDOG5, PHASE_INSTALL, "install");
1619                 dophase(work, &wmsg,
1620                         WDOG5, PHASE_DEINSTALL, "deinstall");
1621 #endif
1622         }
1623
1624         if (MasterPtyFd >= 0) {
1625                 close(MasterPtyFd);
1626                 MasterPtyFd = -1;
1627         }
1628
1629         setproctitle("WORKER [%02d] CLEANUP  %s", slot, portdir);
1630
1631         /*
1632          * Copy the package to the repo.
1633          */
1634         if (work->accum_error == 0) {
1635                 char *b1;
1636                 char *b2;
1637
1638                 asprintf(&b1, "%s/construction/%s/pkg/%s",
1639                          work->basedir, pkg.portdir, pkg.pkgfile);
1640                 asprintf(&b2, "%s/%s", RepositoryPath, pkg.pkgfile);
1641                 if (copyfile(b1, b2)) {
1642                         ++work->accum_error;
1643                         dlog(DLOG_ALL, "[%03d] %s Unable to copy %s to %s\n",
1644                              work->index, pkg.portdir, b1, b2);
1645                 }
1646                 free(b1);
1647                 free(b2);
1648         }
1649
1650         /*
1651          * Unmount, unless we are in DebugStopMode.
1652          */
1653         if (DebugStopMode == 0)
1654                 DoWorkerUnmounts(work);
1655
1656         /*
1657          * Send completion status to master dsynth worker thread.
1658          */
1659         if (work->accum_error) {
1660                 wmsg.cmd = WMSG_CMD_FAILURE;
1661         } else {
1662                 wmsg.cmd = WMSG_CMD_SUCCESS;
1663         }
1664         ipcwritemsg(fd, &wmsg);
1665         if (DebugStopMode) {
1666                 wmsg.cmd = WMSG_CMD_FREEZEWORKER;
1667                 ipcwritemsg(fd, &wmsg);
1668         }
1669 }
1670
1671 static void
1672 dophase(worker_t *work, wmsg_t *wmsg, int wdog, int phaseid, const char *phase)
1673 {
1674         pkg_t *pkg = work->pkg;
1675         char buf[1024];
1676         pid_t pid;
1677         int status;
1678         int ms;
1679         pid_t wpid;
1680         int wpid_reaped;
1681         int fdlog;
1682         time_t start_time;
1683         time_t last_time;
1684         time_t next_time;
1685         time_t wdog_time;
1686         FILE *fp;
1687
1688         if (work->accum_error)
1689                 return;
1690         setproctitle("WORKER [%02d] %8.8s %s",
1691                      work->index, phase, pkg->portdir);
1692         wmsg->phase = phaseid;
1693         if (ipcwritemsg(work->fds[0], wmsg) < 0) {
1694                 dlog(DLOG_ALL, "[%03d] %s Lost Communication with dsynth, "
1695                      "aborting worker\n",
1696                      work->index, pkg->portdir);
1697                 ++work->accum_error;
1698                 return;
1699         }
1700
1701         /*
1702          * Execute the port make command in chroot on a pty
1703          */
1704         fflush(stdout);
1705         fflush(stderr);
1706         if (MasterPtyFd >= 0) {
1707                 int slavefd;
1708
1709                 slavefd = open(ptsname(MasterPtyFd), O_RDWR);
1710                 dassert_errno(slavefd >= 0, "Cannot open slave pty");
1711                 pid = fork();
1712                 if (pid == 0) {
1713                         login_tty(slavefd);
1714                 } else {
1715                         close(slavefd);
1716                 }
1717         } else {
1718                 pid = forkpty(&MasterPtyFd, NULL, NULL, NULL);
1719         }
1720
1721         if (pid == 0) {
1722                 closefrom(3);
1723                 if (chdir(work->basedir) < 0)
1724                         dfatal_errno("chdir in phase initialization");
1725                 if (chroot(work->basedir) < 0)
1726                         dfatal_errno("chroot in phase initialization");
1727
1728                 /*
1729                  * We have a choice here on how to handle stdin (fd 0).
1730                  * We can leave it connected to the pty in which case
1731                  * the build will just block if it tries to ask a
1732                  * question (and the watchdog will kill it, eventually),
1733                  * or we can try to EOF the pty, or we can attach /dev/null
1734                  * to descriptor 0.
1735                  */
1736                 if (NullStdinOpt) {
1737                         int fd;
1738
1739                         fd = open("/dev/null", O_RDWR);
1740                         dassert_errno(fd >= 0, "cannot open /dev/null");
1741                         if (fd != 0) {
1742                                 dup2(fd, 0);
1743                                 close(fd);
1744                         }
1745                 }
1746
1747                 /*
1748                  * Execute the appropriate command.
1749                  */
1750                 switch(phaseid) {
1751                 case PHASE_INSTALL_PKGS:
1752                         snprintf(buf, sizeof(buf), "/tmp/dsynth_install_pkgs");
1753                         execl(buf, buf, NULL);
1754                         break;
1755                 default:
1756                         snprintf(buf, sizeof(buf), "/xports/%s", pkg->portdir);
1757                         execl(MAKE_BINARY, MAKE_BINARY, "-C", buf, phase, NULL);
1758                         break;
1759                 }
1760                 _exit(1);
1761         }
1762         fcntl(MasterPtyFd, F_SETFL, O_NONBLOCK);
1763
1764         if (pid < 0) {
1765                 dlog(DLOG_ALL, "[%03d] %s Fork Failed\n",
1766                      work->index, pkg->logfile);
1767                 ++work->accum_error;
1768                 return;
1769         }
1770
1771         SigPid = pid;
1772
1773         fdlog = open(pkg->logfile, O_RDWR|O_CREAT|O_APPEND, 0644);
1774         if (fdlog < 0) {
1775                 dlog(DLOG_ALL, "[%03d] %s Cannot open logfile '%s': %s\n",
1776                      work->index, pkg->portdir,
1777                      pkg->logfile, strerror(errno));
1778         }
1779         start_time = time(NULL);
1780         last_time = start_time;
1781         wdog_time = start_time;
1782         wpid_reaped = 0;
1783
1784         status = 0;
1785         for (;;) {
1786                 ms = mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time);
1787                 if (ms == MPTY_FAILED) {
1788                         dlog(DLOG_ALL,
1789                              "[%03d] %s lost pty in phase %s, terminating\n",
1790                              work->index, pkg->portdir, phase);
1791                         break;
1792                 }
1793                 if (ms == MPTY_EOF)
1794                         break;
1795
1796                 /*
1797                  * Generally speaking update status once a second.
1798                  * This also allows us to detect if the management
1799                  * dsynth process has gone away.
1800                  */
1801                 next_time = time(NULL);
1802                 if (next_time != last_time) {
1803                         double dload[3];
1804                         double dv;
1805                         int wdog_scaled;
1806
1807                         /*
1808                          * Send status update to the worker management thread
1809                          * in the master dsynth process.  Remember, *WE* are
1810                          * the worker management process sub-fork.
1811                          */
1812                         if (ipcwritemsg(work->fds[0], wmsg) < 0)
1813                                 break;
1814                         last_time = next_time;
1815
1816                         /*
1817                          * Watchdog scaling
1818                          */
1819                         getloadavg(dload, 3);
1820                         dv = dload[2] / NumCores;
1821                         if (dv < (double)NumCores) {
1822                                 wdog_scaled = wdog;
1823                         } else {
1824                                 if (dv > 4.0 * NumCores)
1825                                         dv = 4.0 * NumCores;
1826                                 wdog_scaled = wdog * dv / NumCores;
1827                         }
1828
1829                         /*
1830                          * Watchdog
1831                          */
1832                         if (next_time - wdog_time >= wdog_scaled * 60) {
1833                                 snprintf(buf, sizeof(buf),
1834                                          "\n--------\n"
1835                                          "WATCHDOG TIMEOUT FOR %s in %s "
1836                                          "after %d minutes\n"
1837                                          "Killing pid %d\n"
1838                                          "--------\n",
1839                                          pkg->portdir, phase, wdog_scaled, pid);
1840                                 if (fdlog >= 0)
1841                                         write(fdlog, buf, strlen(buf));
1842                                 dlog(DLOG_ALL,
1843                                      "[%03d] WATCHDOG TIMEOUT FOR %s in %s "
1844                                      "after %d minutes (%d min scaled)\n",
1845                                      work->index, pkg->portdir, phase,
1846                                      wdog, wdog_scaled);
1847                                 kill(pid, SIGKILL);
1848                                 ++work->accum_error;
1849                                 break;
1850                         }
1851                 }
1852
1853                 /*
1854                  * Check process exit.  Normally the pty will EOF
1855                  * but if background processes remain we need to
1856                  * check here to see if our primary exec is done,
1857                  * so we can break out and reap those processes.
1858                  *
1859                  * Generally reap any other processes we have inherited
1860                  * while we are here.
1861                  */
1862                 do {
1863                         wpid = wait3(&status, WNOHANG, NULL);
1864                 } while (wpid > 0 && wpid != pid);
1865                 if (wpid == pid && WIFEXITED(status)) {
1866                         wpid_reaped = 1;
1867                         break;
1868                 }
1869         }
1870
1871         next_time = time(NULL);
1872
1873         setproctitle("WORKER [%02d] EXITREAP %s",
1874                      work->index, pkg->portdir);
1875
1876         /*
1877          * We usually get here due to a mpty EOF, but not always as there
1878          * could be persistent processes still holding the slave.  Finish
1879          * up getting the exit status for the main process we are waiting
1880          * on and clean out any data left on the MasterPtyFd (as it could
1881          * be blocking the exit).
1882          */
1883         while (wpid_reaped == 0) {
1884                 (void)mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time);
1885                 wpid = waitpid(pid, &status, WNOHANG);
1886                 if (wpid == pid && WIFEXITED(status)) {
1887                         wpid_reaped = 1;
1888                         break;
1889                 }
1890                 if (wpid < 0 && errno != EINTR) {
1891                         break;
1892                 }
1893
1894                 /*
1895                  * Safety.  The normal phase waits until the fork/exec'd
1896                  * pid finishes, causing a pty EOF on exit (the slave
1897                  * descriptor is closed by the kernel on exit so the
1898                  * process should already have exited).
1899                  *
1900                  * However, it is also possible to get here if the pty fails
1901                  * for some reason.  In this case, make sure that the process
1902                  * is killed.
1903                  */
1904                 kill(pid, SIGKILL);
1905         }
1906
1907         /*
1908          * Clean out anything left on the pty but don't wait around
1909          * because there could be background processes preventing the
1910          * slave side from closing.
1911          */
1912         while (mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time) == MPTY_DATA)
1913                 ;
1914
1915         /*
1916          * Report on the exit condition.  If the pid was somehow lost
1917          * (probably due to someone gdb'ing the process), assume an error.
1918          */
1919         if (wpid_reaped) {
1920                 if (WEXITSTATUS(status)) {
1921                         dlog(DLOG_ALL, "[%03d] %s Build phase '%s' "
1922                                        "failed exit %d\n",
1923                              work->index, pkg->portdir, phase,
1924                              WEXITSTATUS(status));
1925                         ++work->accum_error;
1926                 }
1927         } else {
1928                 dlog(DLOG_ALL, "[%03d] %s Build phase '%s' failed - lost pid\n",
1929                      work->index, pkg->portdir, phase);
1930                 ++work->accum_error;
1931         }
1932
1933         /*
1934          * Kill any processes still running (sometimes processes end up in
1935          * the background during a dports build), and clean up any other
1936          * children that we have inherited.
1937          */
1938         phaseReapAll();
1939
1940         /*
1941          * Update log
1942          */
1943         if (fdlog >= 0) {
1944                 struct stat st;
1945                 int h;
1946                 int m;
1947                 int s;
1948
1949                 last_time = next_time - start_time;
1950                 s = last_time % 60;
1951                 m = last_time / 60 % 60;
1952                 h = last_time / 3600;
1953
1954                 fp = fdopen(fdlog, "a");
1955                 if (fp == NULL) {
1956                         dlog(DLOG_ALL, "[%03d] %s Cannot fdopen fdlog: %s %d\n",
1957                              work->index, pkg->portdir,
1958                              strerror(errno), fstat(fdlog, &st));
1959                         close(fdlog);
1960                         goto skip;
1961                 }
1962
1963                 fprintf(fp, "--------\n");
1964                 if (work->accum_error) {
1965                         fprintf(fp, "PHASE %s FAILED %02d:%02d:%02d\n",
1966                                 phase, h, m, s);
1967                 } else {
1968                         fprintf(fp, "PHASE %s SUCCEEDED %02d:%02d:%02d\n",
1969                                 phase, h, m, s);
1970                 }
1971                 last_time = next_time - work->start_time;
1972                 s = last_time % 60;
1973                 m = last_time / 60 % 60;
1974                 h = last_time / 3600;
1975                 if (phaseid == PHASE_PACKAGE) {
1976                         fprintf(fp, "TOTAL TIME %02d:%02d:%02d\n", h, m, s);
1977                 }
1978                 fprintf(fp, "--------\n");
1979                 fclose(fp);
1980 skip:
1981                 ;
1982         }
1983
1984 }
1985
1986 static void
1987 phaseReapAll(void)
1988 {
1989         struct reaper_status rs;
1990         int status;
1991
1992         while (procctl(P_PID, getpid(), PROC_REAP_STATUS, &rs) == 0) {
1993                 if ((rs.flags & PROC_REAP_ACQUIRE) == 0)
1994                         break;
1995                 if (rs.pid_head < 0)
1996                         break;
1997                 if (kill(rs.pid_head, SIGKILL) == 0) {
1998                         while (waitpid(rs.pid_head, &status, 0) < 0)
1999                                 ;
2000                 }
2001         }
2002         while (wait3(&status, 0, NULL) > 0)
2003                 ;
2004 }
2005
2006 static void
2007 phaseTerminateSignal(int sig __unused)
2008 {
2009         if (MasterPtyFd >= 0)
2010                 close(MasterPtyFd);
2011         if (SigPid > 1)
2012                 kill(SigPid, SIGKILL);
2013         phaseReapAll();
2014         if (SigWork)
2015                 DoWorkerUnmounts(SigWork);
2016         exit(1);
2017 }
2018
2019 static
2020 char *
2021 buildskipreason(pkglink_t *parent, pkg_t *pkg)
2022 {
2023         pkglink_t *link;
2024         pkg_t *scan;
2025         char *reason = NULL;
2026         char *ptr;
2027         size_t tot;
2028         size_t len;
2029         pkglink_t stack;
2030
2031         tot = 0;
2032         PKGLIST_FOREACH(link, &pkg->idepon_list) {
2033                 scan = link->pkg;
2034                 if (scan == NULL)
2035                         continue;
2036                 if ((scan->flags & (PKGF_ERROR | PKGF_NOBUILD)) == 0)
2037                         continue;
2038                 if (scan->flags & PKGF_NOBUILD) {
2039                         stack.pkg = scan;
2040                         stack.next = parent;
2041                         ptr = buildskipreason(&stack, scan);
2042                         len = strlen(scan->portdir) + strlen(ptr) + 8;
2043                         reason = realloc(reason, tot + len);
2044                         snprintf(reason + tot, len, "%s->%s",
2045                                  scan->portdir, ptr);
2046                         free(ptr);
2047                 } else {
2048                         len = strlen(scan->portdir) + 8;
2049                         reason = realloc(reason, tot + len);
2050                         snprintf(reason + tot, len, "%s", scan->portdir);
2051                 }
2052
2053                 /*
2054                  * Don't try to print the entire graph
2055                  */
2056                 if (parent)
2057                         break;
2058                 tot += strlen(reason + tot);
2059                 reason[tot++] = ' ';
2060                 reason[tot] = 0;
2061         }
2062         return (reason);
2063 }
2064
2065 /*
2066  * The master ptyfd is in non-blocking mode.  Drain up to 1024 bytes
2067  * and update wmsg->lines and *wdog_timep as appropriate.
2068  *
2069  * This function will poll, stalling up to 1 second.
2070  */
2071 static int
2072 mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg, time_t *wdog_timep)
2073 {
2074         struct pollfd pfd;
2075         char buf[1024];
2076         ssize_t r;
2077
2078         pfd.fd = ptyfd;
2079         pfd.events = POLLIN;
2080         pfd.revents = 0;
2081
2082         poll(&pfd, 1, 1000);
2083         if (pfd.revents) {
2084                 r = read(ptyfd, buf, sizeof(buf));
2085                 if (r > 0) {
2086                         *wdog_timep = time(NULL);
2087                         if (r > 0 && fdlog >= 0)
2088                                 write(fdlog, buf, r);
2089                         while (--r >= 0) {
2090                                 if (buf[r] == '\n')
2091                                         ++wmsg->lines;
2092                         }
2093                         return MPTY_DATA;
2094                         return 1;
2095                 } else if (r < 0) {
2096                         if (errno != EINTR && errno != EAGAIN)
2097                                 return MPTY_FAILED;
2098                         return MPTY_AGAIN;
2099                 } else if (r == 0) {
2100                         return MPTY_EOF;
2101                 }
2102         }
2103         return MPTY_AGAIN;
2104 }
2105
2106 /*
2107  * Copy a (package) file from (src) to (dst), use an intermediate file and
2108  * rename to ensure that interruption does not leave us with a corrupt
2109  * package file.
2110  *
2111  * This is called by the WORKER process.
2112  *
2113  * (dsynth management thread -> WORKER process -> sub-processes)
2114  */
2115 #define COPYBLKSIZE     32768
2116
2117 static int
2118 copyfile(char *src, char *dst)
2119 {
2120         char *tmp;
2121         char *buf;
2122         int fd1;
2123         int fd2;
2124         int error = 0;
2125         ssize_t r;
2126
2127         asprintf(&tmp, "%s.new", dst);
2128         buf = malloc(COPYBLKSIZE);
2129
2130         fd1 = open(src, O_RDONLY|O_CLOEXEC);
2131         fd2 = open(tmp, O_RDWR|O_CREAT|O_TRUNC|O_CLOEXEC, 0644);
2132         while ((r = read(fd1, buf, COPYBLKSIZE)) > 0) {
2133                 if (write(fd2, buf, r) != r)
2134                         error = 1;
2135         }
2136         if (r < 0)
2137                 error = 1;
2138         close(fd1);
2139         close(fd2);
2140         if (error) {
2141                 remove(tmp);
2142         } else {
2143                 if (rename(tmp, dst)) {
2144                         error = 1;
2145                         remove(tmp);
2146                 }
2147         }
2148
2149         free(buf);
2150         free(tmp);
2151
2152         return error;
2153 }