Commit | Line | Data |
---|---|---|
8e25f19b | 1 | /* |
3bd7e0a7 | 2 | * Copyright (c) 2019-2022 The DragonFly Project. All rights reserved. |
8e25f19b MD |
3 | * |
4 | * This code is derived from software contributed to The DragonFly Project | |
5 | * by Matthew Dillon <dillon@backplane.com> | |
6 | * | |
7 | * This code uses concepts and configuration based on 'synth', by | |
8 | * John R. Marino <draco@marino.st>, which was written in ada. | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or without | |
11 | * modification, are permitted provided that the following conditions | |
12 | * are met: | |
13 | * | |
14 | * 1. Redistributions of source code must retain the above copyright | |
15 | * notice, this list of conditions and the following disclaimer. | |
16 | * 2. Redistributions in binary form must reproduce the above copyright | |
17 | * notice, this list of conditions and the following disclaimer in | |
18 | * the documentation and/or other materials provided with the | |
19 | * distribution. | |
20 | * 3. Neither the name of The DragonFly Project nor the names of its | |
21 | * contributors may be used to endorse or promote products derived | |
22 | * from this software without specific, prior written permission. | |
23 | * | |
24 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
25 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
26 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS | |
27 | * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE | |
28 | * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, | |
29 | * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, | |
30 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; | |
31 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED | |
32 | * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, | |
33 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT | |
34 | * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |
35 | * SUCH DAMAGE. | |
36 | */ | |
77d66d4e AHJ |
37 | |
38 | #include <netdb.h> | |
39 | ||
8e25f19b MD |
40 | #include "dsynth.h" |
41 | ||
d50f9ae3 SW |
42 | static worker_t WorkerAry[MAXWORKERS]; |
43 | static int BuildInitialized; | |
44 | static int RunningWorkers; | |
f7f25838 | 45 | int DynamicMaxWorkers; |
d50f9ae3 SW |
46 | static int FailedWorkers; |
47 | static long RunningPkgDepSize; | |
7b5ce327 | 48 | static pthread_mutex_t DbmMutex; |
d50f9ae3 SW |
49 | static pthread_mutex_t WorkerMutex; |
50 | static pthread_cond_t WorkerCond; | |
8e25f19b | 51 | |
1fa9d809 MD |
52 | static int build_find_leaves(pkg_t *parent, pkg_t *pkg, |
53 | pkg_t ***build_tailp, int *app, int *hasworkp, | |
54 | int depth, int first, int first_one_only); | |
88c24d72 | 55 | static int buildCalculateDepiDepth(pkg_t *pkg); |
8e25f19b MD |
56 | static void build_clear_trav(pkg_t *pkg); |
57 | static void startbuild(pkg_t **build_listp, pkg_t ***build_tailp); | |
58 | static int qsort_depi(const void *pkg1, const void *pkg2); | |
59 | static int qsort_idep(const void *pkg1, const void *pkg2); | |
60 | static void startworker(pkg_t *pkg, worker_t *work); | |
61 | static void cleanworker(worker_t *work); | |
3699ee09 | 62 | static void waitbuild(int whilematch, int dynamicmax); |
8e25f19b MD |
63 | static void workercomplete(worker_t *work); |
64 | static void *childBuilderThread(void *arg); | |
65 | static int childInstallPkgDeps(worker_t *work); | |
63fcce5b | 66 | static size_t childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list, |
1fa9d809 | 67 | int undoit, int depth, int first_one_only); |
8e25f19b MD |
68 | static void dophase(worker_t *work, wmsg_t *wmsg, |
69 | int wdog, int phaseid, const char *phase); | |
70 | static void phaseReapAll(void); | |
71 | static void phaseTerminateSignal(int sig); | |
3699ee09 | 72 | static char *buildskipreason(pkglink_t *parent, pkg_t *pkg); |
3dd48cfa | 73 | static int buildskipcount_dueto(pkg_t *pkg, int mode); |
7880dcf7 MD |
74 | static int mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg, |
75 | time_t *wdog_timep); | |
68dc2eea MD |
76 | static void doHook(pkg_t *pkg, const char *id, const char *path, int waitfor); |
77 | static void childHookRun(bulk_t *bulk); | |
32fc5bbf | 78 | static void adjloadavg(double *dload); |
b6bd007b | 79 | static void check_packaged(const char *dbmpath, pkg_t *pkgs); |
8e25f19b MD |
80 | |
81 | static worker_t *SigWork; | |
1d6e00cd | 82 | static int MasterPtyFd = -1; |
fef2fc63 | 83 | static int CopyFileFd = -1; |
8e25f19b | 84 | static pid_t SigPid; |
ffc851f6 | 85 | static const char *WorkerFlavorPrt = ""; /* "" or "@flavor" */ |
b6bd007b | 86 | static DBM *CheckDBM; |
8e25f19b | 87 | |
7880dcf7 MD |
88 | #define MPTY_FAILED -2 |
89 | #define MPTY_AGAIN -1 | |
90 | #define MPTY_EOF 0 | |
91 | #define MPTY_DATA 1 | |
92 | ||
8e25f19b MD |
93 | int BuildTotal; |
94 | int BuildCount; | |
95 | int BuildSkipCount; | |
87017ac4 | 96 | int BuildIgnoreCount; |
8e25f19b MD |
97 | int BuildFailCount; |
98 | int BuildSuccessCount; | |
549987f1 | 99 | int BuildMissingCount; |
51705b28 | 100 | int BuildMetaCount; |
dca89e44 | 101 | int PkgVersionPkgSuffix; |
8e25f19b MD |
102 | |
103 | /* | |
104 | * Initialize the WorkerAry[] | |
105 | */ | |
106 | void | |
107 | DoInitBuild(int slot_override) | |
108 | { | |
109 | worker_t *work; | |
110 | struct stat st; | |
5d0718a8 | 111 | char *path; |
8e25f19b MD |
112 | int i; |
113 | ||
114 | ddassert(slot_override < 0 || MaxWorkers == 1); | |
115 | ||
116 | bzero(WorkerAry, MaxWorkers * sizeof(worker_t)); | |
117 | pthread_mutex_init(&WorkerMutex, NULL); | |
7b5ce327 | 118 | pthread_mutex_init(&DbmMutex, NULL); |
8e25f19b MD |
119 | |
120 | for (i = 0; i < MaxWorkers; ++i) { | |
121 | work = &WorkerAry[i]; | |
122 | work->index = (slot_override >= 0) ? slot_override : i; | |
123 | work->state = WORKER_NONE; | |
124 | asprintf(&work->basedir, "%s/SL%02d", BuildBase, work->index); | |
125 | pthread_cond_init(&work->cond, NULL); | |
126 | } | |
127 | BuildCount = 0; | |
128 | ||
129 | /* | |
130 | * Create required sub-directories. The base directories must already | |
131 | * exist as a dsynth configuration safety. | |
132 | */ | |
133 | if (stat(RepositoryPath, &st) < 0) { | |
134 | if (mkdir(RepositoryPath, 0755) < 0) | |
135 | dfatal("Cannot mkdir %s\n", RepositoryPath); | |
136 | } | |
137 | ||
5d0718a8 MD |
138 | /* |
139 | * An empty directory for LOCALBASE= in the pkg scan. | |
140 | */ | |
141 | asprintf(&path, "%s/empty", BuildBase); | |
142 | if (stat(path, &st) < 0) { | |
143 | if (mkdir(path, 0755) < 0) | |
144 | dfatal("Cannot mkdir %s\n", path); | |
145 | } | |
146 | free(path); | |
147 | ||
8e25f19b | 148 | BuildInitialized = 1; |
f7f25838 MD |
149 | |
150 | /* | |
151 | * slow-start (increases at a rate of 1 per 5 seconds) | |
152 | */ | |
153 | if (SlowStartOpt > MaxWorkers) | |
154 | DynamicMaxWorkers = MaxWorkers; | |
155 | else if (SlowStartOpt > 0) | |
156 | DynamicMaxWorkers = SlowStartOpt; | |
157 | else | |
158 | DynamicMaxWorkers = MaxWorkers; | |
8e25f19b MD |
159 | } |
160 | ||
161 | /* | |
162 | * Called by the frontend to clean-up any hanging mounts. | |
163 | */ | |
164 | void | |
87017ac4 | 165 | DoCleanBuild(int resetlogs) |
8e25f19b MD |
166 | { |
167 | int i; | |
168 | ||
169 | ddassert(BuildInitialized); | |
170 | ||
87017ac4 MD |
171 | if (resetlogs) |
172 | dlogreset(); | |
8e25f19b MD |
173 | for (i = 0; i < MaxWorkers; ++i) { |
174 | DoWorkerUnmounts(&WorkerAry[i]); | |
175 | } | |
8e25f19b MD |
176 | } |
177 | ||
178 | void | |
179 | DoBuild(pkg_t *pkgs) | |
180 | { | |
181 | pkg_t *build_list = NULL; | |
182 | pkg_t **build_tail = &build_list; | |
183 | pkg_t *scan; | |
68dc2eea | 184 | bulk_t *bulk; |
8e25f19b MD |
185 | int haswork = 1; |
186 | int first = 1; | |
6fd67931 | 187 | int newtemplate; |
f4094b20 MD |
188 | time_t startTime; |
189 | time_t t; | |
190 | int h, m, s; | |
b6bd007b | 191 | char *dbmpath; |
8e25f19b | 192 | |
68dc2eea MD |
193 | /* |
194 | * We use our bulk system to run hooks. This will be for | |
195 | * Skipped and Ignored. Success and Failure are handled by | |
196 | * WorkerProcess() which is a separately-exec'd dsynth. | |
197 | */ | |
198 | if (UsingHooks) | |
199 | initbulk(childHookRun, MaxBulk); | |
200 | ||
7b609b0a MD |
201 | /* |
202 | * Count up the packages, not counting dummies | |
203 | */ | |
204 | for (scan = pkgs; scan; scan = scan->bnext) { | |
205 | if ((scan->flags & PKGF_DUMMY) == 0) | |
206 | ++BuildTotal; | |
207 | } | |
8e25f19b | 208 | |
b6bd007b MD |
209 | /* |
210 | * Remove binary package files for dports whos directory | |
211 | * has changed. | |
212 | */ | |
213 | asprintf(&dbmpath, "%s/ports_crc", BuildBase); | |
214 | CheckDBM = dbm_open(dbmpath, O_CREAT|O_RDWR, 0644); | |
215 | check_packaged(dbmpath, pkgs); | |
216 | ||
68dc2eea MD |
217 | doHook(NULL, "hook_run_start", HookRunStart, 1); |
218 | ||
8e25f19b MD |
219 | /* |
220 | * The pkg and pkg-static binaries are needed. If already present | |
549987f1 MD |
221 | * then assume that the template is also valid, otherwise add to |
222 | * the list and build both. | |
8e25f19b | 223 | */ |
549987f1 | 224 | scan = GetPkgPkg(&pkgs); |
8e25f19b MD |
225 | |
226 | /* | |
1645cafe MD |
227 | * Create our template. The template will be missing pkg |
228 | * and pkg-static. | |
8e25f19b | 229 | */ |
325ef124 MD |
230 | if (FetchOnlyOpt) { |
231 | newtemplate = DoCreateTemplate(0); | |
232 | } else if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0) { | |
6fd67931 MD |
233 | /* force a fresh template */ |
234 | newtemplate = DoCreateTemplate(1); | |
1d6e00cd | 235 | } else { |
6fd67931 | 236 | newtemplate = DoCreateTemplate(0); |
1645cafe MD |
237 | } |
238 | ||
239 | /* | |
240 | * This will clear the screen and set-up our gui, so sleep | |
241 | * a little first in case the user wants to see what was | |
242 | * printed before. | |
243 | */ | |
244 | sleep(2); | |
8e25f19b | 245 | pthread_mutex_lock(&WorkerMutex); |
f4094b20 | 246 | startTime = time(NULL); |
ea37671d MD |
247 | RunStatsInit(); |
248 | RunStatsReset(); | |
8e25f19b | 249 | |
6fd67931 | 250 | /* |
dca89e44 | 251 | * Build pkg/pkg-static |
6fd67931 | 252 | */ |
325ef124 | 253 | if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0 && FetchOnlyOpt == 0) { |
8e25f19b MD |
254 | build_list = scan; |
255 | build_tail = &scan->build_next; | |
256 | startbuild(&build_list, &build_tail); | |
3699ee09 MD |
257 | while (RunningWorkers == 1) |
258 | waitbuild(1, 0); | |
8e25f19b MD |
259 | |
260 | if (scan->flags & PKGF_NOBUILD) | |
261 | dfatal("Unable to build 'pkg'"); | |
262 | if (scan->flags & PKGF_ERROR) | |
263 | dfatal("Error building 'pkg'"); | |
264 | if ((scan->flags & PKGF_SUCCESS) == 0) | |
265 | dfatal("Error building 'pkg'"); | |
6fd67931 MD |
266 | newtemplate = 1; |
267 | } | |
268 | ||
269 | /* | |
270 | * Install pkg/pkg-static into the template | |
271 | */ | |
325ef124 | 272 | if (newtemplate && FetchOnlyOpt == 0) { |
6fd67931 MD |
273 | char *buf; |
274 | int rc; | |
8e25f19b | 275 | |
8e25f19b MD |
276 | asprintf(&buf, |
277 | "cd %s/Template; " | |
278 | "tar --exclude '+*' --exclude '*/man/*' " | |
1645cafe | 279 | "-xvzpf %s/%s > /dev/null 2>&1", |
8e25f19b MD |
280 | BuildBase, |
281 | RepositoryPath, | |
282 | scan->pkgfile); | |
283 | rc = system(buf); | |
284 | if (rc) | |
285 | dfatal("Command failed: %s\n", buf); | |
ffc851f6 | 286 | freestrp(&buf); |
8e25f19b MD |
287 | } |
288 | ||
dca89e44 MD |
289 | /* |
290 | * Figure out pkg version | |
291 | */ | |
292 | if (scan->version) { | |
293 | int v1; | |
294 | int v2; | |
295 | ||
296 | dlog(DLOG_ALL, "[XXX] pkg version %s\n", scan->version); | |
297 | if (sscanf(scan->version, "%d.%d", &v1, &v2) == 2) { | |
298 | if ((v1 == 1 && v2 >= 17) || v1 > 1) | |
299 | PkgVersionPkgSuffix = 1; | |
300 | } | |
301 | } | |
302 | ||
88c24d72 MD |
303 | /* |
304 | * Calculate depi_depth, the longest chain of dependencies | |
305 | * for who depends on me, weighted by powers of two. | |
306 | */ | |
307 | for (scan = pkgs; scan; scan = scan->bnext) { | |
308 | buildCalculateDepiDepth(scan); | |
309 | } | |
310 | ||
8e25f19b MD |
311 | /* |
312 | * Nominal bulk build sequence | |
313 | */ | |
314 | while (haswork) { | |
315 | haswork = 0; | |
316 | fflush(stdout); | |
317 | for (scan = pkgs; scan; scan = scan->bnext) { | |
318 | ddprintf(0, "SCANLEAVES %08x %s\n", | |
319 | scan->flags, scan->portdir); | |
320 | scan->flags |= PKGF_BUILDLOOP; | |
321 | /* | |
322 | * NOTE: We must still find dependencies if PACKAGED | |
323 | * to fill in the gaps, as some of them may | |
324 | * need to be rebuilt. | |
325 | */ | |
326 | if (scan->flags & (PKGF_SUCCESS | PKGF_FAILURE | | |
3699ee09 | 327 | PKGF_ERROR)) { |
8e25f19b MD |
328 | #if 0 |
329 | ddprintf(0, "%s: already built\n", | |
330 | scan->portdir); | |
331 | #endif | |
332 | } else { | |
333 | int ap = 0; | |
334 | build_find_leaves(NULL, scan, &build_tail, | |
1fa9d809 | 335 | &ap, &haswork, 0, first, 0); |
8e25f19b MD |
336 | ddprintf(0, "TOPLEVEL %s %08x\n", |
337 | scan->portdir, ap); | |
338 | } | |
339 | scan->flags &= ~PKGF_BUILDLOOP; | |
340 | build_clear_trav(scan); | |
341 | } | |
342 | first = 0; | |
343 | fflush(stdout); | |
344 | startbuild(&build_list, &build_tail); | |
345 | ||
346 | if (haswork == 0 && RunningWorkers) { | |
3699ee09 | 347 | waitbuild(RunningWorkers, 1); |
8e25f19b MD |
348 | haswork = 1; |
349 | } | |
350 | } | |
351 | pthread_mutex_unlock(&WorkerMutex); | |
352 | ||
549987f1 MD |
353 | /* |
354 | * What is left that cannot be built? The list really ought to be | |
355 | * empty at this point, report anything that remains. | |
356 | */ | |
357 | for (scan = pkgs; scan; scan = scan->bnext) { | |
358 | if (scan->flags & (PKGF_SUCCESS | PKGF_FAILURE)) | |
359 | continue; | |
360 | dlog(DLOG_ABN, "[XXX] %s lost in the ether [flags=%08x]\n", | |
361 | scan->portdir, scan->flags); | |
362 | ++BuildMissingCount; | |
363 | } | |
364 | ||
365 | /* | |
366 | * Final updates | |
367 | */ | |
8b485838 | 368 | RunStatsUpdateTop(0); |
ea37671d MD |
369 | RunStatsUpdateLogs(); |
370 | RunStatsSync(); | |
371 | RunStatsDone(); | |
3699ee09 | 372 | |
68dc2eea MD |
373 | doHook(NULL, "hook_run_end", HookRunEnd, 1); |
374 | if (UsingHooks) { | |
375 | while ((bulk = getbulk()) != NULL) | |
376 | freebulk(bulk); | |
377 | donebulk(); | |
378 | } | |
379 | ||
f4094b20 MD |
380 | t = time(NULL) - startTime; |
381 | h = t / 3600; | |
382 | m = t / 60 % 60; | |
383 | s = t % 60; | |
384 | ||
b6bd007b MD |
385 | if (CheckDBM) { |
386 | dbm_close(CheckDBM); | |
387 | CheckDBM = NULL; | |
388 | } | |
389 | ||
33170e2d MD |
390 | dlog(DLOG_ALL|DLOG_STDOUT, |
391 | "\n" | |
392 | "Initial queue size: %d\n" | |
393 | " packages built: %d\n" | |
394 | " ignored: %d\n" | |
395 | " skipped: %d\n" | |
396 | " failed: %d\n" | |
549987f1 | 397 | " missing: %d\n" |
51705b28 | 398 | " meta-nodes: %d\n" |
33170e2d MD |
399 | "\n" |
400 | "Duration: %02d:%02d:%02d\n" | |
401 | "\n", | |
402 | BuildTotal, | |
403 | BuildSuccessCount, | |
404 | BuildIgnoreCount, | |
405 | BuildSkipCount, | |
406 | BuildFailCount, | |
549987f1 | 407 | BuildMissingCount, |
51705b28 | 408 | BuildMetaCount, |
33170e2d | 409 | h, m, s); |
8e25f19b MD |
410 | } |
411 | ||
412 | /* | |
413 | * Traverse the packages (pkg) depends on recursively until we find | |
414 | * a leaf to build or report as unbuildable. Calculates and assigns a | |
415 | * dependency count. Returns all parallel-buildable packages. | |
416 | * | |
417 | * (pkg) itself is only added to the list if it is immediately buildable. | |
418 | */ | |
419 | static | |
420 | int | |
421 | build_find_leaves(pkg_t *parent, pkg_t *pkg, pkg_t ***build_tailp, | |
1fa9d809 MD |
422 | int *app, int *hasworkp, int depth, int first, |
423 | int first_one_only) | |
8e25f19b MD |
424 | { |
425 | pkglink_t *link; | |
426 | pkg_t *scan; | |
427 | int idep_count = 0; | |
428 | int apsub; | |
1fa9d809 MD |
429 | int dfirst_one_only; |
430 | int ndepth; | |
8e25f19b MD |
431 | char *buf; |
432 | ||
1fa9d809 MD |
433 | ndepth = depth + 1; |
434 | ||
8e25f19b MD |
435 | /* |
436 | * Already on build list, possibly in-progress, tell caller that | |
437 | * it is not ready. | |
438 | */ | |
1fa9d809 MD |
439 | ddprintf(ndepth, "sbuild_find_leaves %d %s %08x {\n", |
440 | depth, pkg->portdir, pkg->flags); | |
8e25f19b | 441 | if (pkg->flags & PKGF_BUILDLIST) { |
1fa9d809 | 442 | ddprintf(ndepth, "} (already on build list)\n"); |
8e25f19b MD |
443 | *app |= PKGF_NOTREADY; |
444 | return (pkg->idep_count); | |
445 | } | |
446 | ||
447 | /* | |
448 | * Check dependencies | |
449 | */ | |
8e25f19b MD |
450 | PKGLIST_FOREACH(link, &pkg->idepon_list) { |
451 | scan = link->pkg; | |
452 | ||
1fa9d809 MD |
453 | if (scan == NULL) { |
454 | if (first_one_only) | |
455 | break; | |
8e25f19b | 456 | continue; |
1fa9d809 MD |
457 | } |
458 | ddprintf(ndepth, "check %s %08x\t", scan->portdir, scan->flags); | |
459 | ||
460 | /* | |
461 | * If this dependency is to a DUMMY node it is a dependency | |
462 | * only on the default flavor which is only the first node | |
463 | * under this one, not all of them. | |
464 | * | |
349e3a76 MD |
465 | * DUMMY nodes can be marked SUCCESS so the build skips past |
466 | * them, but this doesn't mean that their sub-nodes succeeded. | |
467 | * We have to check, so recurse even if it is marked | |
468 | * successful. | |
469 | * | |
1fa9d809 MD |
470 | * NOTE: The depth is not being for complex dependency type |
471 | * tests like it is in childInstallPkgDeps_recurse(), | |
472 | * so we don't have to hicup it like we do in that | |
473 | * procedure. | |
474 | */ | |
475 | dfirst_one_only = (scan->flags & PKGF_DUMMY) ? 1 : 0; | |
349e3a76 MD |
476 | if (dfirst_one_only) |
477 | goto skip_to_flavor; | |
8e25f19b MD |
478 | |
479 | /* | |
480 | * When accounting for a successful build, just bump | |
481 | * idep_count by one. scan->idep_count will heavily | |
482 | * overlap packages that we count down multiple branches. | |
483 | * | |
484 | * We must still recurse through PACKAGED packages as | |
485 | * some of their dependencies might be missing. | |
486 | */ | |
487 | if (scan->flags & PKGF_SUCCESS) { | |
488 | ddprintf(0, "SUCCESS - OK\n"); | |
489 | ++idep_count; | |
1fa9d809 MD |
490 | if (first_one_only) |
491 | break; | |
8e25f19b MD |
492 | continue; |
493 | } | |
9c4c701f MD |
494 | |
495 | /* | |
496 | * ERROR includes FAILURE, which is set in numerous situations | |
549987f1 MD |
497 | * including when NOBUILD state is finally processed. So |
498 | * check for NOBUILD state first. | |
9c4c701f MD |
499 | * |
500 | * An ERROR in a sub-package causes a NOBUILD in packages | |
501 | * that depend on it. | |
502 | */ | |
8e25f19b MD |
503 | if (scan->flags & PKGF_NOBUILD) { |
504 | ddprintf(0, "NOBUILD - OK " | |
21265a85 | 505 | "(propagate failure upward)\n"); |
8e25f19b | 506 | *app |= PKGF_NOBUILD_S; |
1fa9d809 MD |
507 | if (first_one_only) |
508 | break; | |
8e25f19b MD |
509 | continue; |
510 | } | |
9c4c701f | 511 | if (scan->flags & PKGF_ERROR) { |
21265a85 | 512 | ddprintf(0, "ERROR - OK (propagate failure upward)\n"); |
9c4c701f | 513 | *app |= PKGF_NOBUILD_S; |
1fa9d809 MD |
514 | if (first_one_only) |
515 | break; | |
9c4c701f MD |
516 | continue; |
517 | } | |
8e25f19b MD |
518 | |
519 | /* | |
520 | * If already on build-list this dependency is not ready. | |
521 | */ | |
522 | if (scan->flags & PKGF_BUILDLIST) { | |
523 | ddprintf(0, " [BUILDLIST]"); | |
524 | *app |= PKGF_NOTREADY; | |
525 | } | |
526 | ||
527 | /* | |
528 | * If not packaged this dependency is not ready for | |
529 | * the caller. | |
530 | */ | |
531 | if ((scan->flags & PKGF_PACKAGED) == 0) { | |
532 | ddprintf(0, " [NOT_PACKAGED]"); | |
533 | *app |= PKGF_NOTREADY; | |
534 | } | |
535 | ||
536 | /* | |
537 | * Reduce search complexity, if we have already processed | |
538 | * scan in the traversal it will either already be on the | |
539 | * build list or it will not be buildable. Either way | |
540 | * the parent is not buildable. | |
541 | */ | |
542 | if (scan->flags & PKGF_BUILDTRAV) { | |
543 | ddprintf(0, " [BUILDTRAV]\n"); | |
544 | *app |= PKGF_NOTREADY; | |
1fa9d809 MD |
545 | if (first_one_only) |
546 | break; | |
8e25f19b MD |
547 | continue; |
548 | } | |
50cde9e9 | 549 | skip_to_flavor: |
8e25f19b MD |
550 | |
551 | /* | |
552 | * Assert on dependency loop | |
553 | */ | |
554 | ++idep_count; | |
555 | if (scan->flags & PKGF_BUILDLOOP) { | |
556 | dfatal("pkg dependency loop %s -> %s", | |
557 | parent->portdir, scan->portdir); | |
558 | } | |
1fa9d809 MD |
559 | |
560 | /* | |
561 | * NOTE: For debug tabbing purposes we use (ndepth + 1) | |
562 | * here (i.e. depth + 2) in our iteration. | |
563 | */ | |
8e25f19b MD |
564 | scan->flags |= PKGF_BUILDLOOP; |
565 | apsub = 0; | |
566 | ddprintf(0, " SUBRECURSION {\n"); | |
567 | idep_count += build_find_leaves(pkg, scan, build_tailp, | |
568 | &apsub, hasworkp, | |
1fa9d809 MD |
569 | ndepth + 1, first, |
570 | dfirst_one_only); | |
8e25f19b MD |
571 | scan->flags &= ~PKGF_BUILDLOOP; |
572 | *app |= apsub; | |
573 | if (apsub & PKGF_NOBUILD) { | |
1fa9d809 | 574 | ddprintf(ndepth, "} (sub-nobuild)\n"); |
8e25f19b | 575 | } else if (apsub & PKGF_ERROR) { |
1fa9d809 | 576 | ddprintf(ndepth, "} (sub-error)\n"); |
8e25f19b | 577 | } else if (apsub & PKGF_NOTREADY) { |
1fa9d809 | 578 | ddprintf(ndepth, "} (sub-notready)\n"); |
8e25f19b | 579 | } else { |
1fa9d809 | 580 | ddprintf(ndepth, "} (sub-ok)\n"); |
8e25f19b | 581 | } |
1fa9d809 MD |
582 | if (first_one_only) |
583 | break; | |
8e25f19b | 584 | } |
8e25f19b MD |
585 | pkg->idep_count = idep_count; |
586 | pkg->flags |= PKGF_BUILDTRAV; | |
587 | ||
588 | /* | |
589 | * Incorporate scan results into pkg state. | |
590 | */ | |
591 | if ((pkg->flags & PKGF_NOBUILD) == 0 && (*app & PKGF_NOBUILD)) { | |
592 | *hasworkp = 1; | |
593 | } else if ((pkg->flags & PKGF_ERROR) == 0 && (*app & PKGF_ERROR)) { | |
594 | *hasworkp = 1; | |
595 | } | |
596 | pkg->flags |= *app & ~PKGF_NOTREADY; | |
597 | ||
598 | /* | |
bd0c9b9e MD |
599 | * Clear the PACKAGED bit if sub-dependencies aren't clean. |
600 | * | |
601 | * NOTE: PKGF_NOTREADY is not stored in pkg->flags, only in *app, | |
602 | * so incorporate *app to test for it. | |
8e25f19b MD |
603 | */ |
604 | if ((pkg->flags & PKGF_PACKAGED) && | |
bd0c9b9e | 605 | ((pkg->flags | *app) & (PKGF_NOTREADY|PKGF_ERROR|PKGF_NOBUILD))) { |
8e25f19b MD |
606 | pkg->flags &= ~PKGF_PACKAGED; |
607 | ddassert(pkg->pkgfile); | |
608 | asprintf(&buf, "%s/%s", RepositoryPath, pkg->pkgfile); | |
b6bd007b MD |
609 | if (OverridePkgDeleteOpt >= 1) { |
610 | pkg->flags |= PKGF_PACKAGED; | |
611 | dlog(DLOG_ALL, | |
612 | "[XXX] %s DELETE-PACKAGE %s " | |
613 | "(OVERRIDE, NOT DELETED)\n", | |
614 | pkg->portdir, buf); | |
615 | } else if (remove(buf) < 0) { | |
8e25f19b MD |
616 | dlog(DLOG_ALL, |
617 | "[XXX] %s DELETE-PACKAGE %s (failed)\n", | |
618 | pkg->portdir, buf); | |
619 | } else { | |
620 | dlog(DLOG_ALL, | |
621 | "[XXX] %s DELETE-PACKAGE %s " | |
622 | "(due to dependencies)\n", | |
623 | pkg->portdir, buf); | |
624 | } | |
ffc851f6 | 625 | freestrp(&buf); |
8e25f19b MD |
626 | *hasworkp = 1; |
627 | } | |
628 | ||
87017ac4 MD |
629 | /* |
630 | * Set PKGF_NOBUILD_I if there is IGNORE data | |
631 | */ | |
549987f1 | 632 | if (pkg->ignore) { |
87017ac4 | 633 | pkg->flags |= PKGF_NOBUILD_I; |
549987f1 | 634 | } |
87017ac4 | 635 | |
8e25f19b MD |
636 | /* |
637 | * Handle propagated flags | |
638 | */ | |
639 | if (pkg->flags & PKGF_ERROR) { | |
9c4c701f MD |
640 | /* |
641 | * This can only happen if the ERROR has already been | |
642 | * processed and accounted for. | |
643 | */ | |
1fa9d809 | 644 | ddprintf(depth, "} (ERROR - %s)\n", pkg->portdir); |
8e25f19b MD |
645 | } else if (*app & PKGF_NOTREADY) { |
646 | /* | |
647 | * We don't set PKGF_NOTREADY in the pkg, it is strictly | |
648 | * a transient flag propagated via build_find_leaves(). | |
649 | * | |
650 | * Just don't add the package to the list. | |
9c4c701f MD |
651 | * |
652 | * NOTE: Even if NOBUILD is set (meaning we could list it | |
653 | * and let startbuild() finish it up as a skip, we | |
654 | * don't process it to the list because we want to | |
655 | * process all the dependencies, so someone doing a | |
656 | * manual build can get more complete information and | |
657 | * does not have to iterate each failed dependency one | |
658 | * at a time. | |
8e25f19b MD |
659 | */ |
660 | ; | |
661 | } else if (pkg->flags & PKGF_SUCCESS) { | |
1fa9d809 | 662 | ddprintf(depth, "} (SUCCESS - %s)\n", pkg->portdir); |
8e25f19b | 663 | } else if (pkg->flags & PKGF_DUMMY) { |
7b609b0a MD |
664 | /* |
665 | * Just mark dummy packages as successful when all of their | |
666 | * sub-depends (flavors) complete successfully. Note that | |
667 | * dummy packages are not counted in the total, so do not | |
668 | * decrement BuildTotal. | |
50cde9e9 | 669 | * |
549987f1 MD |
670 | * Do not propagate *app up for the dummy node or add it to |
671 | * the build list. The dummy node itself is not an actual | |
672 | * dependency. Packages which depend on the default flavor | |
673 | * (aka this dummy node) actually depend on the first flavor | |
674 | * under this node. | |
675 | * | |
676 | * So if there is a generic dependency (i.e. no flavor | |
677 | * specified), the upper recursion detects PKGF_DUMMY and | |
678 | * traverses through the dummy node to the default flavor | |
679 | * without checking the error/nobuild flags on this dummy | |
680 | * node. | |
7b609b0a | 681 | */ |
349e3a76 | 682 | if (pkg->flags & PKGF_NOBUILD) { |
549987f1 MD |
683 | ddprintf(depth, "} (DUMMY/META - IGNORED " |
684 | "- MARK SUCCESS ANYWAY)\n"); | |
8e25f19b | 685 | } else { |
349e3a76 | 686 | ddprintf(depth, "} (DUMMY/META - SUCCESS)\n"); |
549987f1 MD |
687 | } |
688 | pkg->flags |= PKGF_SUCCESS; | |
689 | *hasworkp = 1; | |
690 | if (first) { | |
691 | dlog(DLOG_ALL | DLOG_FILTER, | |
692 | "[XXX] %s META-ALREADY-BUILT\n", | |
693 | pkg->portdir); | |
694 | } else { | |
695 | dlog(DLOG_SUCC, "[XXX] %s meta-node complete\n", | |
696 | pkg->portdir); | |
697 | RunStatsUpdateCompletion(NULL, DLOG_SUCC, pkg, "", ""); | |
51705b28 | 698 | ++BuildMetaCount; /* Only for not built meta nodes */ |
8e25f19b MD |
699 | } |
700 | } else if (pkg->flags & PKGF_PACKAGED) { | |
701 | /* | |
702 | * We can just mark the pkg successful. If this is | |
703 | * the first pass, we count this as an initial pruning | |
704 | * pass and reduce BuildTotal. | |
705 | */ | |
1fa9d809 | 706 | ddprintf(depth, "} (PACKAGED - SUCCESS)\n"); |
8e25f19b MD |
707 | pkg->flags |= PKGF_SUCCESS; |
708 | *hasworkp = 1; | |
709 | if (first) { | |
a574cbf0 | 710 | dlog(DLOG_ALL | DLOG_FILTER, |
bd0c9b9e | 711 | "[XXX] %s Already-Built\n", |
8e25f19b MD |
712 | pkg->portdir); |
713 | --BuildTotal; | |
549987f1 MD |
714 | } else { |
715 | dlog(DLOG_ABN | DLOG_FILTER, | |
bd0c9b9e MD |
716 | "[XXX] %s flags=%08x Packaged Unexpectedly\n", |
717 | pkg->portdir, pkg->flags); | |
549987f1 MD |
718 | /* ++BuildSuccessTotal; XXX not sure */ |
719 | goto addlist; | |
8e25f19b MD |
720 | } |
721 | } else { | |
722 | /* | |
723 | * All dependencies are successful, queue new work | |
724 | * and indicate not-ready to the parent (since our | |
725 | * package has to be built). | |
9c4c701f MD |
726 | * |
727 | * NOTE: The NOBUILD case propagates to here as well | |
728 | * and is ultimately handled by startbuild(). | |
8e25f19b | 729 | */ |
549987f1 | 730 | addlist: |
8e25f19b | 731 | *hasworkp = 1; |
87017ac4 | 732 | if (pkg->flags & PKGF_NOBUILD_I) |
1fa9d809 | 733 | ddprintf(depth, "} (ADDLIST(IGNORE/BROKEN) - %s)\n", |
87017ac4 MD |
734 | pkg->portdir); |
735 | else if (pkg->flags & PKGF_NOBUILD) | |
1fa9d809 | 736 | ddprintf(depth, "} (ADDLIST(NOBUILD) - %s)\n", |
9c4c701f MD |
737 | pkg->portdir); |
738 | else | |
1fa9d809 | 739 | ddprintf(depth, "} (ADDLIST - %s)\n", pkg->portdir); |
8e25f19b MD |
740 | pkg->flags |= PKGF_BUILDLIST; |
741 | **build_tailp = pkg; | |
742 | *build_tailp = &pkg->build_next; | |
549987f1 | 743 | pkg->build_next = NULL; |
8e25f19b MD |
744 | *app |= PKGF_NOTREADY; |
745 | } | |
746 | ||
747 | return idep_count; | |
748 | } | |
749 | ||
750 | static | |
751 | void | |
752 | build_clear_trav(pkg_t *pkg) | |
753 | { | |
754 | pkglink_t *link; | |
755 | pkg_t *scan; | |
756 | ||
757 | pkg->flags &= ~PKGF_BUILDTRAV; | |
758 | PKGLIST_FOREACH(link, &pkg->idepon_list) { | |
759 | scan = link->pkg; | |
760 | if (scan && (scan->flags & PKGF_BUILDTRAV)) | |
761 | build_clear_trav(scan); | |
762 | } | |
763 | } | |
764 | ||
88c24d72 MD |
765 | /* |
766 | * Calculate the longest chain of packages that depend on me. The | |
767 | * long the chain, the more important my package is to build earlier | |
768 | * rather than later. | |
769 | */ | |
770 | static int | |
771 | buildCalculateDepiDepth(pkg_t *pkg) | |
772 | { | |
773 | pkglink_t *link; | |
774 | pkg_t *scan; | |
775 | int best_depth = 0; | |
776 | int res; | |
777 | ||
778 | if (pkg->depi_depth) | |
779 | return(pkg->depi_depth + 1); | |
780 | pkg->flags |= PKGF_BUILDLOOP; | |
781 | PKGLIST_FOREACH(link, &pkg->deponi_list) { | |
782 | scan = link->pkg; | |
783 | if (scan && (scan->flags & PKGF_BUILDLOOP) == 0) { | |
784 | res = buildCalculateDepiDepth(scan); | |
785 | if (best_depth < res) | |
786 | best_depth = res; | |
787 | } | |
788 | } | |
789 | pkg->flags &= ~PKGF_BUILDLOOP; | |
790 | pkg->depi_depth = best_depth; | |
791 | ||
792 | return (best_depth + 1); | |
793 | } | |
794 | ||
8e25f19b MD |
795 | /* |
796 | * Take a list of pkg ready to go, sort it, and assign it to worker | |
797 | * slots. This routine blocks in waitbuild() until it can dispose of | |
798 | * the entire list. | |
799 | * | |
800 | * WorkerMutex is held by the caller. | |
801 | */ | |
802 | static | |
803 | void | |
804 | startbuild(pkg_t **build_listp, pkg_t ***build_tailp) | |
805 | { | |
806 | pkg_t *pkg; | |
807 | pkg_t **idep_ary; | |
808 | pkg_t **depi_ary; | |
809 | int count; | |
810 | int idep_index; | |
811 | int depi_index; | |
812 | int i; | |
813 | int n; | |
814 | worker_t *work; | |
815 | static int IterateWorker; | |
816 | ||
817 | /* | |
818 | * Nothing to do | |
819 | */ | |
820 | if (*build_listp == NULL) | |
821 | return; | |
822 | ||
823 | /* | |
824 | * Sort | |
825 | */ | |
826 | count = 0; | |
827 | for (pkg = *build_listp; pkg; pkg = pkg->build_next) | |
828 | ++count; | |
829 | idep_ary = calloc(count, sizeof(pkg_t *)); | |
830 | depi_ary = calloc(count, sizeof(pkg_t *)); | |
831 | ||
832 | count = 0; | |
833 | for (pkg = *build_listp; pkg; pkg = pkg->build_next) { | |
834 | idep_ary[count] = pkg; | |
835 | depi_ary[count] = pkg; | |
836 | ++count; | |
837 | } | |
838 | ||
839 | /* | |
840 | * idep_ary - sorted by #of dependencies this pkg has. | |
841 | * depi_ary - sorted by #of other packages that depend on this pkg. | |
842 | */ | |
843 | qsort(idep_ary, count, sizeof(pkg_t *), qsort_idep); | |
844 | qsort(depi_ary, count, sizeof(pkg_t *), qsort_depi); | |
845 | ||
846 | idep_index = 0; | |
847 | depi_index = 0; | |
848 | ||
849 | /* | |
850 | * Half the workers build based on the highest depi count, | |
851 | * the other half build based on the highest idep count. | |
852 | * | |
853 | * This is an attempt to get projects which many other projects | |
854 | * depend on built first, but to also try to build large projects | |
855 | * (which tend to have a lot of dependencies) earlier rather than | |
856 | * later so the end of the bulk run doesn't inefficiently build | |
857 | * the last few huge projects. | |
858 | * | |
859 | * Loop until we manage to assign slots to everyone. We do not | |
860 | * wait for build completion. | |
861 | * | |
862 | * This is the point where we handle DUMMY packages (these are | |
863 | * dummy unflavored packages which 'cover' all the flavors for | |
864 | * a package). These are not real packages are marked SUCCESS | |
865 | * at this time because their dependencies (the flavors) have all | |
866 | * been built. | |
867 | */ | |
868 | while (idep_index != count || depi_index != count) { | |
869 | pkg_t *pkgi; | |
870 | pkg_t *ipkg; | |
871 | ||
872 | /* | |
873 | * Find candidate to start sorted by depi or idep. | |
874 | */ | |
875 | ipkg = NULL; | |
876 | while (idep_index < count) { | |
877 | ipkg = idep_ary[idep_index]; | |
878 | if ((ipkg->flags & | |
3699ee09 MD |
879 | (PKGF_SUCCESS | PKGF_FAILURE | |
880 | PKGF_ERROR | PKGF_RUNNING)) == 0) { | |
8e25f19b MD |
881 | break; |
882 | } | |
883 | ipkg = NULL; | |
884 | ++idep_index; | |
885 | } | |
886 | ||
887 | pkgi = NULL; | |
888 | while (depi_index < count) { | |
889 | pkgi = depi_ary[depi_index]; | |
890 | if ((pkgi->flags & | |
3699ee09 MD |
891 | (PKGF_SUCCESS | PKGF_FAILURE | |
892 | PKGF_ERROR | PKGF_RUNNING)) == 0) { | |
8e25f19b MD |
893 | break; |
894 | } | |
895 | pkgi = NULL; | |
896 | ++depi_index; | |
897 | } | |
898 | ||
f7f25838 MD |
899 | /* |
900 | * ipkg and pkgi must either both be NULL, or both | |
901 | * be non-NULL. | |
902 | */ | |
8e25f19b MD |
903 | if (ipkg == NULL && pkgi == NULL) |
904 | break; | |
905 | ddassert(ipkg && pkgi); | |
906 | ||
9c4c701f MD |
907 | /* |
908 | * Handle the NOBUILD case right here, there's no point | |
909 | * queueing it anywhere. | |
910 | */ | |
911 | if (ipkg->flags & PKGF_NOBUILD) { | |
912 | char *reason; | |
64b61b2e MD |
913 | char skipbuf[16]; |
914 | int scount; | |
915 | ||
916 | scount = buildskipcount_dueto(ipkg, 1); | |
917 | buildskipcount_dueto(ipkg, 0); | |
918 | if (scount) { | |
919 | snprintf(skipbuf, sizeof(skipbuf), " %d", | |
920 | scount); | |
921 | } else { | |
922 | skipbuf[0] = 0; | |
923 | } | |
9c4c701f MD |
924 | |
925 | ipkg->flags |= PKGF_FAILURE; | |
926 | ipkg->flags &= ~PKGF_BUILDLIST; | |
927 | ||
4ea2ee4d MD |
928 | reason = buildskipreason(NULL, ipkg); |
929 | if (ipkg->flags & PKGF_NOBUILD_I) { | |
87017ac4 | 930 | ++BuildIgnoreCount; |
64b61b2e MD |
931 | dlog(DLOG_IGN, |
932 | "[XXX] %s%s ignored due to %s\n", | |
933 | ipkg->portdir, skipbuf, reason); | |
934 | RunStatsUpdateCompletion(NULL, DLOG_IGN, ipkg, | |
935 | reason, skipbuf); | |
a69ec4d6 | 936 | doHook(ipkg, "hook_pkg_ignored", |
68dc2eea | 937 | HookPkgIgnored, 0); |
4ea2ee4d | 938 | } else { |
87017ac4 | 939 | ++BuildSkipCount; |
64b61b2e MD |
940 | dlog(DLOG_SKIP, |
941 | "[XXX] %s%s skipped due to %s\n", | |
942 | ipkg->portdir, skipbuf, reason); | |
943 | RunStatsUpdateCompletion(NULL, DLOG_SKIP, ipkg, | |
944 | reason, skipbuf); | |
a69ec4d6 | 945 | doHook(ipkg, "hook_pkg_skipped", |
68dc2eea | 946 | HookPkgSkipped, 0); |
4ea2ee4d | 947 | } |
9c4c701f | 948 | free(reason); |
ffc851f6 | 949 | ++BuildCount; |
9c4c701f MD |
950 | continue; |
951 | } | |
952 | if (pkgi->flags & PKGF_NOBUILD) { | |
953 | char *reason; | |
64b61b2e MD |
954 | char skipbuf[16]; |
955 | int scount; | |
956 | ||
957 | scount = buildskipcount_dueto(pkgi, 1); | |
958 | buildskipcount_dueto(pkgi, 0); | |
959 | if (scount) { | |
960 | snprintf(skipbuf, sizeof(skipbuf), " %d", | |
961 | scount); | |
962 | } else { | |
963 | skipbuf[0] = 0; | |
964 | } | |
9c4c701f MD |
965 | |
966 | pkgi->flags |= PKGF_FAILURE; | |
967 | pkgi->flags &= ~PKGF_BUILDLIST; | |
968 | ||
ffc851f6 MD |
969 | reason = buildskipreason(NULL, pkgi); |
970 | if (pkgi->flags & PKGF_NOBUILD_I) { | |
87017ac4 | 971 | ++BuildIgnoreCount; |
64b61b2e MD |
972 | dlog(DLOG_IGN, "[XXX] %s%s ignored due to %s\n", |
973 | pkgi->portdir, skipbuf, reason); | |
974 | RunStatsUpdateCompletion(NULL, DLOG_IGN, pkgi, | |
975 | reason, skipbuf); | |
a69ec4d6 | 976 | doHook(pkgi, "hook_pkg_ignored", |
68dc2eea | 977 | HookPkgIgnored, 0); |
ffc851f6 | 978 | } else { |
87017ac4 | 979 | ++BuildSkipCount; |
64b61b2e MD |
980 | dlog(DLOG_SKIP, |
981 | "[XXX] %s%s skipped due to %s\n", | |
982 | pkgi->portdir, skipbuf, reason); | |
983 | RunStatsUpdateCompletion(NULL, DLOG_SKIP, pkgi, | |
984 | reason, skipbuf); | |
a69ec4d6 | 985 | doHook(pkgi, "hook_pkg_skipped", |
68dc2eea | 986 | HookPkgSkipped, 0); |
ffc851f6 | 987 | } |
9c4c701f | 988 | free(reason); |
ffc851f6 | 989 | ++BuildCount; |
9c4c701f MD |
990 | continue; |
991 | } | |
992 | ||
8e25f19b MD |
993 | /* |
994 | * Block while no slots are available. waitbuild() | |
995 | * will clean out any DONE states. | |
996 | */ | |
8d9409b8 MD |
997 | while (RunningWorkers >= DynamicMaxWorkers || |
998 | RunningWorkers >= MaxWorkers - FailedWorkers) { | |
3699ee09 | 999 | waitbuild(RunningWorkers, 1); |
8e25f19b MD |
1000 | } |
1001 | ||
1002 | /* | |
1003 | * Find an available worker slot, there should be at | |
1004 | * least one. | |
1005 | */ | |
1006 | for (i = 0; i < MaxWorkers; ++i) { | |
1007 | n = IterateWorker % MaxWorkers; | |
1008 | work = &WorkerAry[n]; | |
1009 | ||
1010 | if (work->state == WORKER_DONE || | |
1011 | work->state == WORKER_FAILED) { | |
1012 | workercomplete(work); | |
1013 | } | |
1014 | if (work->state == WORKER_NONE || | |
1015 | work->state == WORKER_IDLE) { | |
1016 | if (n <= MaxWorkers / 2) { | |
1017 | startworker(pkgi, work); | |
1018 | } else { | |
1019 | startworker(ipkg, work); | |
1020 | } | |
ea37671d | 1021 | /*RunStatsUpdate(work);*/ |
8e25f19b MD |
1022 | break; |
1023 | } | |
1024 | ++IterateWorker; | |
1025 | } | |
1026 | ddassert(i != MaxWorkers); | |
1027 | } | |
ea37671d | 1028 | RunStatsSync(); |
8e25f19b MD |
1029 | |
1030 | /* | |
1031 | * We disposed of the whole list | |
1032 | */ | |
1033 | free(idep_ary); | |
1034 | free(depi_ary); | |
1035 | *build_listp = NULL; | |
1036 | *build_tailp = build_listp; | |
1037 | } | |
1038 | ||
1039 | typedef const pkg_t *pkg_tt; | |
1040 | ||
1041 | static int | |
1042 | qsort_idep(const void *pkg1_arg, const void *pkg2_arg) | |
1043 | { | |
1044 | const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg; | |
1045 | const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg; | |
1046 | ||
1047 | return (pkg2->idep_count - pkg1->idep_count); | |
1048 | } | |
1049 | ||
1050 | static int | |
1051 | qsort_depi(const void *pkg1_arg, const void *pkg2_arg) | |
1052 | { | |
1053 | const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg; | |
1054 | const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg; | |
1055 | ||
88c24d72 MD |
1056 | return ((pkg2->depi_count * pkg2->depi_depth) - |
1057 | (pkg1->depi_count * pkg1->depi_depth)); | |
8e25f19b MD |
1058 | } |
1059 | ||
1060 | /* | |
1061 | * Frontend starts a pkg up on a worker | |
1062 | * | |
1063 | * WorkerMutex must be held. | |
1064 | */ | |
1065 | static void | |
1066 | startworker(pkg_t *pkg, worker_t *work) | |
1067 | { | |
1068 | switch(work->state) { | |
1069 | case WORKER_NONE: | |
1070 | pthread_create(&work->td, NULL, childBuilderThread, work); | |
1071 | work->state = WORKER_IDLE; | |
1072 | /* fall through */ | |
1073 | case WORKER_IDLE: | |
fef2fc63 | 1074 | work->pkg_dep_size = |
1fa9d809 MD |
1075 | childInstallPkgDeps_recurse(NULL, &pkg->idepon_list, 0, 1, 0); |
1076 | childInstallPkgDeps_recurse(NULL, &pkg->idepon_list, 1, 1, 0); | |
fef2fc63 MD |
1077 | RunningPkgDepSize += work->pkg_dep_size; |
1078 | ||
a574cbf0 MD |
1079 | dlog(DLOG_ALL, "[%03d] START %s " |
1080 | "##idep=%02d depi=%02d/%02d dep=%-4.2fG\n", | |
88c24d72 MD |
1081 | work->index, pkg->portdir, |
1082 | pkg->idep_count, pkg->depi_count, pkg->depi_depth, | |
a574cbf0 | 1083 | (double)work->pkg_dep_size / (double)ONEGB); |
fef2fc63 | 1084 | |
8e25f19b MD |
1085 | cleanworker(work); |
1086 | pkg->flags |= PKGF_RUNNING; | |
1087 | work->pkg = pkg; | |
1088 | pthread_cond_signal(&work->cond); | |
1089 | ++RunningWorkers; | |
ea37671d | 1090 | /*RunStatsUpdate(work);*/ |
8e25f19b MD |
1091 | break; |
1092 | case WORKER_PENDING: | |
1093 | case WORKER_RUNNING: | |
1094 | case WORKER_DONE: | |
1095 | case WORKER_FAILED: | |
8ec23ca1 | 1096 | case WORKER_FROZEN: |
8e25f19b MD |
1097 | case WORKER_EXITING: |
1098 | default: | |
6fd67931 MD |
1099 | dfatal("startworker: [%03d] Unexpected state %d for worker %d", |
1100 | work->index, work->state, work->index); | |
8e25f19b MD |
1101 | break; |
1102 | } | |
1103 | } | |
1104 | ||
1105 | static void | |
1106 | cleanworker(worker_t *work) | |
1107 | { | |
1108 | work->state = WORKER_PENDING; | |
1109 | work->flags = 0; | |
1110 | work->accum_error = 0; | |
1111 | work->start_time = time(NULL); | |
1112 | } | |
1113 | ||
1114 | /* | |
1115 | * Frontend finishes up a completed pkg on a worker. | |
1116 | * | |
1117 | * If the worker is in a FAILED state we clean the pkg out but (for now) | |
1118 | * leave it in its failed state so we can debug. At this point | |
1119 | * workercomplete() will be called every once in a while on the state | |
1120 | * and we have to deal with the NULL pkg. | |
1121 | * | |
1122 | * WorkerMutex must be held. | |
1123 | */ | |
1124 | static void | |
1125 | workercomplete(worker_t *work) | |
1126 | { | |
1127 | pkg_t *pkg; | |
3232b774 MD |
1128 | time_t t; |
1129 | int h; | |
1130 | int m; | |
1131 | int s; | |
8e25f19b MD |
1132 | |
1133 | /* | |
1134 | * Steady state FAILED case. | |
1135 | */ | |
1136 | if (work->state == WORKER_FAILED) { | |
1137 | if (work->pkg == NULL) | |
1138 | return; | |
1139 | } | |
1140 | ||
3232b774 | 1141 | t = time(NULL) - work->start_time; |
3232b774 | 1142 | h = t / 3600; |
a574cbf0 MD |
1143 | m = t / 60 % 60; |
1144 | s = t % 60; | |
3232b774 | 1145 | |
fef2fc63 MD |
1146 | /* |
1147 | * Reduce total dep size | |
1148 | */ | |
1149 | RunningPkgDepSize -= work->pkg_dep_size; | |
516819d9 MD |
1150 | RunningPkgDepSize -= work->memuse; |
1151 | work->pkg_dep_size = 0; | |
1152 | work->memuse = 0; | |
fef2fc63 | 1153 | |
8e25f19b MD |
1154 | /* |
1155 | * Process pkg out of the worker | |
1156 | */ | |
1157 | pkg = work->pkg; | |
1158 | if (pkg->flags & (PKGF_ERROR|PKGF_NOBUILD)) { | |
64b61b2e MD |
1159 | char skipbuf[16]; |
1160 | int scount; | |
1161 | ||
d1e52415 MD |
1162 | /* |
1163 | * Normally mark the package as failed, but if we are doing | |
1164 | * a fetch-only, mark it as successful so dependant ports | |
1165 | * still get fetched. | |
1166 | */ | |
1167 | if (FetchOnlyOpt) | |
1168 | pkg->flags |= PKGF_SUCCESS; | |
1169 | else | |
1170 | pkg->flags |= PKGF_FAILURE; | |
9c4c701f | 1171 | |
64b61b2e MD |
1172 | scount = buildskipcount_dueto(pkg, 1); |
1173 | buildskipcount_dueto(pkg, 0); | |
1174 | if (scount) { | |
1175 | snprintf(skipbuf, sizeof(skipbuf), " %d", | |
1176 | scount); | |
1177 | } else { | |
1178 | skipbuf[0] = 0; | |
1179 | } | |
1180 | ||
9c4c701f MD |
1181 | /* |
1182 | * This NOBUILD condition XXX can occur if the package is | |
1183 | * not allowed to be built. | |
1184 | */ | |
8e25f19b MD |
1185 | if (pkg->flags & PKGF_NOBUILD) { |
1186 | char *reason; | |
1187 | ||
3232b774 MD |
1188 | reason = buildskipreason(NULL, pkg); |
1189 | if (pkg->flags & PKGF_NOBUILD_I) { | |
87017ac4 | 1190 | ++BuildIgnoreCount; |
64b61b2e MD |
1191 | dlog(DLOG_SKIP, "[%03d] IGNORD %s%s - %s\n", |
1192 | work->index, pkg->portdir, | |
1193 | skipbuf, reason); | |
1194 | RunStatsUpdateCompletion(work, DLOG_SKIP, pkg, | |
1195 | reason, skipbuf); | |
68dc2eea MD |
1196 | doHook(pkg, "hook_pkg_ignored", |
1197 | HookPkgIgnored, 0); | |
3232b774 | 1198 | } else { |
87017ac4 | 1199 | ++BuildSkipCount; |
64b61b2e MD |
1200 | dlog(DLOG_SKIP, "[%03d] SKIPPD %s%s - %s\n", |
1201 | work->index, pkg->portdir, | |
1202 | skipbuf, reason); | |
1203 | RunStatsUpdateCompletion(work, DLOG_SKIP, pkg, | |
1204 | reason, skipbuf); | |
68dc2eea MD |
1205 | doHook(pkg, "hook_pkg_skipped", |
1206 | HookPkgSkipped, 0); | |
3232b774 | 1207 | } |
8e25f19b MD |
1208 | free(reason); |
1209 | } else { | |
1210 | ++BuildFailCount; | |
a574cbf0 | 1211 | dlog(DLOG_FAIL | DLOG_RED, |
3dd48cfa MD |
1212 | "[%03d] FAILURE %s%s ##%16.16s %02d:%02d:%02d\n", |
1213 | work->index, pkg->portdir, skipbuf, | |
3cebe4a8 MD |
1214 | getphasestr(work->phase), |
1215 | h, m, s); | |
64b61b2e MD |
1216 | RunStatsUpdateCompletion(work, DLOG_FAIL, pkg, |
1217 | skipbuf, ""); | |
68dc2eea | 1218 | doHook(pkg, "hook_pkg_failure", HookPkgFailure, 0); |
8e25f19b MD |
1219 | } |
1220 | } else { | |
b6bd007b MD |
1221 | if (CheckDBM) { |
1222 | datum key; | |
1223 | datum data; | |
1224 | ||
1225 | key.dptr = pkg->portdir; | |
1226 | key.dsize = strlen(pkg->portdir); | |
1227 | data.dptr = &pkg->crc32; | |
1228 | data.dsize = sizeof(pkg->crc32); | |
7b5ce327 MD |
1229 | #ifndef __DB_IS_THREADSAFE |
1230 | pthread_mutex_lock(&DbmMutex); | |
1231 | #endif | |
b6bd007b | 1232 | dbm_store(CheckDBM, key, data, DBM_REPLACE); |
7b5ce327 MD |
1233 | #ifndef __DB_IS_THREADSAFE |
1234 | pthread_mutex_unlock(&DbmMutex); | |
1235 | #endif | |
b6bd007b | 1236 | } |
68dc2eea MD |
1237 | pkg->flags |= PKGF_SUCCESS; |
1238 | ++BuildSuccessCount; | |
a574cbf0 MD |
1239 | dlog(DLOG_SUCC | DLOG_GRN, |
1240 | "[%03d] SUCCESS %s ##%02d:%02d:%02d\n", | |
8e25f19b | 1241 | work->index, pkg->portdir, h, m, s); |
64b61b2e | 1242 | RunStatsUpdateCompletion(work, DLOG_SUCC, pkg, "", ""); |
68dc2eea | 1243 | doHook(pkg, "hook_pkg_success", HookPkgSuccess, 0); |
8e25f19b MD |
1244 | } |
1245 | ++BuildCount; | |
1246 | pkg->flags &= ~PKGF_BUILDLIST; | |
8e25f19b MD |
1247 | pkg->flags &= ~PKGF_RUNNING; |
1248 | work->pkg = NULL; | |
1249 | --RunningWorkers; | |
1250 | ||
1251 | if (work->state == WORKER_FAILED) { | |
1252 | dlog(DLOG_ALL, "[%03d] XXX/XXX WORKER IS IN A FAILED STATE\n", | |
1253 | work->index); | |
8d9409b8 | 1254 | ++FailedWorkers; |
8ec23ca1 | 1255 | } else if (work->flags & WORKERF_FREEZE) { |
a574cbf0 MD |
1256 | dlog(DLOG_ALL, "[%03d] FROZEN(DEBUG) %s\n", |
1257 | work->index, pkg->portdir); | |
8ec23ca1 | 1258 | work->state = WORKER_FROZEN; |
8e25f19b MD |
1259 | } else { |
1260 | work->state = WORKER_IDLE; | |
1261 | } | |
1262 | } | |
1263 | ||
1264 | /* | |
1265 | * Wait for one or more workers to complete. | |
1266 | * | |
1267 | * WorkerMutex must be held. | |
1268 | */ | |
1269 | static void | |
3699ee09 | 1270 | waitbuild(int whilematch, int dynamicmax) |
8e25f19b | 1271 | { |
1645cafe | 1272 | static time_t wblast_time; |
7f0eca56 | 1273 | static time_t dmlast_time; |
8e25f19b MD |
1274 | struct timespec ts; |
1275 | worker_t *work; | |
1645cafe | 1276 | time_t t; |
8e25f19b MD |
1277 | int i; |
1278 | ||
1279 | if (whilematch == 0) | |
1280 | whilematch = 1; | |
1281 | ||
1282 | while (RunningWorkers == whilematch) { | |
1283 | for (i = 0; i < MaxWorkers; ++i) { | |
1284 | work = &WorkerAry[i]; | |
1285 | if (work->state == WORKER_DONE || | |
1286 | work->state == WORKER_FAILED) { | |
1287 | workercomplete(work); | |
1288 | } else { | |
1289 | pthread_cond_signal(&work->cond); | |
1290 | } | |
aac7a6d9 | 1291 | RunStatsUpdate(work, NULL); |
8e25f19b | 1292 | } |
8b485838 | 1293 | RunStatsUpdateTop(1); |
ea37671d MD |
1294 | RunStatsUpdateLogs(); |
1295 | RunStatsSync(); | |
8e25f19b MD |
1296 | if (RunningWorkers == whilematch) { |
1297 | clock_gettime(CLOCK_REALTIME, &ts); | |
1298 | ts.tv_sec += 1; | |
1299 | ts.tv_nsec = 0; | |
1300 | pthread_cond_timedwait(&WorkerCond, &WorkerMutex, &ts); | |
1301 | } | |
1645cafe MD |
1302 | |
1303 | /* | |
1304 | * Dynamically reduce MaxWorkers based on the load. When | |
1305 | * the load exceeds 2 x ncpus we reduce the number of workers | |
9c4c701f | 1306 | * up to 75% of MaxWorkers @ (5 x ncpus) load. |
f7f25838 MD |
1307 | * |
1308 | * Dynamically reduce MaxWorkers based on swap use, starting | |
1309 | * at 10% swap and up to 75% of MaxWorkers at 40% swap. | |
1310 | * | |
1311 | * NOTE! Generally speaking this allows more workers to be | |
1312 | * configured which helps in two ways. First it allows | |
1313 | * a higher build rate for smaller packages. Second | |
1314 | * it allows dsynth to ratchet-down the number of slots | |
1315 | * when large packages are forcing the load up. | |
1316 | * | |
1317 | * A high load doesn't hurt efficiency, but swap usage | |
1318 | * due to loading the tmpfs in lots of worker slots up | |
1319 | * with tons of pkg install's (pre-reqs for a build) | |
1320 | * does. Reducing the number of worker slots has a | |
1321 | * huge beneficial effect on reducing swap use / paging. | |
1645cafe MD |
1322 | */ |
1323 | t = time(NULL); | |
3699ee09 MD |
1324 | if (dynamicmax && (wblast_time == 0 || |
1325 | (unsigned)(t - wblast_time) >= 5)) { | |
1326 | double min_load = 1.5 * NumCores; | |
f5fc9cfa | 1327 | double max_load = 5.0 * NumCores; |
3699ee09 MD |
1328 | double min_swap = 0.10; |
1329 | double max_swap = 0.40; | |
1645cafe | 1330 | double dload[3]; |
3699ee09 | 1331 | double dswap; |
7f0eca56 MD |
1332 | int max1; |
1333 | int max2; | |
1334 | int max3; | |
1335 | int max_sel; | |
3699ee09 | 1336 | int noswap; |
1645cafe MD |
1337 | |
1338 | wblast_time = t; | |
1645cafe | 1339 | |
fef2fc63 | 1340 | /* |
7f0eca56 | 1341 | * Cap based on load. This is back-loaded. |
fef2fc63 MD |
1342 | */ |
1343 | getloadavg(dload, 3); | |
32fc5bbf | 1344 | adjloadavg(dload); |
3699ee09 | 1345 | if (dload[0] < min_load) { |
7f0eca56 | 1346 | max1 = MaxWorkers; |
3699ee09 | 1347 | } else if (dload[0] <= max_load) { |
7f0eca56 MD |
1348 | max1 = MaxWorkers - |
1349 | MaxWorkers * 0.75 * | |
1350 | (dload[0] - min_load) / | |
1351 | (max_load - min_load); | |
3699ee09 | 1352 | } else { |
7f0eca56 | 1353 | max1 = MaxWorkers * 25 / 100; |
3699ee09 MD |
1354 | } |
1355 | ||
fef2fc63 | 1356 | /* |
7f0eca56 | 1357 | * Cap based on swap use. This is back-loaded. |
fef2fc63 MD |
1358 | */ |
1359 | dswap = getswappct(&noswap); | |
3699ee09 | 1360 | if (dswap < min_swap) { |
7f0eca56 | 1361 | max2 = MaxWorkers; |
3699ee09 | 1362 | } else if (dswap <= max_swap) { |
7f0eca56 MD |
1363 | max2 = MaxWorkers - |
1364 | MaxWorkers * 0.75 * | |
1365 | (dswap - min_swap) / | |
1366 | (max_swap - min_swap); | |
3699ee09 | 1367 | } else { |
7f0eca56 | 1368 | max2 = MaxWorkers * 25 / 100; |
3699ee09 MD |
1369 | } |
1370 | ||
1371 | /* | |
7f0eca56 MD |
1372 | * Cap based on aggregate pkg-dependency memory |
1373 | * use installed in worker slots. This is | |
1374 | * front-loaded. | |
1375 | * | |
1376 | * Since it can take a while for workers to retire | |
1377 | * (to reduce RunningPkgDepSize), just set our | |
1378 | * target 1 below the current run count to allow | |
1379 | * jobs to retire without being replaced with new | |
1380 | * jobs. | |
1381 | * | |
1382 | * In addition, in order to avoid a paging 'shock', | |
1383 | * We enforce a 30 second-per-increment slow-start | |
1384 | * once RunningPkgDepSize exceeds 1/2 the target. | |
3699ee09 | 1385 | */ |
7f0eca56 MD |
1386 | if (RunningPkgDepSize > PkgDepMemoryTarget) { |
1387 | max3 = RunningWorkers - 1; | |
1388 | } else if (RunningPkgDepSize > PkgDepMemoryTarget / 2) { | |
1389 | if (dmlast_time == 0 || | |
1390 | (unsigned)(t - dmlast_time) >= 30) { | |
1391 | dmlast_time = t; | |
1392 | max3 = RunningWorkers + 1; | |
1393 | } else { | |
1394 | max3 = RunningWorkers; | |
1395 | } | |
1396 | } else { | |
1397 | max3 = MaxWorkers; | |
1398 | } | |
fef2fc63 | 1399 | |
4ad2f7b8 MD |
1400 | /* |
1401 | * Dynamic scale adjustment | |
1402 | */ | |
1403 | ||
1404 | if (PkgDepScaleTarget != 100) { | |
1405 | max1 = max1 * (PkgDepScaleTarget + 50) / 100; | |
1406 | max2 = max2 * (PkgDepScaleTarget + 50) / 100; | |
1407 | max3 = max3 * (PkgDepScaleTarget + 50) / 100; | |
1408 | } | |
1409 | ||
fef2fc63 MD |
1410 | /* |
1411 | * Priority reduction, convert to DynamicMaxWorkers | |
1412 | */ | |
7f0eca56 MD |
1413 | max_sel = max1; |
1414 | if (max_sel > max2) | |
1415 | max_sel = max2; | |
1416 | if (max_sel > max3) | |
1417 | max_sel = max3; | |
3699ee09 MD |
1418 | |
1419 | /* | |
7f0eca56 MD |
1420 | * Restrict to allowed range, and also handle |
1421 | * slow-start. | |
3699ee09 | 1422 | */ |
7f0eca56 MD |
1423 | if (max_sel < 1) |
1424 | max_sel = 1; | |
1425 | if (max_sel > DynamicMaxWorkers + 1) | |
1426 | max_sel = DynamicMaxWorkers + 1; | |
1427 | if (max_sel > MaxWorkers) | |
1428 | max_sel = MaxWorkers; | |
3699ee09 MD |
1429 | |
1430 | /* | |
7f0eca56 MD |
1431 | * Stop waiting if DynamicMaxWorkers is going to |
1432 | * increase. | |
3699ee09 | 1433 | */ |
7f0eca56 | 1434 | if (DynamicMaxWorkers < max1) |
3699ee09 MD |
1435 | whilematch = -1; |
1436 | ||
1437 | /* | |
1438 | * And adjust | |
1439 | */ | |
7f0eca56 | 1440 | if (DynamicMaxWorkers != max1) { |
a574cbf0 | 1441 | dlog(DLOG_ALL | DLOG_FILTER, |
7f0eca56 | 1442 | "[XXX] Load=%-6.2f(%2d) " |
6fd67931 | 1443 | "Swap=%-3.2f%%(%2d) " |
7f0eca56 | 1444 | "Mem=%3.2fG(%2d) " |
fef2fc63 | 1445 | "Adjust Workers %d->%d\n", |
7f0eca56 MD |
1446 | dload[0], max1, |
1447 | dswap * 100.0, max2, | |
1448 | RunningPkgDepSize / (double)ONEGB, max3, | |
1449 | DynamicMaxWorkers, max_sel); | |
1450 | DynamicMaxWorkers = max_sel; | |
1645cafe MD |
1451 | } |
1452 | } | |
8e25f19b MD |
1453 | } |
1454 | } | |
1455 | ||
1456 | ||
1457 | /* | |
1458 | * Worker pthread (WorkerAry) | |
1459 | * | |
3bd7e0a7 | 1460 | * This thread belongs to the dsynth master process and handles a worker slot. |
8e25f19b MD |
1461 | * (this_thread) -> WORKER fork/exec (WorkerPocess) -> (pty) -> sub-processes. |
1462 | */ | |
1463 | static void * | |
1464 | childBuilderThread(void *arg) | |
1465 | { | |
1466 | char *envary[1] = { NULL }; | |
1467 | worker_t *work = arg; | |
1468 | wmsg_t wmsg; | |
1469 | pkg_t *pkg; | |
1470 | pid_t pid; | |
1471 | int status; | |
710838f7 | 1472 | int flags; |
8e25f19b MD |
1473 | volatile int dowait; |
1474 | char slotbuf[8]; | |
1475 | char fdbuf[8]; | |
8ec23ca1 | 1476 | char flagsbuf[16]; |
8e25f19b | 1477 | |
3bd7e0a7 | 1478 | setNumaDomain(work->index); |
8e25f19b | 1479 | pthread_mutex_lock(&WorkerMutex); |
3bd7e0a7 | 1480 | |
8e25f19b MD |
1481 | while (work->terminate == 0) { |
1482 | dowait = 1; | |
1483 | ||
1484 | switch(work->state) { | |
1485 | case WORKER_IDLE: | |
1486 | break; | |
1487 | case WORKER_PENDING: | |
1488 | /* | |
1489 | * Fork the management process for the pkg operation | |
1490 | * on this worker slot. | |
1491 | * | |
1492 | * This process will set up the environment, do the | |
1493 | * mounts, will become the reaper, and will process | |
1494 | * pipe commands and handle chroot operations. | |
1495 | * | |
1496 | * NOTE: If SOCK_CLOEXEC is not supported WorkerMutex | |
1497 | * is sufficient to interlock F_SETFD FD_CLOEXEC | |
1498 | * operations. | |
1499 | */ | |
1500 | ddassert(work->pkg); | |
1501 | if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, | |
1502 | PF_UNSPEC, work->fds)) { | |
1503 | dfatal_errno("socketpair() during worker fork"); | |
1504 | } | |
710838f7 MD |
1505 | snprintf(slotbuf, sizeof(slotbuf), "%d", work->index); |
1506 | snprintf(fdbuf, sizeof(fdbuf), "3"); | |
1507 | ||
1508 | /* | |
1509 | * Pass global flags and add-in the DEBUGSTOP if | |
1510 | * the package is flagged for debugging. | |
1511 | */ | |
1512 | flags = WorkerProcFlags; | |
dca89e44 MD |
1513 | |
1514 | if (work->pkg->flags & PKGF_DEBUGSTOP) | |
710838f7 | 1515 | flags |= WORKER_PROC_DEBUGSTOP; |
dca89e44 | 1516 | else |
710838f7 | 1517 | flags &= ~WORKER_PROC_DEBUGSTOP; |
dca89e44 MD |
1518 | |
1519 | if (PkgVersionPkgSuffix) | |
1520 | flags |= WORKER_PROC_PKGV17; | |
1521 | else | |
1522 | flags &= ~WORKER_PROC_PKGV17; | |
1523 | ||
710838f7 | 1524 | snprintf(flagsbuf, sizeof(flagsbuf), "%d", flags); |
8e25f19b MD |
1525 | |
1526 | /* | |
1527 | * fds[0] - master | |
1528 | * fds[1] - slave | |
1529 | * | |
1530 | * We pass the salve descriptor in fd 3 and close all | |
1531 | * other descriptors for security. | |
1532 | */ | |
1533 | pthread_mutex_unlock(&WorkerMutex); | |
1534 | pid = vfork(); | |
1535 | if (pid == 0) { | |
1536 | close(work->fds[0]); | |
1537 | dup2(work->fds[1], 3); | |
1538 | closefrom(4); | |
1539 | fcntl(3, F_SETFD, 0); | |
1540 | execle(DSynthExecPath, DSynthExecPath, | |
a361ab31 | 1541 | "-C", ConfigBase, |
9e1d0b12 | 1542 | "-p", Profile, |
8e25f19b MD |
1543 | "WORKER", slotbuf, fdbuf, |
1544 | work->pkg->portdir, work->pkg->pkgfile, | |
8ec23ca1 | 1545 | flagsbuf, |
8e25f19b MD |
1546 | NULL, envary); |
1547 | write(2, "EXECLE FAILURE\n", 15); | |
1548 | _exit(1); | |
1549 | } | |
1550 | pthread_mutex_lock(&WorkerMutex); | |
1551 | close(work->fds[1]); | |
1552 | work->phase = PHASE_PENDING; | |
1553 | work->lines = 0; | |
516819d9 | 1554 | work->memuse = 0; |
8e25f19b MD |
1555 | work->pid = pid; |
1556 | work->state = WORKER_RUNNING; | |
1557 | /* fall through */ | |
1558 | case WORKER_RUNNING: | |
1559 | /* | |
1560 | * Poll for status updates, if NULL is returned | |
1561 | * and status is non-zero, the communications link | |
1562 | * failed unexpectedly. | |
1563 | */ | |
1564 | pkg = work->pkg; | |
1565 | pthread_mutex_unlock(&WorkerMutex); | |
1566 | status = ipcreadmsg(work->fds[0], &wmsg); | |
1567 | pthread_mutex_lock(&WorkerMutex); | |
1568 | ||
1569 | if (status == 0) { | |
1570 | /* | |
1571 | * Normal message, can include normal | |
1572 | * termination which changes us over | |
1573 | * to another state. | |
1574 | */ | |
1575 | dowait = 0; | |
1576 | switch(wmsg.cmd) { | |
1577 | case WMSG_CMD_INSTALL_PKGS: | |
1578 | wmsg.cmd = WMSG_RES_INSTALL_PKGS; | |
1579 | wmsg.status = childInstallPkgDeps(work); | |
1580 | pthread_mutex_unlock(&WorkerMutex); | |
1581 | ipcwritemsg(work->fds[0], &wmsg); | |
1582 | pthread_mutex_lock(&WorkerMutex); | |
1583 | break; | |
1584 | case WMSG_CMD_STATUS_UPDATE: | |
1585 | work->phase = wmsg.phase; | |
1586 | work->lines = wmsg.lines; | |
516819d9 MD |
1587 | if (work->memuse != wmsg.memuse) { |
1588 | RunningPkgDepSize += | |
1589 | wmsg.memuse - work->memuse; | |
1590 | work->memuse = wmsg.memuse; | |
1591 | } | |
8e25f19b MD |
1592 | break; |
1593 | case WMSG_CMD_SUCCESS: | |
1594 | work->flags |= WORKERF_SUCCESS; | |
1595 | break; | |
1596 | case WMSG_CMD_FAILURE: | |
1597 | work->flags |= WORKERF_FAILURE; | |
1598 | break; | |
8ec23ca1 MD |
1599 | case WMSG_CMD_FREEZEWORKER: |
1600 | work->flags |= WORKERF_FREEZE; | |
1601 | break; | |
8e25f19b MD |
1602 | default: |
1603 | break; | |
1604 | } | |
aac7a6d9 | 1605 | RunStatsUpdate(work, NULL); |
ea37671d | 1606 | RunStatsSync(); |
8e25f19b MD |
1607 | } else { |
1608 | close(work->fds[0]); | |
1609 | pthread_mutex_unlock(&WorkerMutex); | |
1610 | while (waitpid(work->pid, &status, 0) < 0 && | |
1611 | errno == EINTR) { | |
1612 | ; | |
1613 | } | |
1614 | pthread_mutex_lock(&WorkerMutex); | |
1615 | ||
1616 | if (work->flags & WORKERF_SUCCESS) { | |
1617 | pkg->flags |= PKGF_SUCCESS; | |
1618 | work->state = WORKER_DONE; | |
1619 | } else if (work->flags & WORKERF_FAILURE) { | |
1620 | pkg->flags |= PKGF_FAILURE; | |
1621 | work->state = WORKER_DONE; | |
1622 | } else { | |
1623 | pkg->flags |= PKGF_FAILURE; | |
1624 | work->state = WORKER_FAILED; | |
1625 | } | |
1626 | work->flags |= WORKERF_STATUS_UPDATE; | |
1627 | pthread_cond_signal(&WorkerCond); | |
1628 | } | |
1629 | break; | |
1630 | case WORKER_DONE: | |
1631 | /* | |
1632 | * pkg remains attached until frontend processes the | |
1633 | * completion. The frontend will then set the state | |
1634 | * back to idle. | |
1635 | */ | |
1636 | break; | |
1637 | case WORKER_FAILED: | |
1638 | /* | |
1639 | * A worker failure means that the worker did not | |
1640 | * send us a WMSG_CMD_SUCCESS or WMSG_CMD_FAILURE | |
1641 | * ipc before terminating. | |
1642 | * | |
1643 | * We just sit in this state until the front-end | |
1644 | * does something about it. | |
1645 | */ | |
1646 | break; | |
710838f7 MD |
1647 | case WORKER_FROZEN: |
1648 | /* | |
1649 | * A worker getting frozen is debug-related. We | |
1650 | * just sit in this state (likely forever). | |
1651 | */ | |
1652 | break; | |
8e25f19b | 1653 | default: |
710838f7 MD |
1654 | dfatal("worker: [%03d] Unexpected state %d " |
1655 | "for worker %d", | |
6fd67931 | 1656 | work->index, work->state, work->index); |
8e25f19b MD |
1657 | /* NOT REACHED */ |
1658 | break; | |
1659 | } | |
1660 | ||
1661 | /* | |
1662 | * The dsynth frontend will poll us approximately once | |
1663 | * a second (its variable). | |
1664 | */ | |
1665 | if (dowait) | |
1666 | pthread_cond_wait(&work->cond, &WorkerMutex); | |
1667 | } | |
1668 | ||
1669 | /* | |
1670 | * Scrap the comm socket if running, this should cause the worker | |
1671 | * process to kill its sub-programs and cleanup. | |
1672 | */ | |
1673 | if (work->state == WORKER_RUNNING) { | |
1674 | pthread_mutex_unlock(&WorkerMutex); | |
1675 | close(work->fds[0]); | |
1676 | while (waitpid(work->pid, &status, 0) < 0 && | |
1677 | errno == EINTR); | |
1678 | pthread_mutex_lock(&WorkerMutex); | |
1679 | } | |
1680 | ||
1681 | /* | |
1682 | * Final handshake | |
1683 | */ | |
1684 | work->state = WORKER_EXITING; | |
1685 | pthread_cond_signal(&WorkerCond); | |
1686 | pthread_mutex_unlock(&WorkerMutex); | |
1687 | ||
1688 | return NULL; | |
1689 | } | |
1690 | ||
1691 | /* | |
1692 | * Install all the binary packages (we have already built them) that | |
1693 | * the current work package depends on, without duplicates, in a script | |
1694 | * which will be run from within the specified work jail. | |
1695 | * | |
1696 | * Locked by WorkerMutex (global) | |
1697 | */ | |
1698 | static int | |
1699 | childInstallPkgDeps(worker_t *work) | |
1700 | { | |
1701 | char *buf; | |
1702 | FILE *fp; | |
1703 | ||
1704 | if (PKGLIST_EMPTY(&work->pkg->idepon_list)) | |
1705 | return 0; | |
1706 | ||
1707 | asprintf(&buf, "%s/tmp/dsynth_install_pkgs", work->basedir); | |
1708 | fp = fopen(buf, "w"); | |
1709 | ddassert(fp != NULL); | |
1710 | fprintf(fp, "#!/bin/sh\n"); | |
1711 | fprintf(fp, "#\n"); | |
1712 | fchmod(fileno(fp), 0755); | |
1713 | ||
1fa9d809 MD |
1714 | childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 0, 1, 0); |
1715 | childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 1, 1, 0); | |
8e25f19b MD |
1716 | fprintf(fp, "\nexit 0\n"); |
1717 | fclose(fp); | |
ffc851f6 | 1718 | freestrp(&buf); |
8e25f19b MD |
1719 | |
1720 | return 1; | |
1721 | } | |
1722 | ||
9e1d0b12 MD |
1723 | /* |
1724 | * Recursive child install dependencies. | |
1725 | * | |
1726 | * first_one_only is only specified if the pkg the list comes from | |
1727 | * is a generic unflavored package that has flavors, telling us to | |
1728 | * dive the first flavor only. | |
1729 | * | |
1730 | * However, in nearly all cases this flag will now be zero because | |
1731 | * this code now dives the first flavor when encountering a dummy node | |
1732 | * and clears nfirst on success. Hence if you are asking why 'nfirst' | |
1733 | * is set to 1, and then zero, instead of just being removed entirely, | |
1734 | * it is because there might still be an edge case here. | |
1735 | */ | |
fef2fc63 | 1736 | static size_t |
1fa9d809 MD |
1737 | childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list, int undoit, |
1738 | int depth, int first_one_only) | |
8e25f19b MD |
1739 | { |
1740 | pkglink_t *link; | |
1741 | pkg_t *pkg; | |
fef2fc63 | 1742 | size_t tot = 0; |
1fa9d809 MD |
1743 | int ndepth; |
1744 | int nfirst; | |
8e25f19b MD |
1745 | |
1746 | PKGLIST_FOREACH(link, list) { | |
1747 | pkg = link->pkg; | |
1748 | ||
1fa9d809 MD |
1749 | /* |
1750 | * We don't want to mess up our depth test just below if | |
1751 | * a DUMMY node had to be inserted. The nodes under the | |
1752 | * dummy node. | |
1753 | * | |
1754 | * The elements under a dummy node represent all the flabor, | |
1755 | * a dependency that directly references a dummy node only | |
1756 | * uses the first flavor (first_one_only / nfirst). | |
1757 | */ | |
1758 | ndepth = (pkg->flags & PKGF_DUMMY) ? depth : depth + 1; | |
1759 | nfirst = (pkg->flags & PKGF_DUMMY) ? 1 : 0; | |
1760 | ||
63fcce5b MD |
1761 | /* |
1762 | * We only need all packages for the top-level dependencies. | |
1763 | * The deeper ones only need DEP_TYPE_LIB and DEP_TYPE_RUN | |
1764 | * (types greater than DEP_TYPE_BUILD) since they are already | |
1765 | * built. | |
1766 | */ | |
1fa9d809 MD |
1767 | if (depth > 1 && link->dep_type <= DEP_TYPE_BUILD) { |
1768 | if (first_one_only) | |
1769 | break; | |
63fcce5b | 1770 | continue; |
1fa9d809 | 1771 | } |
63fcce5b | 1772 | |
9e1d0b12 MD |
1773 | /* |
1774 | * If this is a dummy node with no package, the originator | |
1775 | * is requesting a flavored package. We select the default | |
1776 | * flavor which we presume is the first one. | |
1777 | */ | |
1778 | if (pkg->pkgfile == NULL && (pkg->flags & PKGF_DUMMY)) { | |
1779 | pkg_t *spkg = pkg->idepon_list.next->pkg; | |
1780 | ||
1781 | if (spkg) { | |
1782 | if (fp) { | |
1783 | fprintf(fp, | |
1784 | "echo 'UNFLAVORED %s -> use " | |
1785 | "%s'\n", | |
1786 | pkg->portdir, | |
1787 | spkg->portdir); | |
1788 | } | |
1789 | pkg = spkg; | |
1790 | nfirst = 0; | |
1791 | } else { | |
1792 | if (fp) { | |
1793 | fprintf(fp, | |
1794 | "echo 'CANNOT FIND DEFAULT " | |
1795 | "FLAVOR FOR %s'\n", | |
1796 | pkg->portdir); | |
1797 | } | |
1798 | } | |
1799 | } | |
1800 | ||
8e25f19b MD |
1801 | if (undoit) { |
1802 | if (pkg->dsynth_install_flg == 1) { | |
1803 | pkg->dsynth_install_flg = 0; | |
fef2fc63 | 1804 | tot += childInstallPkgDeps_recurse(fp, |
8e25f19b | 1805 | &pkg->idepon_list, |
1fa9d809 MD |
1806 | undoit, |
1807 | ndepth, nfirst); | |
8e25f19b | 1808 | } |
1fa9d809 MD |
1809 | if (first_one_only) |
1810 | break; | |
8e25f19b MD |
1811 | continue; |
1812 | } | |
9e1d0b12 | 1813 | |
8e25f19b | 1814 | if (pkg->dsynth_install_flg) { |
fef2fc63 | 1815 | if (DebugOpt >= 2 && pkg->pkgfile && fp) { |
8e25f19b MD |
1816 | fprintf(fp, "echo 'AlreadyHave %s'\n", |
1817 | pkg->pkgfile); | |
1818 | } | |
1fa9d809 MD |
1819 | if (first_one_only) |
1820 | break; | |
8e25f19b MD |
1821 | continue; |
1822 | } | |
1823 | ||
63fcce5b | 1824 | tot += childInstallPkgDeps_recurse(fp, &pkg->idepon_list, |
1fa9d809 MD |
1825 | undoit, ndepth, nfirst); |
1826 | if (pkg->dsynth_install_flg) { | |
1827 | if (first_one_only) | |
1828 | break; | |
8e25f19b | 1829 | continue; |
1fa9d809 | 1830 | } |
8e25f19b MD |
1831 | pkg->dsynth_install_flg = 1; |
1832 | ||
8e25f19b MD |
1833 | /* |
1834 | * Generate package installation command | |
1835 | */ | |
fef2fc63 | 1836 | if (fp && pkg->pkgfile) { |
8e25f19b MD |
1837 | fprintf(fp, "echo 'Installing /packages/All/%s'\n", |
1838 | pkg->pkgfile); | |
349e3a76 | 1839 | fprintf(fp, "pkg install -q -U -y /packages/All/%s " |
8e25f19b MD |
1840 | "|| exit 1\n", |
1841 | pkg->pkgfile); | |
fef2fc63 | 1842 | } else if (fp) { |
8e25f19b MD |
1843 | fprintf(fp, "echo 'CANNOT FIND PKG FOR %s'\n", |
1844 | pkg->portdir); | |
1845 | } | |
fef2fc63 MD |
1846 | |
1847 | if (pkg->pkgfile) { | |
1848 | struct stat st; | |
1849 | char *path; | |
1850 | char *ptr; | |
1851 | ||
1852 | asprintf(&path, "%s/%s", RepositoryPath, pkg->pkgfile); | |
1853 | ptr = strrchr(pkg->pkgfile, '.'); | |
1854 | if (stat(path, &st) == 0) { | |
1855 | if (strcmp(ptr, ".tar") == 0) | |
1856 | tot += st.st_size; | |
1857 | else if (strcmp(ptr, ".tgz") == 0) | |
1858 | tot += st.st_size * 3; | |
1859 | else if (strcmp(ptr, ".txz") == 0) | |
1860 | tot += st.st_size * 5; | |
ef0c0bf5 KP |
1861 | else if (strcmp(ptr, ".tzst") == 0) |
1862 | tot += st.st_size * 5; | |
fef2fc63 MD |
1863 | else if (strcmp(ptr, ".tbz") == 0) |
1864 | tot += st.st_size * 3; | |
1865 | else | |
1866 | tot += st.st_size * 2; | |
1867 | } | |
1868 | free(path); | |
1869 | } | |
1fa9d809 MD |
1870 | if (first_one_only) |
1871 | break; | |
8e25f19b | 1872 | } |
fef2fc63 | 1873 | return tot; |
8e25f19b MD |
1874 | } |
1875 | ||
1876 | /* | |
1877 | * Worker process interactions. | |
1878 | * | |
1879 | * The worker process is responsible for managing the build of a single | |
1880 | * package. It is exec'd by the master dsynth and only loads the | |
1881 | * configuration. | |
1882 | * | |
1883 | * This process does not run in the chroot. It will become the reaper for | |
1884 | * all sub-processes and it will enter the chroot to execute various phases. | |
1885 | * It catches SIGINTR, SIGHUP, and SIGPIPE and will iterate, terminate, and | |
1886 | * reap all sub-process upon kill or exit. | |
1887 | * | |
1888 | * The command line forwarded to this function is: | |
1889 | * | |
1890 | * WORKER slot# socketfd portdir/subdir | |
1891 | * | |
1892 | * TERM=dumb | |
1893 | * USER=root | |
1894 | * HOME=/root | |
1895 | * LANG=C | |
1896 | * SSL_NO_VERIFY_PEER=1 | |
78f42860 | 1897 | * USE_PACKAGE_DEPENDS_ONLY=1 |
8e25f19b | 1898 | * PORTSDIR=/xports |
64824e6c MD |
1899 | * PORT_DBDIR=/options For ports options |
1900 | * PACKAGE_BUILDING=yes Don't build packages that aren't legally | |
1901 | * buildable for a binary repo. | |
8e25f19b MD |
1902 | * PKG_DBDIR=/var/db/pkg |
1903 | * PKG_CACHEDIR=/var/cache/pkg | |
64824e6c | 1904 | * PKG_CREATE_VERBOSE=yes Ensure periodic output during packaging |
8e25f19b MD |
1905 | * (custom environment) |
1906 | * PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin | |
1907 | * UNAME_s=DragonFly (example) | |
1908 | * UNAME_v=DragonFly 5.7-SYNTH (example) | |
1909 | * UNAME_p=x86_64 (example) | |
1910 | * UNAME_m=x86_64 (example) | |
1911 | * UNAME_r=5.7-SYNTH (example) | |
78f42860 | 1912 | * NO_DEPENDS=yes (conditional based on phase) |
8e25f19b MD |
1913 | * DISTDIR=/distfiles |
1914 | * WRKDIRPREFIX=/construction | |
1915 | * BATCH=yes | |
1916 | * MAKE_JOBS_NUMBER=n | |
1917 | * | |
1918 | * SETUP: | |
1919 | * ldconfig -R | |
1920 | * /usr/local/sbin/pkg-static install /packages/All/<the pkg pkg> | |
1921 | * /usr/local/sbin/pkg-static install /packages/All/<pkg> | |
1922 | * (for all dependencies) | |
1923 | * | |
1924 | * PHASES: make -C path FLAVOR=flavor <phase> | |
1925 | * check-sanity | |
1926 | * pkg-depends | |
1927 | * fetch-depends | |
1928 | * fetch | |
1929 | * checksum | |
1930 | * extract-depends | |
1931 | * extract | |
1932 | * patch-depends | |
1933 | * patch | |
1934 | * build-depends | |
1935 | * lib-depends | |
1936 | * configure | |
1937 | * build | |
1938 | * run-depends | |
1939 | * stage | |
f7f25838 | 1940 | * test (skipped) |
32f62172 | 1941 | * check-plist ('dsynth test blahblah' or 'dsynth -D everything' only) |
8e25f19b | 1942 | * package e.g. /construction/lang/perl5.28/pkg/perl5-5.28.2.txz |
f7f25838 MD |
1943 | * install (skipped) |
1944 | * deinstall (skipped) | |
8e25f19b MD |
1945 | */ |
1946 | void | |
1947 | WorkerProcess(int ac, char **av) | |
1948 | { | |
1949 | wmsg_t wmsg; | |
1950 | int fd; | |
1951 | int slot; | |
1952 | int tmpfd; | |
1953 | int pkgpkg = 0; | |
8ec23ca1 | 1954 | int status; |
8e25f19b MD |
1955 | int len; |
1956 | int do_install_phase; | |
1957 | char *portdir; | |
1958 | char *pkgfile; | |
1959 | char *flavor; | |
1960 | char *buf; | |
1961 | worker_t *work; | |
68dc2eea | 1962 | bulk_t *bulk; |
8e25f19b MD |
1963 | pkg_t pkg; |
1964 | buildenv_t *benv; | |
6fd67931 | 1965 | FILE *fp; |
8e25f19b MD |
1966 | |
1967 | /* | |
1968 | * Parse arguments | |
1969 | */ | |
8ec23ca1 | 1970 | if (ac != 6) { |
1645cafe | 1971 | dlog(DLOG_ALL, "WORKER PROCESS %d- bad arguments\n", getpid()); |
8e25f19b MD |
1972 | exit(1); |
1973 | } | |
1974 | slot = strtol(av[1], NULL, 0); | |
3bd7e0a7 MD |
1975 | setNumaDomain(slot); |
1976 | ||
8e25f19b MD |
1977 | fd = strtol(av[2], NULL, 0); /* master<->slave messaging */ |
1978 | portdir = av[3]; | |
1979 | pkgfile = av[4]; | |
3bd7e0a7 | 1980 | |
8e25f19b | 1981 | flavor = strchr(portdir, '@'); |
ffc851f6 | 1982 | if (flavor) { |
8e25f19b | 1983 | *flavor++ = 0; |
ffc851f6 MD |
1984 | asprintf(&buf, "@%s", flavor); |
1985 | WorkerFlavorPrt = buf; | |
1986 | buf = NULL; /* safety */ | |
1987 | } | |
6fd67931 | 1988 | WorkerProcFlags = strtol(av[5], NULL, 0); |
8ec23ca1 | 1989 | |
325ef124 MD |
1990 | if (WorkerProcFlags & WORKER_PROC_FETCHONLY) |
1991 | FetchOnlyOpt = 1; | |
dca89e44 MD |
1992 | if (WorkerProcFlags & WORKER_PROC_PKGV17) |
1993 | PkgVersionPkgSuffix = 1; | |
1994 | ||
8e25f19b MD |
1995 | bzero(&wmsg, sizeof(wmsg)); |
1996 | ||
ffc851f6 MD |
1997 | setproctitle("[%02d] WORKER STARTUP %s%s", |
1998 | slot, portdir, WorkerFlavorPrt); | |
8e25f19b MD |
1999 | |
2000 | if (strcmp(portdir, "ports-mgmt/pkg") == 0) | |
2001 | pkgpkg = 1; | |
2002 | ||
2003 | signal(SIGTERM, phaseTerminateSignal); | |
2004 | signal(SIGINT, phaseTerminateSignal); | |
2005 | signal(SIGHUP, phaseTerminateSignal); | |
2006 | ||
2007 | /* | |
2008 | * Set up the environment | |
2009 | */ | |
2010 | setenv("TERM", "dumb", 1); | |
2011 | setenv("USER", "root", 1); | |
2012 | setenv("HOME", "/root", 1); | |
2013 | setenv("LANG", "C", 1); | |
2014 | setenv("SSL_NO_VERIFY_PEER", "1", 1); | |
6fd67931 | 2015 | |
bd73b895 MD |
2016 | /* |
2017 | * NOTE: PKG_SUFX - pkg versions older than 1.17 | |
2018 | * PKG_COMPRESSION_FORMAT - pkg versions >= 1.17 | |
dca89e44 MD |
2019 | * |
2020 | * Avoid WARNING messages in the logs by omitting | |
2021 | * PKG_SUFX when we know the pkg version is >= 1.17. | |
bd73b895 | 2022 | */ |
6fd67931 MD |
2023 | addbuildenv("USE_PACKAGE_DEPENDS_ONLY", "yes", BENV_MAKECONF); |
2024 | addbuildenv("PORTSDIR", "/xports", BENV_MAKECONF); | |
2025 | addbuildenv("PORT_DBDIR", "/options", BENV_MAKECONF); | |
2026 | addbuildenv("PKG_DBDIR", "/var/db/pkg", BENV_MAKECONF); | |
2027 | addbuildenv("PKG_CACHEDIR", "/var/cache/pkg", BENV_MAKECONF); | |
bd73b895 | 2028 | addbuildenv("PKG_COMPRESSION_FORMAT", UsePkgSufx, BENV_MAKECONF); |
dca89e44 MD |
2029 | if (PkgVersionPkgSuffix == 0) |
2030 | addbuildenv("PKG_SUFX", UsePkgSufx, BENV_MAKECONF); | |
2031 | ||
2032 | /* | |
2033 | * We are exec'ing the worker process so various bits of global | |
2034 | * state that we want to inherit have to be passed in. | |
2035 | */ | |
6fd67931 MD |
2036 | if (WorkerProcFlags & WORKER_PROC_DEVELOPER) |
2037 | addbuildenv("DEVELOPER", "1", BENV_MAKECONF); | |
2038 | ||
2039 | /* | |
f4094b20 MD |
2040 | * CCache is a horrible unreliable hack but... leave the |
2041 | * mechanism in-place in case someone has a death wish. | |
6fd67931 MD |
2042 | */ |
2043 | if (UseCCache) { | |
2044 | addbuildenv("WITH_CCACHE_BUILD", "yes", BENV_MAKECONF); | |
2045 | addbuildenv("CCACHE_DIR", "/ccache", BENV_MAKECONF); | |
2046 | } | |
667fb2cb | 2047 | |
0df5ec4e MD |
2048 | addbuildenv("UID", "0", BENV_MAKECONF); |
2049 | addbuildenv("ARCH", ArchitectureName, BENV_MAKECONF); | |
2050 | ||
5ea034f4 AHJ |
2051 | /* |
2052 | * Always honor either the operating system detection or the | |
2053 | * operating system selection in the config file. | |
2054 | */ | |
2055 | addbuildenv("OPSYS", OperatingSystemName, BENV_MAKECONF); | |
2056 | ||
0df5ec4e | 2057 | #ifdef __DragonFly__ |
0df5ec4e MD |
2058 | addbuildenv("DFLYVERSION", VersionFromParamHeader, BENV_MAKECONF); |
2059 | addbuildenv("OSVERSION", "9999999", BENV_MAKECONF); | |
2060 | #else | |
2061 | #error "Need OS-specific data to generate make.conf" | |
8ec23ca1 MD |
2062 | #endif |
2063 | ||
0df5ec4e MD |
2064 | addbuildenv("OSREL", ReleaseName, BENV_MAKECONF); |
2065 | addbuildenv("_OSRELEASE", VersionOnlyName, BENV_MAKECONF); | |
2066 | ||
8e25f19b MD |
2067 | setenv("PATH", |
2068 | "/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin", | |
2069 | 1); | |
2070 | ||
2071 | setenv("UNAME_s", OperatingSystemName, 1); | |
2072 | setenv("UNAME_v", VersionName, 1); | |
2073 | setenv("UNAME_p", ArchitectureName, 1); | |
2074 | setenv("UNAME_m", MachineName, 1); | |
2075 | setenv("UNAME_r", ReleaseName, 1); | |
2076 | ||
6fd67931 MD |
2077 | addbuildenv("DISTDIR", "/distfiles", BENV_MAKECONF); |
2078 | addbuildenv("WRKDIRPREFIX", "/construction", BENV_MAKECONF); | |
2079 | addbuildenv("BATCH", "yes", BENV_MAKECONF); | |
8e25f19b | 2080 | |
64824e6c MD |
2081 | /* |
2082 | * Special consideration | |
2083 | * | |
2084 | * PACKAGE_BUILDING - Disallow packaging ports which do not allow | |
2085 | * for binary distribution. | |
2086 | * | |
2087 | * PKG_CREATE_VERBOSE - Ensure periodic output during the packaging | |
2088 | * process to avoid a watchdog timeout. | |
2089 | * | |
2090 | */ | |
6fd67931 MD |
2091 | addbuildenv("PACKAGE_BUILDING", "yes", BENV_MAKECONF); |
2092 | addbuildenv("PKG_CREATE_VERBOSE", "yes", BENV_MAKECONF); | |
8e25f19b | 2093 | asprintf(&buf, "%d", MaxJobs); |
6fd67931 | 2094 | addbuildenv("MAKE_JOBS_NUMBER", buf, BENV_MAKECONF); |
ffc851f6 | 2095 | freestrp(&buf); |
8e25f19b MD |
2096 | |
2097 | if (flavor) | |
2098 | setenv("FLAVOR", flavor, 1); | |
2099 | ||
8e25f19b MD |
2100 | /* |
2101 | * Become the reaper | |
2102 | */ | |
2103 | if (procctl(P_PID, getpid(), PROC_REAP_ACQUIRE, NULL) < 0) | |
2104 | dfatal_errno("procctl() - Cannot become reaper"); | |
2105 | ||
2106 | /* | |
2107 | * Initialize a worker structure | |
2108 | */ | |
2109 | DoInitBuild(slot); | |
2110 | ||
2111 | bzero(&pkg, sizeof(pkg)); | |
2112 | pkg.portdir = portdir; /* sans flavor */ | |
2113 | pkg.pkgfile = pkgfile; | |
2114 | if (strchr(portdir, '/')) | |
2115 | len = strchr(portdir, '/') - portdir; | |
2116 | else | |
2117 | len = 0; | |
2118 | ||
2119 | /* | |
2120 | * Setup the logfile | |
2121 | */ | |
2122 | asprintf(&pkg.logfile, | |
2123 | "%s/%*.*s___%s%s%s.log", | |
2124 | LogsPath, len, len, portdir, | |
2125 | ((portdir[len] == '/') ? portdir + len + 1 : portdir + len), | |
2126 | (flavor ? "@" : ""), | |
2127 | (flavor ? flavor : "")); | |
2128 | tmpfd = open(pkg.logfile, O_RDWR|O_CREAT|O_TRUNC, 0666); | |
2129 | if (tmpfd >= 0) { | |
54f2fefc MD |
2130 | if (DebugOpt >= 2) { |
2131 | dlog(DLOG_ALL, "[%03d] %s LOGFILE %s\n", | |
2132 | slot, pkg.portdir, pkg.logfile); | |
2133 | } | |
8e25f19b MD |
2134 | close(tmpfd); |
2135 | } else { | |
2136 | dlog(DLOG_ALL, "[%03d] LOGFILE %s (create failed)\n", | |
2137 | slot, pkg.logfile); | |
2138 | } | |
2139 | ||
2140 | /* | |
2141 | * Setup the work structure. Because this is an exec'd sub-process, | |
2142 | * there is only one work structure. | |
2143 | */ | |
2144 | work = &WorkerAry[0]; | |
2145 | work->flavor = flavor; | |
2146 | work->fds[0] = fd; | |
2147 | work->pkg = &pkg; | |
2148 | work->start_time = time(NULL); | |
2149 | ||
2150 | /* | |
6fd67931 | 2151 | * Do mounts |
8e25f19b MD |
2152 | */ |
2153 | SigWork = work; | |
ffc851f6 MD |
2154 | setproctitle("[%02d] WORKER MOUNTS %s%s", |
2155 | slot, portdir, WorkerFlavorPrt); | |
8e25f19b MD |
2156 | DoWorkerMounts(work); |
2157 | ||
6fd67931 MD |
2158 | /* |
2159 | * Generate an /etc/make.conf in the build base | |
2160 | */ | |
2161 | asprintf(&buf, "%s/etc/make.conf", work->basedir); | |
2162 | fp = fopen(buf, "w"); | |
2163 | dassert_errno(fp, "Unable to create %s\n", buf); | |
2164 | for (benv = BuildEnv; benv; benv = benv->next) { | |
31f2ea22 MD |
2165 | if (benv->type & BENV_PKGLIST) |
2166 | continue; | |
2167 | if ((benv->type & BENV_CMDMASK) == BENV_MAKECONF) { | |
2168 | if (DebugOpt >= 2) { | |
2169 | dlog(DLOG_ALL, "[%03d] ENV %s=%s\n", | |
2170 | slot, benv->label, benv->data); | |
2171 | } | |
6fd67931 | 2172 | fprintf(fp, "%s=%s\n", benv->label, benv->data); |
31f2ea22 | 2173 | } |
6fd67931 MD |
2174 | } |
2175 | fclose(fp); | |
ffc851f6 | 2176 | freestrp(&buf); |
6fd67931 | 2177 | |
68dc2eea MD |
2178 | /* |
2179 | * Set up our hooks | |
2180 | */ | |
2181 | if (UsingHooks) | |
2182 | initbulk(childHookRun, MaxBulk); | |
2183 | ||
6fd67931 MD |
2184 | /* |
2185 | * Start phases | |
2186 | */ | |
8e25f19b MD |
2187 | wmsg.cmd = WMSG_CMD_INSTALL_PKGS; |
2188 | ipcwritemsg(fd, &wmsg); | |
2189 | status = ipcreadmsg(fd, &wmsg); | |
2190 | if (status < 0 || wmsg.cmd != WMSG_RES_INSTALL_PKGS) | |
2191 | dfatal("pkg installation handshake failed"); | |
2192 | do_install_phase = wmsg.status; | |
325ef124 MD |
2193 | if (FetchOnlyOpt) |
2194 | do_install_phase = 0; | |
8e25f19b MD |
2195 | |
2196 | wmsg.cmd = WMSG_CMD_STATUS_UPDATE; | |
2197 | wmsg.phase = PHASE_INSTALL_PKGS; | |
2198 | wmsg.lines = 0; | |
2199 | ||
2200 | status = ipcwritemsg(fd, &wmsg); | |
2201 | ||
325ef124 | 2202 | if (pkgpkg && FetchOnlyOpt == 0) { |
8e25f19b MD |
2203 | dophase(work, &wmsg, |
2204 | WDOG5, PHASE_PACKAGE, "package"); | |
2205 | } else { | |
9fdb3f28 AHJ |
2206 | /* |
2207 | * Dump as much information of the build process as possible. | |
2208 | * Will help troubleshooting port build breakages. | |
2209 | * Only enabled when DEVELOPER is set. | |
2210 | * | |
2211 | * This sort of mimics what synth did. | |
2212 | */ | |
2213 | if (WorkerProcFlags & WORKER_PROC_DEVELOPER) { | |
2214 | dophase(work, &wmsg, | |
2215 | WDOG2, PHASE_DUMP_ENV, "Environment"); | |
2216 | dophase(work, &wmsg, | |
2217 | WDOG2, PHASE_SHOW_CONFIG, "showconfig"); | |
2218 | dophase(work, &wmsg, | |
2219 | WDOG2, PHASE_DUMP_VAR, "CONFIGURE_ENV"); | |
2220 | dophase(work, &wmsg, | |
2221 | WDOG2, PHASE_DUMP_VAR, "CONFIGURE_ARGS"); | |
2222 | dophase(work, &wmsg, | |
2223 | WDOG2, PHASE_DUMP_VAR, "MAKE_ENV"); | |
2224 | dophase(work, &wmsg, | |
2225 | WDOG2, PHASE_DUMP_VAR, "MAKE_ARGS"); | |
2226 | dophase(work, &wmsg, | |
2227 | WDOG2, PHASE_DUMP_VAR, "PLIST_SUB"); | |
2228 | dophase(work, &wmsg, | |
2229 | WDOG2, PHASE_DUMP_VAR, "SUB_LIST"); | |
2230 | dophase(work, &wmsg, | |
2231 | WDOG2, PHASE_DUMP_MAKECONF, "/etc/make.conf"); | |
2232 | } | |
2233 | ||
8e25f19b MD |
2234 | if (do_install_phase) { |
2235 | dophase(work, &wmsg, | |
2236 | WDOG4, PHASE_INSTALL_PKGS, "setup"); | |
2237 | } | |
2238 | dophase(work, &wmsg, | |
2239 | WDOG2, PHASE_CHECK_SANITY, "check-sanity"); | |
325ef124 MD |
2240 | if (FetchOnlyOpt == 0) { |
2241 | dophase(work, &wmsg, | |
2242 | WDOG2, PHASE_PKG_DEPENDS, "pkg-depends"); | |
2243 | } | |
8e25f19b MD |
2244 | dophase(work, &wmsg, |
2245 | WDOG7, PHASE_FETCH_DEPENDS, "fetch-depends"); | |
2246 | dophase(work, &wmsg, | |
2247 | WDOG7, PHASE_FETCH, "fetch"); | |
2248 | dophase(work, &wmsg, | |
2249 | WDOG2, PHASE_CHECKSUM, "checksum"); | |
325ef124 MD |
2250 | if (FetchOnlyOpt == 0) { |
2251 | dophase(work, &wmsg, | |
2252 | WDOG3, PHASE_EXTRACT_DEPENDS, "extract-depends"); | |
2253 | dophase(work, &wmsg, | |
2254 | WDOG3, PHASE_EXTRACT, "extract"); | |
2255 | dophase(work, &wmsg, | |
2256 | WDOG2, PHASE_PATCH_DEPENDS, "patch-depends"); | |
2257 | dophase(work, &wmsg, | |
2258 | WDOG2, PHASE_PATCH, "patch"); | |
2259 | dophase(work, &wmsg, | |
2260 | WDOG5, PHASE_BUILD_DEPENDS, "build-depends"); | |
2261 | dophase(work, &wmsg, | |
2262 | WDOG5, PHASE_LIB_DEPENDS, "lib-depends"); | |
2263 | dophase(work, &wmsg, | |
2264 | WDOG3, PHASE_CONFIGURE, "configure"); | |
2265 | dophase(work, &wmsg, | |
2266 | WDOG9, PHASE_BUILD, "build"); | |
2267 | dophase(work, &wmsg, | |
2268 | WDOG5, PHASE_RUN_DEPENDS, "run-depends"); | |
2269 | dophase(work, &wmsg, | |
2270 | WDOG5, PHASE_STAGE, "stage"); | |
8e25f19b | 2271 | #if 0 |
325ef124 MD |
2272 | dophase(work, &wmsg, |
2273 | WDOG5, PHASE_TEST, "test"); | |
8e25f19b | 2274 | #endif |
325ef124 MD |
2275 | if (WorkerProcFlags & WORKER_PROC_CHECK_PLIST) { |
2276 | dophase(work, &wmsg, | |
2277 | WDOG1, PHASE_CHECK_PLIST, "check-plist"); | |
2278 | } | |
32f62172 | 2279 | dophase(work, &wmsg, |
325ef124 | 2280 | WDOG5, PHASE_PACKAGE, "package"); |
29d8adfc | 2281 | |
325ef124 MD |
2282 | if (WorkerProcFlags & WORKER_PROC_INSTALL) { |
2283 | dophase(work, &wmsg, | |
2284 | WDOG5, PHASE_INSTALL, "install"); | |
2285 | } | |
29d8adfc | 2286 | |
325ef124 MD |
2287 | if (WorkerProcFlags & WORKER_PROC_DEINSTALL) { |
2288 | dophase(work, &wmsg, | |
2289 | WDOG5, PHASE_DEINSTALL, "deinstall"); | |
2290 | } | |
29d8adfc | 2291 | } |
8e25f19b MD |
2292 | } |
2293 | ||
1d6e00cd MD |
2294 | if (MasterPtyFd >= 0) { |
2295 | close(MasterPtyFd); | |
2296 | MasterPtyFd = -1; | |
2297 | } | |
2298 | ||
ffc851f6 MD |
2299 | setproctitle("[%02d] WORKER CLEANUP %s%s", |
2300 | slot, portdir, WorkerFlavorPrt); | |
8e25f19b MD |
2301 | |
2302 | /* | |
2303 | * Copy the package to the repo. | |
2304 | */ | |
325ef124 | 2305 | if (work->accum_error == 0 && FetchOnlyOpt == 0) { |
8e25f19b MD |
2306 | char *b1; |
2307 | char *b2; | |
2308 | ||
2309 | asprintf(&b1, "%s/construction/%s/pkg/%s", | |
2310 | work->basedir, pkg.portdir, pkg.pkgfile); | |
2311 | asprintf(&b2, "%s/%s", RepositoryPath, pkg.pkgfile); | |
2312 | if (copyfile(b1, b2)) { | |
2313 | ++work->accum_error; | |
2314 | dlog(DLOG_ALL, "[%03d] %s Unable to copy %s to %s\n", | |
2315 | work->index, pkg.portdir, b1, b2); | |
2316 | } | |
2317 | free(b1); | |
2318 | free(b2); | |
2319 | } | |
2320 | ||
8ec23ca1 MD |
2321 | /* |
2322 | * Unmount, unless we are in DebugStopMode. | |
2323 | */ | |
6fd67931 | 2324 | if ((WorkerProcFlags & WORKER_PROC_DEBUGSTOP) == 0) |
8ec23ca1 | 2325 | DoWorkerUnmounts(work); |
8e25f19b | 2326 | |
8ec23ca1 MD |
2327 | /* |
2328 | * Send completion status to master dsynth worker thread. | |
2329 | */ | |
8e25f19b MD |
2330 | if (work->accum_error) { |
2331 | wmsg.cmd = WMSG_CMD_FAILURE; | |
2332 | } else { | |
2333 | wmsg.cmd = WMSG_CMD_SUCCESS; | |
2334 | } | |
8ec23ca1 | 2335 | ipcwritemsg(fd, &wmsg); |
6fd67931 | 2336 | if (WorkerProcFlags & WORKER_PROC_DEBUGSTOP) { |
8ec23ca1 MD |
2337 | wmsg.cmd = WMSG_CMD_FREEZEWORKER; |
2338 | ipcwritemsg(fd, &wmsg); | |
2339 | } | |
68dc2eea MD |
2340 | if (UsingHooks) { |
2341 | while ((bulk = getbulk()) != NULL) | |
2342 | freebulk(bulk); | |
2343 | donebulk(); | |
2344 | } | |
8e25f19b MD |
2345 | } |
2346 | ||
77d66d4e AHJ |
2347 | static int |
2348 | check_dns(void) | |
2349 | { | |
2350 | char check_domains[4][24] = { | |
2351 | "www.dragonflybsd.org", | |
2352 | "www.freebsd.org", | |
2353 | "www.openbsd.org", | |
2354 | "www.netbsd.org", | |
2355 | }; | |
2356 | int failures = 0; | |
2357 | ||
2358 | for (int i = 0; i < 4; i++) | |
2359 | if (gethostbyname (check_domains[i]) == NULL) | |
2360 | failures++; | |
2361 | if (failures > 1) | |
2362 | return -1; | |
2363 | ||
2364 | return 0; | |
2365 | } | |
2366 | ||
8e25f19b MD |
2367 | static void |
2368 | dophase(worker_t *work, wmsg_t *wmsg, int wdog, int phaseid, const char *phase) | |
2369 | { | |
2370 | pkg_t *pkg = work->pkg; | |
2371 | char buf[1024]; | |
2372 | pid_t pid; | |
8e25f19b | 2373 | int status; |
7880dcf7 MD |
2374 | int ms; |
2375 | pid_t wpid; | |
2376 | int wpid_reaped; | |
8e25f19b MD |
2377 | int fdlog; |
2378 | time_t start_time; | |
2379 | time_t last_time; | |
2380 | time_t next_time; | |
2381 | time_t wdog_time; | |
2382 | FILE *fp; | |
2383 | ||
2384 | if (work->accum_error) | |
2385 | return; | |
ffc851f6 MD |
2386 | setproctitle("[%02d] WORKER %-8.8s %s%s", |
2387 | work->index, phase, pkg->portdir, WorkerFlavorPrt); | |
8e25f19b MD |
2388 | wmsg->phase = phaseid; |
2389 | if (ipcwritemsg(work->fds[0], wmsg) < 0) { | |
2390 | dlog(DLOG_ALL, "[%03d] %s Lost Communication with dsynth, " | |
2391 | "aborting worker\n", | |
2392 | work->index, pkg->portdir); | |
2393 | ++work->accum_error; | |
2394 | return; | |
2395 | } | |
2396 | ||
2397 | /* | |
168bca18 | 2398 | * Execute the port make command in chroot on a pty. |
8e25f19b MD |
2399 | */ |
2400 | fflush(stdout); | |
2401 | fflush(stderr); | |
1d6e00cd MD |
2402 | if (MasterPtyFd >= 0) { |
2403 | int slavefd; | |
2404 | ||
168bca18 MD |
2405 | /* |
2406 | * NOTE: We can't open the slave in the child because the | |
2407 | * master may race a disconnection test. If we open | |
2408 | * it in the parent our close() will flush any pending | |
2409 | * output not read by the master (which is the same | |
2410 | * parent process) and deadlock. | |
2411 | * | |
2412 | * Solve this by hand-shaking the slave tty to give | |
f51d778e MD |
2413 | * the master time to close its slavefd (after this |
2414 | * section). | |
a9300d32 MD |
2415 | * |
2416 | * Leave the tty defaults intact, which also likely | |
2417 | * means it will be in line-buffered mode, so handshake | |
2418 | * with a full line. | |
2074bc97 MD |
2419 | * |
2420 | * TODO: Our handshake probably echos back to the master pty | |
9bb2c592 MD |
2421 | * due to tty echo, and ends up in the log, so just |
2422 | * pass through a newline. | |
168bca18 | 2423 | */ |
1d6e00cd MD |
2424 | slavefd = open(ptsname(MasterPtyFd), O_RDWR); |
2425 | dassert_errno(slavefd >= 0, "Cannot open slave pty"); | |
f51d778e MD |
2426 | |
2427 | /* | |
2428 | * Now do the fork. | |
2429 | */ | |
1d6e00cd MD |
2430 | pid = fork(); |
2431 | if (pid == 0) { | |
2432 | login_tty(slavefd); | |
a9300d32 | 2433 | /* login_tty() closes slavefd */ |
1d6e00cd MD |
2434 | } else { |
2435 | close(slavefd); | |
2436 | } | |
2437 | } else { | |
f51d778e MD |
2438 | /* |
2439 | * Initial MasterPtyFd for the slot, just use forkpty(). | |
2440 | */ | |
1d6e00cd MD |
2441 | pid = forkpty(&MasterPtyFd, NULL, NULL, NULL); |
2442 | } | |
2443 | ||
f51d778e MD |
2444 | /* |
2445 | * The slave must make sure the master has time to close slavefd | |
2446 | * in the re-use case before going its merry way. The master needs | |
2447 | * to set terminal modes and the window as well. | |
2448 | */ | |
1d6e00cd | 2449 | if (pid == 0) { |
f51d778e | 2450 | /* |
6d1478d9 | 2451 | * Slave nices itself and waits for handshake |
f51d778e MD |
2452 | */ |
2453 | char ttybuf[2]; | |
a574cbf0 | 2454 | |
6d1478d9 MD |
2455 | /* |
2456 | * Self-nice to be nice (ignore any error) | |
2457 | */ | |
2458 | if (NiceOpt) | |
2459 | setpriority(PRIO_PROCESS, 0, NiceOpt); | |
2460 | ||
f51d778e MD |
2461 | read(0, ttybuf, 1); |
2462 | } else { | |
a574cbf0 MD |
2463 | /* |
2464 | * We are going through a pty, so set the tty modes to | |
2465 | * Set tty modes so we do not get ^M's in the log files. | |
2466 | * | |
2467 | * This isn't fatal if it doesn't work. Remember that | |
2468 | * our output goes through the pty to the management | |
2469 | * process which will log it. | |
2470 | */ | |
f51d778e MD |
2471 | struct termios tio; |
2472 | struct winsize win; | |
2473 | ||
2474 | if (tcgetattr(MasterPtyFd, &tio) == 0) { | |
a574cbf0 MD |
2475 | tio.c_oflag |= OPOST | ONOCR; |
2476 | tio.c_oflag &= ~(OCRNL | ONLCR); | |
2477 | tio.c_iflag |= ICRNL; | |
f51d778e MD |
2478 | tio.c_iflag &= ~(INLCR | IGNCR); |
2479 | if (tcsetattr(MasterPtyFd, TCSANOW, &tio)) { | |
a574cbf0 MD |
2480 | printf("tcsetattr failed: %s\n", |
2481 | strerror(errno)); | |
2482 | } | |
f51d778e MD |
2483 | |
2484 | /* | |
2485 | * Give the tty a non-zero columns field. | |
2486 | * This fixes at least one port (textproc/po4a) | |
2487 | */ | |
2488 | if (ioctl(MasterPtyFd, TIOCGWINSZ, &win) == 0) { | |
2489 | win.ws_col = 80; | |
2490 | ioctl(MasterPtyFd, TIOCSWINSZ, &win); | |
2491 | } else { | |
2492 | printf("TIOCGWINSZ failed: %s\n", | |
2493 | strerror(errno)); | |
2494 | } | |
2495 | ||
a574cbf0 MD |
2496 | } else { |
2497 | printf("tcgetattr failed: %s\n", strerror(errno)); | |
2498 | } | |
2499 | ||
f51d778e MD |
2500 | /* |
2501 | * Master issues handshake | |
2502 | */ | |
2503 | write(MasterPtyFd, "\n", 1); | |
2504 | } | |
2505 | ||
2506 | if (pid == 0) { | |
78f42860 MD |
2507 | /* |
2508 | * Additional phase-specific environment variables | |
2509 | * | |
2510 | * - Do not try to process missing depends outside of the | |
2511 | * depends phases. Also relies on USE_PACKAGE_DEPENDS_ONLY | |
2512 | * in the make.conf. | |
2513 | */ | |
2514 | switch(phaseid) { | |
2515 | case PHASE_CHECK_SANITY: | |
2516 | case PHASE_FETCH: | |
2517 | case PHASE_CHECKSUM: | |
2518 | case PHASE_EXTRACT: | |
2519 | case PHASE_PATCH: | |
2520 | case PHASE_CONFIGURE: | |
2521 | case PHASE_STAGE: | |
2522 | case PHASE_TEST: | |
2523 | case PHASE_CHECK_PLIST: | |
78f42860 MD |
2524 | case PHASE_INSTALL: |
2525 | case PHASE_DEINSTALL: | |
2526 | break; | |
2527 | case PHASE_PKG_DEPENDS: | |
2528 | case PHASE_FETCH_DEPENDS: | |
2529 | case PHASE_EXTRACT_DEPENDS: | |
2530 | case PHASE_PATCH_DEPENDS: | |
2531 | case PHASE_BUILD_DEPENDS: | |
2532 | case PHASE_LIB_DEPENDS: | |
2533 | case PHASE_RUN_DEPENDS: | |
2534 | break; | |
2535 | default: | |
2536 | setenv("NO_DEPENDS", "1", 1); | |
2537 | break; | |
2538 | } | |
2539 | ||
a574cbf0 MD |
2540 | /* |
2541 | * Clean-up, chdir, and chroot. | |
2542 | */ | |
8e25f19b MD |
2543 | closefrom(3); |
2544 | if (chdir(work->basedir) < 0) | |
2545 | dfatal_errno("chdir in phase initialization"); | |
2546 | if (chroot(work->basedir) < 0) | |
2547 | dfatal_errno("chroot in phase initialization"); | |
2548 | ||
77d66d4e AHJ |
2549 | /* Explicitly fail when DNS is not working */ |
2550 | if (check_dns() != 0) | |
2551 | dfatal("DNS resolution not working"); | |
2552 | ||
9c4c701f MD |
2553 | /* |
2554 | * We have a choice here on how to handle stdin (fd 0). | |
2555 | * We can leave it connected to the pty in which case | |
2556 | * the build will just block if it tries to ask a | |
2557 | * question (and the watchdog will kill it, eventually), | |
2558 | * or we can try to EOF the pty, or we can attach /dev/null | |
2559 | * to descriptor 0. | |
2560 | */ | |
2561 | if (NullStdinOpt) { | |
2562 | int fd; | |
2563 | ||
2564 | fd = open("/dev/null", O_RDWR); | |
2565 | dassert_errno(fd >= 0, "cannot open /dev/null"); | |
2566 | if (fd != 0) { | |
2567 | dup2(fd, 0); | |
2568 | close(fd); | |
2569 | } | |
2570 | } | |
2571 | ||
2b3f93ea MD |
2572 | /* |
2573 | * Capability restrictions to make root safer. | |
2574 | */ | |
2575 | set_capability_restrictions(); | |
2576 | ||
8e25f19b MD |
2577 | /* |
2578 | * Execute the appropriate command. | |
2579 | */ | |
2580 | switch(phaseid) { | |
2581 | case PHASE_INSTALL_PKGS: | |
2582 | snprintf(buf, sizeof(buf), "/tmp/dsynth_install_pkgs"); | |
2583 | execl(buf, buf, NULL); | |
2584 | break; | |
9fdb3f28 AHJ |
2585 | case PHASE_DUMP_ENV: |
2586 | snprintf(buf, sizeof(buf), "/usr/bin/env"); | |
2587 | execl(buf, buf, NULL); | |
2588 | break; | |
2589 | case PHASE_DUMP_VAR: | |
2590 | snprintf(buf, sizeof(buf), "/xports/%s", pkg->portdir); | |
2591 | execl(MAKE_BINARY, MAKE_BINARY, "-C", buf, "-V", phase, | |
2592 | NULL); | |
2593 | break; | |
2594 | case PHASE_DUMP_MAKECONF: | |
2595 | snprintf(buf, sizeof(buf), "/bin/cat"); | |
2596 | execl(buf, buf, "/etc/make.conf", NULL); | |
2597 | break; | |
2598 | case PHASE_SHOW_CONFIG: | |
2599 | /* fall-through */ | |
8e25f19b MD |
2600 | default: |
2601 | snprintf(buf, sizeof(buf), "/xports/%s", pkg->portdir); | |
2602 | execl(MAKE_BINARY, MAKE_BINARY, "-C", buf, phase, NULL); | |
2603 | break; | |
2604 | } | |
2605 | _exit(1); | |
2606 | } | |
7880dcf7 | 2607 | fcntl(MasterPtyFd, F_SETFL, O_NONBLOCK); |
8e25f19b | 2608 | |
1d6e00cd | 2609 | if (pid < 0) { |
6a3a20b1 MD |
2610 | dlog(DLOG_ALL, "[%03d] %s Fork Failed: %s\n", |
2611 | work->index, pkg->logfile, strerror(errno)); | |
1d6e00cd MD |
2612 | ++work->accum_error; |
2613 | return; | |
2614 | } | |
2615 | ||
8e25f19b MD |
2616 | SigPid = pid; |
2617 | ||
2618 | fdlog = open(pkg->logfile, O_RDWR|O_CREAT|O_APPEND, 0644); | |
2619 | if (fdlog < 0) { | |
2620 | dlog(DLOG_ALL, "[%03d] %s Cannot open logfile '%s': %s\n", | |
2621 | work->index, pkg->portdir, | |
2622 | pkg->logfile, strerror(errno)); | |
2623 | } | |
6fd67931 MD |
2624 | |
2625 | snprintf(buf, sizeof(buf), | |
3cebe4a8 MD |
2626 | "----------------------------------------" |
2627 | "---------------------------------------\n" | |
6fd67931 | 2628 | "-- Phase: %s\n" |
3cebe4a8 MD |
2629 | "----------------------------------------" |
2630 | "---------------------------------------\n", | |
6fd67931 MD |
2631 | phase); |
2632 | write(fdlog, buf, strlen(buf)); | |
2633 | ||
8e25f19b MD |
2634 | start_time = time(NULL); |
2635 | last_time = start_time; | |
2636 | wdog_time = start_time; | |
7880dcf7 | 2637 | wpid_reaped = 0; |
8e25f19b MD |
2638 | |
2639 | status = 0; | |
2640 | for (;;) { | |
7880dcf7 MD |
2641 | ms = mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time); |
2642 | if (ms == MPTY_FAILED) { | |
2643 | dlog(DLOG_ALL, | |
2644 | "[%03d] %s lost pty in phase %s, terminating\n", | |
2645 | work->index, pkg->portdir, phase); | |
2646 | break; | |
8e25f19b | 2647 | } |
7880dcf7 MD |
2648 | if (ms == MPTY_EOF) |
2649 | break; | |
8e25f19b MD |
2650 | |
2651 | /* | |
2652 | * Generally speaking update status once a second. | |
2653 | * This also allows us to detect if the management | |
2654 | * dsynth process has gone away. | |
2655 | */ | |
2656 | next_time = time(NULL); | |
2657 | if (next_time != last_time) { | |
2658 | double dload[3]; | |
2659 | double dv; | |
2660 | int wdog_scaled; | |
2661 | ||
2662 | /* | |
2663 | * Send status update to the worker management thread | |
2664 | * in the master dsynth process. Remember, *WE* are | |
2665 | * the worker management process sub-fork. | |
2666 | */ | |
2667 | if (ipcwritemsg(work->fds[0], wmsg) < 0) | |
2668 | break; | |
2669 | last_time = next_time; | |
2670 | ||
2671 | /* | |
2672 | * Watchdog scaling | |
2673 | */ | |
2674 | getloadavg(dload, 3); | |
32fc5bbf | 2675 | adjloadavg(dload); |
8e25f19b MD |
2676 | dv = dload[2] / NumCores; |
2677 | if (dv < (double)NumCores) { | |
2678 | wdog_scaled = wdog; | |
2679 | } else { | |
2680 | if (dv > 4.0 * NumCores) | |
2681 | dv = 4.0 * NumCores; | |
2682 | wdog_scaled = wdog * dv / NumCores; | |
2683 | } | |
2684 | ||
2685 | /* | |
2686 | * Watchdog | |
2687 | */ | |
2688 | if (next_time - wdog_time >= wdog_scaled * 60) { | |
2689 | snprintf(buf, sizeof(buf), | |
2690 | "\n--------\n" | |
2691 | "WATCHDOG TIMEOUT FOR %s in %s " | |
2692 | "after %d minutes\n" | |
2693 | "Killing pid %d\n" | |
2694 | "--------\n", | |
2695 | pkg->portdir, phase, wdog_scaled, pid); | |
2696 | if (fdlog >= 0) | |
2697 | write(fdlog, buf, strlen(buf)); | |
2698 | dlog(DLOG_ALL, | |
a574cbf0 | 2699 | "[%03d] %s WATCHDOG TIMEOUT in %s " |
8e25f19b MD |
2700 | "after %d minutes (%d min scaled)\n", |
2701 | work->index, pkg->portdir, phase, | |
2702 | wdog, wdog_scaled); | |
2703 | kill(pid, SIGKILL); | |
2704 | ++work->accum_error; | |
2705 | break; | |
2706 | } | |
2707 | } | |
2708 | ||
2709 | /* | |
2710 | * Check process exit. Normally the pty will EOF | |
2711 | * but if background processes remain we need to | |
2712 | * check here to see if our primary exec is done, | |
2713 | * so we can break out and reap those processes. | |
e86766ee MD |
2714 | * |
2715 | * Generally reap any other processes we have inherited | |
2716 | * while we are here. | |
8e25f19b | 2717 | */ |
e86766ee MD |
2718 | do { |
2719 | wpid = wait3(&status, WNOHANG, NULL); | |
2720 | } while (wpid > 0 && wpid != pid); | |
7880dcf7 MD |
2721 | if (wpid == pid && WIFEXITED(status)) { |
2722 | wpid_reaped = 1; | |
8e25f19b | 2723 | break; |
7880dcf7 | 2724 | } |
8e25f19b | 2725 | } |
8e25f19b MD |
2726 | |
2727 | next_time = time(NULL); | |
2728 | ||
ffc851f6 MD |
2729 | setproctitle("[%02d] WORKER EXITREAP %s%s", |
2730 | work->index, pkg->portdir, WorkerFlavorPrt); | |
7880dcf7 MD |
2731 | |
2732 | /* | |
2733 | * We usually get here due to a mpty EOF, but not always as there | |
2734 | * could be persistent processes still holding the slave. Finish | |
2735 | * up getting the exit status for the main process we are waiting | |
2736 | * on and clean out any data left on the MasterPtyFd (as it could | |
2737 | * be blocking the exit). | |
2738 | */ | |
2739 | while (wpid_reaped == 0) { | |
2740 | (void)mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time); | |
2741 | wpid = waitpid(pid, &status, WNOHANG); | |
2742 | if (wpid == pid && WIFEXITED(status)) { | |
2743 | wpid_reaped = 1; | |
2744 | break; | |
2745 | } | |
2746 | if (wpid < 0 && errno != EINTR) { | |
2747 | break; | |
8e25f19b MD |
2748 | } |
2749 | ||
7880dcf7 MD |
2750 | /* |
2751 | * Safety. The normal phase waits until the fork/exec'd | |
2752 | * pid finishes, causing a pty EOF on exit (the slave | |
2753 | * descriptor is closed by the kernel on exit so the | |
2754 | * process should already have exited). | |
2755 | * | |
2756 | * However, it is also possible to get here if the pty fails | |
2757 | * for some reason. In this case, make sure that the process | |
2758 | * is killed. | |
2759 | */ | |
2760 | kill(pid, SIGKILL); | |
8e25f19b MD |
2761 | } |
2762 | ||
7880dcf7 MD |
2763 | /* |
2764 | * Clean out anything left on the pty but don't wait around | |
2765 | * because there could be background processes preventing the | |
2766 | * slave side from closing. | |
2767 | */ | |
2768 | while (mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time) == MPTY_DATA) | |
2769 | ; | |
2770 | ||
2771 | /* | |
2772 | * Report on the exit condition. If the pid was somehow lost | |
2773 | * (probably due to someone gdb'ing the process), assume an error. | |
2774 | */ | |
2775 | if (wpid_reaped) { | |
2776 | if (WEXITSTATUS(status)) { | |
a574cbf0 MD |
2777 | dlog(DLOG_ALL | DLOG_FILTER, |
2778 | "[%03d] %s Build phase '%s' failed exit %d\n", | |
7880dcf7 MD |
2779 | work->index, pkg->portdir, phase, |
2780 | WEXITSTATUS(status)); | |
2781 | ++work->accum_error; | |
2782 | } | |
2783 | } else { | |
2784 | dlog(DLOG_ALL, "[%03d] %s Build phase '%s' failed - lost pid\n", | |
8e25f19b MD |
2785 | work->index, pkg->portdir, phase); |
2786 | ++work->accum_error; | |
2787 | } | |
2788 | ||
2789 | /* | |
2790 | * Kill any processes still running (sometimes processes end up in | |
2791 | * the background during a dports build), and clean up any other | |
2792 | * children that we have inherited. | |
2793 | */ | |
2794 | phaseReapAll(); | |
2795 | ||
516819d9 MD |
2796 | /* |
2797 | * After the extraction phase add the space used by /construction | |
2798 | * to the memory use. This helps us reduce the amount of paging | |
2799 | * we do due to extremely large package extractions (languages, | |
2800 | * chromium, etc). | |
2801 | * | |
2802 | * (dsynth already estimated the space used by the package deps | |
2803 | * up front, but this will help us further). | |
2804 | */ | |
2805 | if (work->accum_error == 0 && phaseid == PHASE_EXTRACT) { | |
2806 | struct statfs sfs; | |
2807 | char *b1; | |
2808 | ||
2809 | asprintf(&b1, "%s/construction", work->basedir); | |
2810 | if (statfs(b1, &sfs) == 0) { | |
2811 | wmsg->memuse = (sfs.f_blocks - sfs.f_bfree) * | |
2812 | sfs.f_bsize; | |
2813 | ipcwritemsg(work->fds[0], wmsg); | |
2814 | } | |
2815 | } | |
2816 | ||
8e25f19b MD |
2817 | /* |
2818 | * Update log | |
2819 | */ | |
2820 | if (fdlog >= 0) { | |
2821 | struct stat st; | |
2822 | int h; | |
2823 | int m; | |
2824 | int s; | |
2825 | ||
2826 | last_time = next_time - start_time; | |
2827 | s = last_time % 60; | |
2828 | m = last_time / 60 % 60; | |
2829 | h = last_time / 3600; | |
2830 | ||
2831 | fp = fdopen(fdlog, "a"); | |
2832 | if (fp == NULL) { | |
2833 | dlog(DLOG_ALL, "[%03d] %s Cannot fdopen fdlog: %s %d\n", | |
2834 | work->index, pkg->portdir, | |
2835 | strerror(errno), fstat(fdlog, &st)); | |
2836 | close(fdlog); | |
2837 | goto skip; | |
2838 | } | |
2839 | ||
6fd67931 | 2840 | fprintf(fp, "\n"); |
8e25f19b | 2841 | if (work->accum_error) { |
6fd67931 | 2842 | fprintf(fp, "FAILED %02d:%02d:%02d\n", h, m, s); |
8e25f19b | 2843 | } else { |
516819d9 MD |
2844 | if (phaseid == PHASE_EXTRACT && wmsg->memuse) { |
2845 | fprintf(fp, "Extracted Memory Use: %6.2fM\n", | |
2846 | wmsg->memuse / (1024.0 * 1024.0)); | |
2847 | } | |
6fd67931 | 2848 | fprintf(fp, "SUCCEEDED %02d:%02d:%02d\n", h, m, s); |
8e25f19b MD |
2849 | } |
2850 | last_time = next_time - work->start_time; | |
2851 | s = last_time % 60; | |
2852 | m = last_time / 60 % 60; | |
2853 | h = last_time / 3600; | |
2854 | if (phaseid == PHASE_PACKAGE) { | |
2855 | fprintf(fp, "TOTAL TIME %02d:%02d:%02d\n", h, m, s); | |
2856 | } | |
6fd67931 | 2857 | fprintf(fp, "\n"); |
8e25f19b MD |
2858 | fclose(fp); |
2859 | skip: | |
2860 | ; | |
2861 | } | |
2862 | ||
2863 | } | |
2864 | ||
2865 | static void | |
2866 | phaseReapAll(void) | |
2867 | { | |
2868 | struct reaper_status rs; | |
2869 | int status; | |
2870 | ||
2871 | while (procctl(P_PID, getpid(), PROC_REAP_STATUS, &rs) == 0) { | |
2872 | if ((rs.flags & PROC_REAP_ACQUIRE) == 0) | |
2873 | break; | |
2874 | if (rs.pid_head < 0) | |
2875 | break; | |
2876 | if (kill(rs.pid_head, SIGKILL) == 0) { | |
2877 | while (waitpid(rs.pid_head, &status, 0) < 0) | |
2878 | ; | |
2879 | } | |
2880 | } | |
2881 | while (wait3(&status, 0, NULL) > 0) | |
2882 | ; | |
2883 | } | |
2884 | ||
2885 | static void | |
2886 | phaseTerminateSignal(int sig __unused) | |
2887 | { | |
fef2fc63 MD |
2888 | if (CopyFileFd >= 0) |
2889 | close(CopyFileFd); | |
1d6e00cd MD |
2890 | if (MasterPtyFd >= 0) |
2891 | close(MasterPtyFd); | |
8e25f19b MD |
2892 | if (SigPid > 1) |
2893 | kill(SigPid, SIGKILL); | |
2894 | phaseReapAll(); | |
2895 | if (SigWork) | |
2896 | DoWorkerUnmounts(SigWork); | |
2897 | exit(1); | |
2898 | } | |
2899 | ||
2900 | static | |
2901 | char * | |
3699ee09 | 2902 | buildskipreason(pkglink_t *parent, pkg_t *pkg) |
8e25f19b MD |
2903 | { |
2904 | pkglink_t *link; | |
2905 | pkg_t *scan; | |
2906 | char *reason = NULL; | |
3699ee09 | 2907 | char *ptr; |
8e25f19b MD |
2908 | size_t tot; |
2909 | size_t len; | |
3699ee09 | 2910 | pkglink_t stack; |
8e25f19b | 2911 | |
4ea2ee4d MD |
2912 | if ((pkg->flags & PKGF_NOBUILD_I) && pkg->ignore) |
2913 | asprintf(&reason, "%s ", pkg->ignore); | |
2914 | ||
8e25f19b MD |
2915 | tot = 0; |
2916 | PKGLIST_FOREACH(link, &pkg->idepon_list) { | |
63fcce5b MD |
2917 | #if 0 |
2918 | if (link->dep_type > DEP_TYPE_BUILD) | |
2919 | continue; | |
2920 | #endif | |
8e25f19b MD |
2921 | scan = link->pkg; |
2922 | if (scan == NULL) | |
2923 | continue; | |
2924 | if ((scan->flags & (PKGF_ERROR | PKGF_NOBUILD)) == 0) | |
2925 | continue; | |
3699ee09 MD |
2926 | if (scan->flags & PKGF_NOBUILD) { |
2927 | stack.pkg = scan; | |
2928 | stack.next = parent; | |
2929 | ptr = buildskipreason(&stack, scan); | |
2930 | len = strlen(scan->portdir) + strlen(ptr) + 8; | |
2931 | reason = realloc(reason, tot + len); | |
2932 | snprintf(reason + tot, len, "%s->%s", | |
2933 | scan->portdir, ptr); | |
2934 | free(ptr); | |
2935 | } else { | |
2936 | len = strlen(scan->portdir) + 8; | |
2937 | reason = realloc(reason, tot + len); | |
2938 | snprintf(reason + tot, len, "%s", scan->portdir); | |
2939 | } | |
2940 | ||
2941 | /* | |
2942 | * Don't try to print the entire graph | |
2943 | */ | |
2944 | if (parent) | |
2945 | break; | |
8e25f19b | 2946 | tot += strlen(reason + tot); |
3699ee09 MD |
2947 | reason[tot++] = ' '; |
2948 | reason[tot] = 0; | |
8e25f19b MD |
2949 | } |
2950 | return (reason); | |
2951 | } | |
2952 | ||
3dd48cfa MD |
2953 | /* |
2954 | * Count number of packages that would be skipped due to the | |
2955 | * specified package having failed. | |
2956 | * | |
2957 | * Call with mode 1 to count, and mode 0 to clear the | |
2958 | * cumulative rscan flag (used to de-duplicate the count). | |
2959 | * | |
2960 | * Must be serialized. | |
2961 | */ | |
2962 | static int | |
2963 | buildskipcount_dueto(pkg_t *pkg, int mode) | |
2964 | { | |
2965 | pkglink_t *link; | |
2966 | pkg_t *scan; | |
2967 | int total; | |
2968 | ||
2969 | total = 0; | |
2970 | PKGLIST_FOREACH(link, &pkg->deponi_list) { | |
2971 | scan = link->pkg; | |
2972 | if (scan == NULL || scan->rscan == mode) | |
2973 | continue; | |
2974 | scan->rscan = mode; | |
2975 | ++total; | |
2976 | total += buildskipcount_dueto(scan, mode); | |
2977 | } | |
2978 | return total; | |
2979 | } | |
2980 | ||
7880dcf7 MD |
2981 | /* |
2982 | * The master ptyfd is in non-blocking mode. Drain up to 1024 bytes | |
2983 | * and update wmsg->lines and *wdog_timep as appropriate. | |
2984 | * | |
2985 | * This function will poll, stalling up to 1 second. | |
2986 | */ | |
2987 | static int | |
2988 | mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg, time_t *wdog_timep) | |
2989 | { | |
2990 | struct pollfd pfd; | |
2991 | char buf[1024]; | |
2992 | ssize_t r; | |
2993 | ||
2994 | pfd.fd = ptyfd; | |
2995 | pfd.events = POLLIN; | |
2996 | pfd.revents = 0; | |
2997 | ||
2998 | poll(&pfd, 1, 1000); | |
2999 | if (pfd.revents) { | |
3000 | r = read(ptyfd, buf, sizeof(buf)); | |
3001 | if (r > 0) { | |
3002 | *wdog_timep = time(NULL); | |
3003 | if (r > 0 && fdlog >= 0) | |
3004 | write(fdlog, buf, r); | |
3005 | while (--r >= 0) { | |
3006 | if (buf[r] == '\n') | |
3007 | ++wmsg->lines; | |
3008 | } | |
3009 | return MPTY_DATA; | |
7880dcf7 MD |
3010 | } else if (r < 0) { |
3011 | if (errno != EINTR && errno != EAGAIN) | |
3012 | return MPTY_FAILED; | |
3013 | return MPTY_AGAIN; | |
3014 | } else if (r == 0) { | |
3015 | return MPTY_EOF; | |
3016 | } | |
3017 | } | |
3018 | return MPTY_AGAIN; | |
3019 | } | |
3020 | ||
8e25f19b MD |
3021 | /* |
3022 | * Copy a (package) file from (src) to (dst), use an intermediate file and | |
3023 | * rename to ensure that interruption does not leave us with a corrupt | |
3024 | * package file. | |
3025 | * | |
3026 | * This is called by the WORKER process. | |
3027 | * | |
3028 | * (dsynth management thread -> WORKER process -> sub-processes) | |
3029 | */ | |
3030 | #define COPYBLKSIZE 32768 | |
3031 | ||
8b485838 | 3032 | int |
8e25f19b MD |
3033 | copyfile(char *src, char *dst) |
3034 | { | |
3035 | char *tmp; | |
3036 | char *buf; | |
3037 | int fd1; | |
3038 | int fd2; | |
3039 | int error = 0; | |
fef2fc63 | 3040 | int mask; |
8e25f19b MD |
3041 | ssize_t r; |
3042 | ||
3043 | asprintf(&tmp, "%s.new", dst); | |
3044 | buf = malloc(COPYBLKSIZE); | |
3045 | ||
fef2fc63 | 3046 | mask = sigsetmask(sigmask(SIGTERM)|sigmask(SIGINT)|sigmask(SIGHUP)); |
8e25f19b MD |
3047 | fd1 = open(src, O_RDONLY|O_CLOEXEC); |
3048 | fd2 = open(tmp, O_RDWR|O_CREAT|O_TRUNC|O_CLOEXEC, 0644); | |
fef2fc63 MD |
3049 | CopyFileFd = fd1; |
3050 | sigsetmask(mask); | |
8e25f19b MD |
3051 | while ((r = read(fd1, buf, COPYBLKSIZE)) > 0) { |
3052 | if (write(fd2, buf, r) != r) | |
3053 | error = 1; | |
3054 | } | |
f7f25838 MD |
3055 | if (r < 0) |
3056 | error = 1; | |
fef2fc63 MD |
3057 | mask = sigsetmask(sigmask(SIGTERM)|sigmask(SIGINT)|sigmask(SIGHUP)); |
3058 | CopyFileFd = -1; | |
8e25f19b MD |
3059 | close(fd1); |
3060 | close(fd2); | |
fef2fc63 | 3061 | sigsetmask(mask); |
8e25f19b MD |
3062 | if (error) { |
3063 | remove(tmp); | |
3064 | } else { | |
3065 | if (rename(tmp, dst)) { | |
3066 | error = 1; | |
3067 | remove(tmp); | |
3068 | } | |
3069 | } | |
3070 | ||
ffc851f6 MD |
3071 | freestrp(&buf); |
3072 | freestrp(&tmp); | |
8e25f19b MD |
3073 | |
3074 | return error; | |
3075 | } | |
68dc2eea MD |
3076 | |
3077 | /* | |
3078 | * doHook() | |
3079 | * | |
3080 | * primary process (threaded) - run_start, run_end, pkg_ignored, pkg_skipped | |
3081 | * worker process (threaded) - pkg_sucess, pkg_failure | |
3082 | * | |
3083 | * If waitfor is non-zero this hook will be serialized. | |
3084 | */ | |
3085 | static void | |
3086 | doHook(pkg_t *pkg, const char *id, const char *path, int waitfor) | |
3087 | { | |
3088 | if (path == NULL) | |
3089 | return; | |
3090 | while (waitfor && getbulk() != NULL) | |
3091 | ; | |
3092 | if (pkg) | |
3093 | queuebulk(pkg->portdir, id, path, pkg->pkgfile); | |
3094 | else | |
3095 | queuebulk(NULL, id, path, NULL); | |
3096 | while (waitfor && getbulk() != NULL) | |
3097 | ; | |
3098 | } | |
3099 | ||
3100 | /* | |
3101 | * Execute hook (backend) | |
3102 | * | |
3103 | * s1 - portdir | |
3104 | * s2 - id | |
3105 | * s3 - script path | |
3106 | * s4 - pkgfile (if applicable) | |
3107 | */ | |
3108 | static void | |
3109 | childHookRun(bulk_t *bulk) | |
3110 | { | |
3111 | const char *cav[MAXCAC]; | |
3112 | buildenv_t benv[MAXCAC]; | |
3113 | char buf1[128]; | |
3114 | char buf2[128]; | |
3115 | char buf3[128]; | |
3116 | char buf4[128]; | |
3117 | FILE *fp; | |
3118 | char *ptr; | |
3119 | size_t len; | |
3120 | pid_t pid; | |
3121 | int cac; | |
3122 | int bi; | |
3123 | ||
3124 | cac = 0; | |
3125 | bi = 0; | |
3126 | bzero(benv, sizeof(benv)); | |
3127 | ||
3128 | cav[cac++] = bulk->s3; | |
3129 | ||
3130 | benv[bi].label = "PROFILE"; | |
3131 | benv[bi].data = Profile; | |
3132 | ++bi; | |
3133 | ||
3134 | benv[bi].label = "DIR_PACKAGES"; | |
3135 | benv[bi].data = PackagesPath; | |
3136 | ++bi; | |
3137 | ||
3138 | benv[bi].label = "DIR_REPOSITORY"; | |
3139 | benv[bi].data = RepositoryPath; | |
3140 | ++bi; | |
3141 | ||
3142 | benv[bi].label = "DIR_PORTS"; | |
3143 | benv[bi].data = DPortsPath; | |
3144 | ++bi; | |
3145 | ||
3146 | benv[bi].label = "DIR_OPTIONS"; | |
3147 | benv[bi].data = OptionsPath; | |
3148 | ++bi; | |
3149 | ||
3150 | benv[bi].label = "DIR_DISTFILES"; | |
3151 | benv[bi].data = DistFilesPath; | |
3152 | ++bi; | |
3153 | ||
3154 | benv[bi].label = "DIR_LOGS"; | |
3155 | benv[bi].data = LogsPath; | |
3156 | ++bi; | |
3157 | ||
3158 | benv[bi].label = "DIR_BUILDBASE"; | |
3159 | benv[bi].data = BuildBase; | |
3160 | ++bi; | |
3161 | ||
3162 | if (strcmp(bulk->s2, "hook_run_start") == 0) { | |
3163 | snprintf(buf1, sizeof(buf1), "%d", BuildTotal); | |
3164 | benv[bi].label = "PORTS_QUEUED"; | |
3165 | benv[bi].data = buf1; | |
3166 | ++bi; | |
3167 | } else if (strcmp(bulk->s2, "hook_run_end") == 0) { | |
3168 | snprintf(buf1, sizeof(buf1), "%d", BuildSuccessCount); | |
3169 | benv[bi].label = "PORTS_BUILT"; | |
3170 | benv[bi].data = buf1; | |
3171 | ++bi; | |
3172 | snprintf(buf2, sizeof(buf2), "%d", BuildFailCount); | |
3173 | benv[bi].label = "PORTS_FAILED"; | |
3174 | benv[bi].data = buf2; | |
3175 | ++bi; | |
3176 | snprintf(buf3, sizeof(buf3), "%d", BuildIgnoreCount); | |
3177 | benv[bi].label = "PORTS_IGNORED"; | |
3178 | benv[bi].data = buf3; | |
3179 | ++bi; | |
3180 | snprintf(buf4, sizeof(buf4), "%d", BuildSkipCount); | |
3181 | benv[bi].label = "PORTS_SKIPPED"; | |
3182 | benv[bi].data = buf4; | |
3183 | ++bi; | |
3184 | } else { | |
3185 | /* | |
3186 | * success, failure, ignored, skipped | |
3187 | */ | |
3188 | benv[bi].label = "RESULT"; | |
3189 | if (strcmp(bulk->s2, "hook_pkg_success") == 0) { | |
3190 | benv[bi].data = "success"; | |
3191 | } else if (strcmp(bulk->s2, "hook_pkg_failure") == 0) { | |
3192 | benv[bi].data = "failure"; | |
3193 | } else if (strcmp(bulk->s2, "hook_pkg_ignored") == 0) { | |
3194 | benv[bi].data = "ignored"; | |
3195 | } else if (strcmp(bulk->s2, "hook_pkg_skipped") == 0) { | |
3196 | benv[bi].data = "skipped"; | |
3197 | } else { | |
3198 | dfatal("Unknown hook id: %s", bulk->s2); | |
3199 | /* NOT REACHED */ | |
3200 | } | |
3201 | ++bi; | |
3202 | ||
3203 | /* | |
3204 | * For compatibility with synth: | |
3205 | * | |
3206 | * ORIGIN does not include any @flavor, thus it is suitable | |
3207 | * for finding the actual port directory/subdirectory. | |
3208 | * | |
3209 | * FLAVOR is set to ORIGIN if there is no flavor, otherwise | |
3210 | * it is set to only the flavor sans the '@'. | |
3211 | */ | |
3212 | if ((ptr = strchr(bulk->s1, '@')) != NULL) { | |
3213 | snprintf(buf1, sizeof(buf1), "%*.*s", | |
3214 | (int)(ptr - bulk->s1), | |
3215 | (int)(ptr - bulk->s1), | |
3216 | bulk->s1); | |
3217 | benv[bi].label = "ORIGIN"; | |
3218 | benv[bi].data = buf1; | |
3219 | ++bi; | |
3220 | benv[bi].label = "FLAVOR"; | |
3221 | benv[bi].data = ptr + 1; | |
3222 | ++bi; | |
3223 | } else { | |
3224 | benv[bi].label = "ORIGIN"; | |
3225 | benv[bi].data = bulk->s1; | |
3226 | ++bi; | |
3227 | benv[bi].label = "FLAVOR"; | |
3228 | benv[bi].data = bulk->s1; | |
3229 | ++bi; | |
3230 | } | |
3231 | benv[bi].label = "PKGNAME"; | |
3232 | benv[bi].data = bulk->s4; | |
3233 | ++bi; | |
3234 | } | |
3235 | ||
3236 | benv[bi].label = NULL; | |
3237 | benv[bi].data = NULL; | |
3238 | ||
f9d29536 | 3239 | fp = dexec_open(bulk->s1, cav, cac, &pid, benv, 0, 0); |
68dc2eea MD |
3240 | while ((ptr = fgetln(fp, &len)) != NULL) |
3241 | ; | |
3242 | ||
3243 | if (dexec_close(fp, pid)) { | |
3244 | dlog(DLOG_ALL, | |
3245 | "[XXX] %s SCRIPT %s (%s)\n", | |
3246 | bulk->s1, bulk->s2, bulk->s3); | |
3247 | } | |
3248 | } | |
32fc5bbf MD |
3249 | |
3250 | /* | |
3251 | * Adjusts dload[0] by adding in t_pw (processes waiting on page-fault). | |
3252 | * We don't want load reductions due to e.g. thrashing to cause dsynth | |
3253 | * to increase the dynamic limit because it thinks the load is low. | |
f5fc9cfa MD |
3254 | * |
3255 | * This has a desirable property. If the system pager cannot keep up | |
3256 | * with process demand t_pw will spike while loadavg will only drop | |
3257 | * slowly, resulting in a high adjusted load calculation that causes | |
3258 | * dsynth to quickly clamp-down the limit. If the condition alleviates, | |
3259 | * the limit will then rise slowly again, possibly even before existing | |
3260 | * jobs are retired to meet the clamp-down from the original spike. | |
32fc5bbf MD |
3261 | */ |
3262 | static void | |
3263 | adjloadavg(double *dload) | |
3264 | { | |
3265 | #if defined(__DragonFly__) | |
3266 | struct vmtotal total; | |
3267 | size_t size; | |
3268 | ||
3269 | size = sizeof(total); | |
3270 | if (sysctlbyname("vm.vmtotal", &total, &size, NULL, 0) == 0) { | |
3271 | dload[0] += (double)total.t_pw; | |
3272 | } | |
3273 | #else | |
3274 | dload[0] += 0.0; /* just avoid compiler 'unused' warnings */ | |
3275 | #endif | |
3276 | } | |
b6bd007b MD |
3277 | |
3278 | /* | |
8d6cd058 MD |
3279 | * The list of pkgs has already been flagged PKGF_PACKAGED if a pkg |
3280 | * file exists. Check if the ports directory contents for such packages | |
3281 | * has changed by comparing against a small DBM database that we maintain. | |
3282 | * | |
3283 | * Force-clear PKGF_PACKAGED if the ports directory content has changed. | |
3284 | * | |
3285 | * If no DBM database entry is present, update the entry and assume that | |
3286 | * the package does not need to be rebuilt (allows the .dbm file to be | |
3287 | * manually deleted without forcing a complete rebuild). | |
b6bd007b MD |
3288 | */ |
3289 | static | |
3290 | void | |
3291 | check_packaged(const char *dbmpath, pkg_t *pkgs) | |
3292 | { | |
3293 | pkg_t *scan; | |
3294 | datum key; | |
3295 | datum data; | |
3296 | char *buf; | |
3297 | ||
3298 | if (CheckDBM == NULL) { | |
3299 | dlog(DLOG_ABN, "[XXX] Unable to open/create dbm %s\n", dbmpath); | |
3300 | return; | |
3301 | } | |
3302 | for (scan = pkgs; scan; scan = scan->bnext) { | |
3303 | if ((scan->flags & PKGF_PACKAGED) == 0) | |
3304 | continue; | |
3305 | key.dptr = scan->portdir; | |
3306 | key.dsize = strlen(scan->portdir); | |
3307 | data = dbm_fetch(CheckDBM, key); | |
3308 | if (data.dptr && data.dsize == sizeof(uint32_t) && | |
3309 | *(uint32_t *)data.dptr != scan->crc32) { | |
3310 | scan->flags &= ~PKGF_PACKAGED; | |
3311 | asprintf(&buf, "%s/%s", RepositoryPath, scan->pkgfile); | |
3312 | if (OverridePkgDeleteOpt >= 2) { | |
3313 | scan->flags |= PKGF_PACKAGED; | |
3314 | dlog(DLOG_ALL, | |
3315 | "[XXX] %s DELETE-PACKAGE %s " | |
3316 | "(port content changed CRC %08x/%08x " | |
3317 | "OVERRIDE, NOT DELETED)\n", | |
3318 | scan->portdir, buf, | |
3319 | *(uint32_t *)data.dptr, scan->crc32); | |
3320 | } else if (remove(buf) < 0) { | |
3321 | dlog(DLOG_ALL, | |
3322 | "[XXX] %s DELETE-PACKAGE %s (failed)\n", | |
3323 | scan->portdir, buf); | |
3324 | } else { | |
3325 | dlog(DLOG_ALL, | |
3326 | "[XXX] %s DELETE-PACKAGE %s " | |
3327 | "(port content changed CRC %08x/%08x)\n", | |
3328 | scan->portdir, buf, | |
3329 | *(uint32_t *)data.dptr, scan->crc32); | |
3330 | } | |
3331 | freestrp(&buf); | |
3332 | } else if (data.dptr == NULL) { | |
3333 | data.dptr = &scan->crc32; | |
3334 | data.dsize = sizeof(scan->crc32); | |
3335 | dbm_store(CheckDBM, key, data, DBM_REPLACE); | |
3336 | } | |
3337 | } | |
3338 | } |