2 * Copyright (c) 2019 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * This code uses concepts and configuration based on 'synth', by
8 * John R. Marino <draco@marino.st>, which was written in ada.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in
18 * the documentation and/or other materials provided with the
20 * 3. Neither the name of The DragonFly Project nor the names of its
21 * contributors may be used to endorse or promote products derived
22 * from this software without specific, prior written permission.
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 #include <sys/types.h>
41 #include <sys/sysctl.h>
42 #include <sys/socket.h>
43 #include <sys/mount.h>
44 #include <sys/procctl.h>
62 * More esoteric headers
64 #include <libutil.h> /* forkpty() */
65 #include <arpa/inet.h> /* ntohl() */
66 #include <elf.h> /* try to get elf info */
70 #define DSYNTH_VERSION "1.01"
71 #define MAXWORKERS 1024
72 #define MAXLOGLINES 1024
73 #define MAXJOBS 8192 /* just used for -j sanity */
74 #define MAXBULK MAXWORKERS
76 #define MAKE_BINARY "/usr/bin/make"
77 #define PKG_BINARY "/usr/local/sbin/pkg"
78 #define MOUNT_BINARY "/sbin/mount"
79 #define MOUNT_NULLFS_BINARY "/sbin/mount_null"
80 #define MOUNT_TMPFS_BINARY "/sbin/mount_tmpfs"
81 #define MOUNT_DEVFS_BINARY "/sbin/mount_devfs"
82 #define MOUNT_PROCFS_BINARY "/sbin/mount_procfs"
83 #define UMOUNT_BINARY "/sbin/umount"
85 #define STATS_FILE "monitor.dat" /* under LogsPath */
86 #define STATS_LOCKFILE "monitor.lk" /* under LogsPath */
88 #define ONEGB (1024L * 1024 * 1024)
89 #define DISABLED_STR "disabled"
92 * This can be ".tar", ".tgz", ".txz", or ".tbz".
94 * .tar - very fast but you'll need 1TB+ of storage just for the package files.
95 * .txz - very compact but decompression speed is horrible.
96 * .tgz - reasonable compression, extremely fast decompression. Roughly
97 * 1.1x to 2.0x the size of a .txz, but decompresses 10x faster.
98 * .tbz - worse than .tgz generally
100 * NOTE: Decompression speed does effect bulk builds since each slot has
101 * to install pre-reqs before building any particular package. Set
102 * the default to .txz to remain close to synth's default.
104 #define USE_PKG_SUFX ".txz"
109 typedef struct pkglink {
110 struct pkglink *next;
111 struct pkglink *prev;
116 #define DEP_TYPE_FETCH 1
117 #define DEP_TYPE_EXT 2
118 #define DEP_TYPE_PATCH 3
119 #define DEP_TYPE_BUILD 4
120 #define DEP_TYPE_LIB 5
121 #define DEP_TYPE_RUN 6
124 * Describes a [flavored] package
127 struct pkg *build_next; /* topology inversion build list */
128 struct pkg *bnext; /* linked list from bulk return */
129 struct pkg *hnext1; /* hash based on portdir */
130 struct pkg *hnext2; /* hash based on pkgfile */
131 pkglink_t idepon_list; /* I need these pkgs */
132 pkglink_t deponi_list; /* pkgs which depend on me */
133 char *portdir; /* origin name e.g. www/chromium[@flavor] */
134 char *logfile; /* relative logfile path */
135 char *version; /* PKGVERSION - e.g. 3.5.0_1 */
136 char *pkgfile; /* PKGFILE - e.g. flav-blah-3.5.0_1.txz */
137 char *distfiles; /* DISTFILES - e.g. blah-68.0.source.tar.xz */
138 char *distsubdir; /* DIST_SUBDIR- e.g. cabal */
139 char *ignore; /* IGNORE (also covers BROKEN) */
140 char *fetch_deps; /* FETCH_DEPENDS */
141 char *ext_deps; /* EXTRACT_DEPENDS */
142 char *patch_deps; /* PATCH_DEPENDS */
143 char *build_deps; /* BUILD_DEPENDS */
144 char *lib_deps; /* LIB_DEPENDS */
145 char *run_deps; /* RUN_DEPENDS */
146 char *pos_options; /* SELECTED_OPTIONS */
147 char *neg_options; /* DESELECTED_OPTIONS */
148 char *flavors; /* FLAVORS - e.g. py36 py27 */
149 char *uses; /* USES (metaport test) */
150 int make_jobs_number; /* MAKE_JOBS_NUMBER */
151 int use_linux; /* USE_LINUX */
152 int idep_count; /* count recursive idepon build deps */
153 int depi_count; /* count recursive deponi build deps */
154 int depi_depth; /* tree depth who depends on me */
155 int dsynth_install_flg; /* locked with WorkerMutex */
157 int rscan; /* recursive scan flag (serialized use) */
158 size_t pkgfile_size; /* size of pkgfile */
161 #define PKGF_PACKAGED 0x00000001 /* has a repo package */
162 #define PKGF_DUMMY 0x00000002 /* generic root for flavors */
163 #define PKGF_NOTFOUND 0x00000004 /* dport not found */
164 #define PKGF_CORRUPT 0x00000008 /* dport corrupt */
165 #define PKGF_PLACEHOLD 0x00000010 /* pre-entered */
166 #define PKGF_BUILDLIST 0x00000020 /* on build_list */
167 #define PKGF_BUILDLOOP 0x00000040 /* traversal loop test */
168 #define PKGF_BUILDTRAV 0x00000080 /* traversal optimization */
169 #define PKGF_NOBUILD_D 0x00000100 /* can't build - dependency problem */
170 #define PKGF_NOBUILD_S 0x00000200 /* can't build - skipped */
171 #define PKGF_NOBUILD_F 0x00000400 /* can't build - failed */
172 #define PKGF_NOBUILD_I 0x00000800 /* can't build - ignored or broken */
173 #define PKGF_SUCCESS 0x00001000 /* build complete */
174 #define PKGF_FAILURE 0x00002000 /* build complete */
175 #define PKGF_RUNNING 0x00004000 /* build complete */
176 #define PKGF_PKGPKG 0x00008000 /* pkg/pkg-static special */
177 #define PKGF_NOTREADY 0x00010000 /* build_find_leaves() only */
178 #define PKGF_MANUALSEL 0x00020000 /* manually specified */
179 #define PKGF_META 0x00040000 /* USES contains 'metaport' */
180 #define PKGF_DEBUGSTOP 0x00080000 /* freeze slot on completion */
182 #define PKGF_ERROR (PKGF_PLACEHOLD | PKGF_CORRUPT | PKGF_NOTFOUND | \
184 #define PKGF_NOBUILD (PKGF_NOBUILD_D | PKGF_NOBUILD_S | PKGF_NOBUILD_F | \
187 #define PKGLIST_EMPTY(pkglink) ((pkglink)->next == (pkglink))
188 #define PKGLIST_FOREACH(var, head) \
189 for (var = (head)->next; var != (head); var = (var)->next)
191 typedef struct bulk {
196 enum { UNLISTED, ONSUBMIT, ONRUN, ISRUNNING, ONRESPONSE } state;
205 pkg_t *list; /* pkgs linked by bnext */
209 * Worker state (up to MAXWORKERS). Each worker operates within a
210 * chroot or jail. A system mirror is setup and the template
215 * /sbin - nullfs (ro)
217 * /libexec - nullfs (ro)
218 * /usr/bin - nullfs (ro)
219 * /usr/include - nullfs (ro)
220 * /usr/lib - nullfs (ro)
221 * /usr/libdata - nullfs (ro)
222 * /usr/libexec - nullfs (ro)
223 * /usr/sbin - nullfs (ro)
224 * /usr/share - nullfs (ro)
225 * /xports - nullfs (ro)
226 * /options - nullfs (ro)
227 * /packages - nullfs (ro)
228 * /distfiles - nullfs (ro)
229 * construction - tmpfs
231 * /boot - nullfs (ro)
232 * /boot/modules.local - tmpfs
233 * /usr/games - nullfs (ro)
234 * /usr/src - nullfs (ro)
237 enum worker_state { WORKER_NONE, WORKER_IDLE, WORKER_PENDING,
238 WORKER_RUNNING, WORKER_DONE, WORKER_FAILED,
239 WORKER_FROZEN, WORKER_EXITING };
240 typedef enum worker_state worker_state_t;
242 enum worker_phase { PHASE_PENDING,
249 PHASE_EXTRACT_DEPENDS,
267 typedef enum worker_phase worker_phase_t;
270 * Watchdog timeouts, in minutes, baseline, scales up with load/ncpus but
271 * does not scale down.
278 #define WDOG6 (60 + 30)
279 #define WDOG7 (60 * 2)
280 #define WDOG8 (60 * 2 + 30)
281 #define WDOG9 (60 * 3)
283 typedef struct worker {
284 int index; /* worker number 0..N-1 */
286 int accum_error; /* cumulative error */
287 int mount_error; /* mount and unmount error */
288 int terminate : 1; /* request sub-thread to terminate */
289 char *basedir; /* base directory including id */
291 pthread_t td; /* pthread */
292 pthread_cond_t cond; /* interlock cond (w/ WorkerMutex) */
294 worker_state_t state; /* general worker state */
295 worker_phase_t phase; /* phase control in childBuilderThread */
300 int fds[2]; /* forked environment process */
302 size_t pkg_dep_size; /* pkg dependency size(s) */
305 #define WORKERF_STATUS_UPDATE 0x0001 /* display update */
306 #define WORKERF_SUCCESS 0x0002 /* completion flag */
307 #define WORKERF_FAILURE 0x0004 /* completion flag */
308 #define WORKERF_FREEZE 0x0008 /* freeze the worker */
310 #define MOUNT_TYPE_MASK 0x000F
311 #define MOUNT_TYPE_TMPFS 0x0001
312 #define MOUNT_TYPE_NULLFS 0x0002
313 #define MOUNT_TYPE_DEVFS 0x0003
314 #define MOUNT_TYPE_PROCFS 0x0004
315 #define MOUNT_TYPE_RW 0x0010
316 #define MOUNT_TYPE_BIG 0x0020
317 #define MOUNT_TYPE_TMP 0x0040
319 #define NULLFS_RO (MOUNT_TYPE_NULLFS)
320 #define NULLFS_RW (MOUNT_TYPE_NULLFS | MOUNT_TYPE_RW)
321 #define PROCFS_RO (MOUNT_TYPE_PROCFS)
322 #define TMPFS_RW (MOUNT_TYPE_TMPFS | MOUNT_TYPE_RW)
323 #define TMPFS_RW_BIG (MOUNT_TYPE_TMPFS | MOUNT_TYPE_RW | \
325 #define DEVFS_RW (MOUNT_TYPE_DEVFS | MOUNT_TYPE_RW)
328 * IPC messages between the worker support thread and the worker process.
330 typedef struct wmsg {
335 worker_phase_t phase;
338 #define WMSG_CMD_STATUS_UPDATE 0x0001
339 #define WMSG_CMD_SUCCESS 0x0002
340 #define WMSG_CMD_FAILURE 0x0003
341 #define WMSG_CMD_INSTALL_PKGS 0x0004
342 #define WMSG_RES_INSTALL_PKGS 0x0005
343 #define WMSG_CMD_FREEZEWORKER 0x0006
346 * Make variables and build environment
348 typedef struct buildenv {
349 struct buildenv *next;
352 char *a1; /* allocations */
353 char *a2; /* allocations */
358 * Operating systems recognized by dsynth
361 OS_UNKNOWN, OS_DRAGONFLY, OS_FREEBSD, OS_NETBSD, OS_LINUX
364 typedef enum os_id os_id_t;
369 #define DLOG_ALL 0 /* Usually stdout when curses disabled */
370 #define DLOG_SUCC 1 /* success_list.log */
371 #define DLOG_FAIL 2 /* failure_list.log */
372 #define DLOG_IGN 3 /* ignored_list.log */
373 #define DLOG_SKIP 4 /* skipped_list.log */
374 #define DLOG_ABN 5 /* abnormal_command_output */
375 #define DLOG_OBS 6 /* obsolete_packages.log */
376 #define DLOG_COUNT 7 /* total number of DLOGs */
377 #define DLOG_MASK 0x0FF
379 #define DLOG_FILTER 0x100 /* Filter out of stdout in non-curses mode */
380 #define DLOG_RED 0x200 /* Print in color */
381 #define DLOG_GRN 0x400 /* Print in color */
382 #define DLOG_STDOUT 0x800 /* And stdout */
384 #define dassert(exp, fmt, ...) \
385 if (!(exp)) dpanic(fmt, ## __VA_ARGS__)
387 #define ddassert(exp) \
388 dassert((exp), "\"%s\" line %d", __FILE__, __LINE__)
390 #define dassert_errno(exp, fmt, ...) \
391 if (!(exp)) dpanic_errno(fmt, ## __VA_ARGS__)
393 #define dlog(which, fmt, ...) \
394 _dlog(which, fmt, ## __VA_ARGS__)
396 #define dlog_tsnl(which, fmt, ...) \
397 _dlog(which, fmt, ## __VA_ARGS__)
399 #define dfatal(fmt, ...) \
400 _dfatal(__FILE__, __LINE__, __func__, 0, fmt, ## __VA_ARGS__)
402 #define dpanic(fmt, ...) \
403 _dfatal(__FILE__, __LINE__, __func__, 2, fmt, ## __VA_ARGS__)
405 #define dfatal_errno(fmt, ...) \
406 _dfatal(__FILE__, __LINE__, __func__, 1, fmt, ## __VA_ARGS__)
408 #define dpanic_errno(fmt, ...) \
409 _dfatal(__FILE__, __LINE__, __func__, 3, fmt, ## __VA_ARGS__)
411 #define ddprintf(tab, fmt, ...) \
412 do { if (DebugOpt >= 2) _ddprintf(tab, fmt, ## __VA_ARGS__); } while(0)
415 * addbuildenv() types
417 #define BENV_ENVIRONMENT 1
418 #define BENV_MAKECONF 2
419 #define BENV_CMDMASK 0x000F
421 #define BENV_PKGLIST 0x0010
424 * WORKER process flags
426 #define WORKER_PROC_DEBUGSTOP 0x0001
427 #define WORKER_PROC_DEVELOPER 0x0002
428 #define WORKER_PROC_CHECK_PLIST 0x0004
433 #define DOSTRING(label) #label
434 #define SCRIPTPATH(x) DOSTRING(x)
438 * RunStats satellite modules
440 typedef struct topinfo {
458 typedef struct runstats {
459 struct runstats *next;
463 void (*update)(worker_t *work, const char *portdir);
464 void (*updateTop)(topinfo_t *info);
465 void (*updateLogs)(void);
469 typedef struct monitorlog {
475 int buf_discard_mode;
479 extern runstats_t NCursesRunStats;
480 extern runstats_t MonitorRunStats;
481 extern runstats_t HtmlRunStats;
483 extern int BuildCount;
484 extern int BuildTotal;
485 extern int BuildFailCount;
486 extern int BuildSkipCount;
487 extern int BuildIgnoreCount;
488 extern int BuildSuccessCount;
489 extern int DynamicMaxWorkers;
491 extern buildenv_t *BuildEnv;
492 extern int WorkerProcFlags;
495 extern int SlowStartOpt;
497 extern int NullStdinOpt;
498 extern int UseCCache;
499 extern int UseUsrSrc;
503 extern long PkgDepMemoryTarget;
505 extern int MaxWorkers;
507 extern int UseTmpfsWork;
508 extern int UseTmpfsBase;
509 extern int UseNCurses;
510 extern int LeveragePrebuilt;
511 extern char *DSynthExecPath;
512 extern char *ProfileOverrideOpt;
514 extern const char *OperatingSystemName;
515 extern const char *ArchitectureName;
516 extern const char *MachineName;
517 extern const char *ReleaseName;
518 extern const char *VersionName;
519 extern const char *VersionOnlyName;
520 extern const char *VersionFromParamHeader;
522 extern const char *ConfigBase1;
523 extern const char *ConfigBase2;
524 extern const char *ConfigBase;
525 extern const char *DPortsPath;
526 extern const char *CCachePath;
527 extern const char *PackagesPath;
528 extern const char *RepositoryPath;
529 extern const char *OptionsPath;
530 extern const char *DistFilesPath;
531 extern const char *BuildBase;
532 extern const char *LogsPath;
533 extern const char *SystemPath;
534 extern const char *UsePkgSufx;
535 extern const char *Profile;
536 extern char *StatsBase;
537 extern char *StatsFilePath;
538 extern char *StatsLockPath;
540 extern int UsingHooks;
541 extern const char *HookRunStart;
542 extern const char *HookRunEnd;
543 extern const char *HookPkgSuccess;
544 extern const char *HookPkgFailure;
545 extern const char *HookPkgIgnored;
546 extern const char *HookPkgSkipped;
548 void _dfatal(const char *file, int line, const char *func, int do_errno,
549 const char *fmt, ...);
550 void _ddprintf(int tab, const char *fmt, ...);
551 void _dlog(int which, const char *fmt, ...);
552 char *strdup_or_null(char *str);
553 void dlogreset(void);
555 void addbuildenv(const char *label, const char *data, int type);
556 void delbuildenv(const char *label);
557 int readlogline(monitorlog_t *log, char **bufp);
559 void initbulk(void (*func)(bulk_t *bulk), int jobs);
560 void queuebulk(const char *s1, const char *s2, const char *s3,
562 bulk_t *getbulk(void);
564 void freebulk(bulk_t *bulk);
565 void freestrp(char **strp);
566 void dupstrp(char **strp);
567 int askyn(const char *ctl, ...);
568 double getswappct(int *noswapp);
569 FILE *dexec_open(const char **cav, int cac, pid_t *pidp, buildenv_t *xenv,
570 int with_env, int with_mvars);
571 int dexec_close(FILE *fp, pid_t pid);
572 const char *getphasestr(worker_phase_t phase);
574 void ParseConfiguration(int isworker);
575 pkg_t *ParsePackageList(int ac, char **av, int debugstop);
576 void FreePackageList(pkg_t *pkgs);
577 pkg_t *GetLocalPackageList(void);
578 pkg_t *GetFullPackageList(void);
579 pkg_t *GetPkgPkg(pkg_t *list);
581 void DoConfigure(void);
582 void DoStatus(pkg_t *pkgs);
583 void DoBuild(pkg_t *pkgs);
584 void DoInitBuild(int slot_override);
585 void DoCleanBuild(int resetlogs);
586 void OptimizeEnv(void);
587 void WorkerProcess(int ac, char **av);
589 int DoCreateTemplate(int force);
590 void DoDestroyTemplate(void);
591 void DoWorkerMounts(worker_t *work);
592 void DoWorkerUnmounts(worker_t *work);
593 void DoRebuildRepo(int ask);
594 void DoUpgradePkgs(pkg_t *pkgs, int ask);
595 void RemovePackages(pkg_t *pkgs);
596 void PurgeDistfiles(pkg_t *pkgs);
598 void RunStatsInit(void);
599 void RunStatsDone(void);
600 void RunStatsReset(void);
601 void RunStatsUpdate(worker_t *work, const char *portdir);
602 void RunStatsUpdateTop(void);
603 void RunStatsUpdateLogs(void);
604 void RunStatsSync(void);
606 int ipcreadmsg(int fd, wmsg_t *msg);
607 int ipcwritemsg(int fd, wmsg_t *msg);
608 extern void MonitorDirective(const char *datfile, const char *lkfile);