From d2e9c9d8664f753a0d599eceed1dd98ffa7ef479 Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Tue, 29 Nov 2011 13:38:34 -0800 Subject: [PATCH] build - Significantly improve parallel buildworld times * Change the serialized command sequence used for bootstrap tools, btools, and ctools into dependencies so buildworld can build them concurrently. * Utilize the new SUBDIR concurrent build feature by specifying a SUBDIR_ORDERED make variable (usually empty) in various high-level Makefile's. --- Makefile.inc1 | 79 ++++++++++++++++++++++++++++---------------- bin/Makefile | 4 +++ games/Makefile | 4 +++ gnu/lib/Makefile | 4 +++ gnu/usr.bin/Makefile | 4 +++ libexec/Makefile | 4 +++ sbin/Makefile | 4 +++ secure/lib/Makefile | 4 +++ usr.bin/Makefile | 5 +++ usr.sbin/Makefile | 4 +++ 10 files changed, 87 insertions(+), 29 deletions(-) diff --git a/Makefile.inc1 b/Makefile.inc1 index 0b6e7d1277..136ed906e9 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -754,27 +754,38 @@ installmost: _strfile= games/fortune/strfile .endif -bootstrap-tools: +BSTRAPDIRS= ${_strfile} \ + usr.bin/patch \ + bin/chmod bin/cp bin/cpdup bin/dd bin/mkdir bin/rm bin/echo \ + bin/test bin/cat bin/ln bin/mv bin/csh bin/expr bin/sh \ + bin/hostname bin/kill \ + usr.bin/yacc usr.bin/colldef usr.bin/uudecode usr.bin/xinstall \ + usr.bin/m4 usr.bin/rpcgen usr.bin/make usr.bin/awk usr.bin/stat \ + usr.bin/find usr.bin/lex usr.bin/sed usr.bin/uname usr.bin/touch \ + usr.bin/mkdep usr.bin/mktemp usr.bin/lorder usr.bin/file2c \ + usr.bin/tsort usr.bin/tr usr.bin/join usr.bin/wc usr.bin/basename \ + usr.bin/gencat usr.bin/chflags usr.bin/expand usr.bin/paste \ + usr.bin/mklocale usr.bin/uuencode usr.bin/compile_et usr.bin/hexdump \ + usr.bin/cap_mkdb usr.bin/true usr.bin/false \ + usr.bin/cmp usr.bin/xargs usr.bin/id usr.bin/env usr.bin/dirname \ + usr.bin/tail usr.bin/unifdef \ + usr.sbin/chown usr.sbin/mtree usr.sbin/config \ + usr.sbin/btxld usr.sbin/zic usr.sbin/makewhatis \ + gnu/usr.bin/texinfo gnu/usr.bin/grep usr.bin/sort \ + usr.bin/gzip usr.bin/bzip2 usr.bin/mkcsmapper usr.bin/mkesdb + +bootstrap-tools: bootstrap-tools-before bootstrap-tools-targets + touch ${BTOOLSDEST}/.bootstrap_done + +bootstrap-tools-before: ${LN} -fs /bin/date ${BTOOLSDEST}/bin/date -.for _tool in ${_strfile} \ - usr.bin/patch \ - bin/chmod bin/cp bin/cpdup bin/dd bin/mkdir bin/rm bin/echo bin/test \ - bin/cat bin/ln bin/mv bin/csh bin/expr bin/sh \ - bin/hostname bin/kill \ - usr.bin/yacc usr.bin/colldef usr.bin/uudecode usr.bin/xinstall \ - usr.bin/m4 usr.bin/rpcgen usr.bin/make usr.bin/awk usr.bin/stat \ - usr.bin/find usr.bin/lex usr.bin/sed usr.bin/uname usr.bin/touch \ - usr.bin/mkdep usr.bin/mktemp usr.bin/lorder usr.bin/file2c \ - usr.bin/tsort usr.bin/tr usr.bin/join usr.bin/wc usr.bin/basename \ - usr.bin/gencat usr.bin/chflags usr.bin/expand usr.bin/paste \ - usr.bin/mklocale usr.bin/uuencode usr.bin/compile_et usr.bin/hexdump \ - usr.bin/cap_mkdb usr.bin/true usr.bin/false \ - usr.bin/cmp usr.bin/xargs usr.bin/id usr.bin/env usr.bin/dirname \ - usr.bin/tail usr.bin/unifdef \ - usr.sbin/chown usr.sbin/mtree usr.sbin/config \ - usr.sbin/btxld usr.sbin/zic usr.sbin/makewhatis \ - gnu/usr.bin/texinfo gnu/usr.bin/grep usr.bin/sort \ - usr.bin/gzip usr.bin/bzip2 usr.bin/mkcsmapper usr.bin/mkesdb + +bootstrap-tools-targets: ${BSTRAPDIRS:S/^/bstrap-/} + +.ORDER: bootstrap-tools-before bootstrap-tools-targets + +.for _tool in ${BSTRAPDIRS} +bstrap-${_tool}! ${ECHODIR} "===> ${_tool} (bootstrap-tools)"; \ cd ${.CURDIR}/${_tool}; \ ${MAKE} DIRPRFX=${_tool}/ obj; \ @@ -782,7 +793,6 @@ bootstrap-tools: ${MAKE} DIRPRFX=${_tool}/ all; \ ${MAKE} DIRPRFX=${_tool}/ DESTDIR=${BTOOLSDEST} install .endfor - touch ${BTOOLSDEST}/.bootstrap_done # build-tools: Build special purpose build tools. # @@ -802,8 +812,15 @@ _gcc44_tools= gnu/usr.bin/cc44/cc_prep gnu/usr.bin/cc44/cc_tools _custom_cross= libexec/customcc _binutils= gnu/usr.bin/${WORLD_BINUTILSVER} -build-tools: -.for _tool in ${_gcc41_tools} ${_gcc44_tools} ${_libkrb5} ${_share} +BTOOLSDIRS= ${_gcc41_tools} ${_gcc44_tools} ${_libkrb5} ${_share} + +build-tools: build-tools-targets + touch ${BTOOLSDEST}/.build_done + +build-tools-targets: ${BTOOLSDIRS:S/^/btools-/} + +.for _tool in ${BTOOLSDIRS} +btools-${_tool}! ${ECHODIR} "===> ${_tool} (build-tools)"; \ cd ${.CURDIR}/${_tool}; \ ${MAKE} DIRPRFX=${_tool}/ obj; \ @@ -811,7 +828,6 @@ build-tools: ${MAKE} DIRPRFX=${_tool}/ all; \ ${MAKE} DIRPRFX=${_tool}/ DESTDIR=${BTOOLSDEST} install .endfor - touch ${BTOOLSDEST}/.build_done # # cross-tools: Build cross-building tools @@ -820,10 +836,16 @@ build-tools: _btxld= usr.sbin/btxld .endif -cross-tools: -.for _tool in ${_btxld} ${_binutils} \ - usr.bin/objformat \ - ${_gcc41_cross} ${_gcc44_cross} ${_custom_cross} +CTOOLSDIRS= ${_btxld} ${_binutils} usr.bin/objformat \ + ${_gcc41_cross} ${_gcc44_cross} ${_custom_cross} + +cross-tools: cross-tools-targets + touch ${CTOOLSDEST}/.cross_done + +cross-tools-targets: ${CTOOLSDIRS:S/^/ctools-/} + +.for _tool in ${CTOOLSDIRS} +ctools-${_tool}! ${ECHODIR} "===> ${_tool} (cross-tools)"; \ cd ${.CURDIR}/${_tool}; \ ${MAKE} DIRPRFX=${_tool}/ obj; \ @@ -831,7 +853,6 @@ cross-tools: ${MAKE} DIRPRFX=${_tool}/ all; \ ${MAKE} DIRPRFX=${_tool}/ DESTDIR=${CTOOLSDEST} install .endfor - touch ${CTOOLSDEST}/.cross_done # # hierarchy - ensure that all the needed directories are present diff --git a/bin/Makefile b/bin/Makefile index 93efd523e8..8befa98e95 100644 --- a/bin/Makefile +++ b/bin/Makefile @@ -40,4 +40,8 @@ SUBDIR= cat \ SUBDIR+=rmail .endif +# maximum parallelism +# +SUBDIR_ORDERED= + .include diff --git a/games/Makefile b/games/Makefile index a07b6ae097..017d088dd8 100644 --- a/games/Makefile +++ b/games/Makefile @@ -44,4 +44,8 @@ SUBDIR= adventure \ worms \ wump +# maximum parallelism +# +SUBDIR_ORDERED= + .include diff --git a/gnu/lib/Makefile b/gnu/lib/Makefile index 2ff12bdd41..fba8be19bf 100644 --- a/gnu/lib/Makefile +++ b/gnu/lib/Makefile @@ -6,4 +6,8 @@ SUBDIR= gcc44 libcryptsetup libdevmapper libdialog libluks liblvm SUBDIR+= gcc41 .endif +# maximum parallelism +# +SUBDIR_ORDERED= + .include diff --git a/gnu/usr.bin/Makefile b/gnu/usr.bin/Makefile index b79a7ceaee..db6f6c5a3a 100644 --- a/gnu/usr.bin/Makefile +++ b/gnu/usr.bin/Makefile @@ -21,4 +21,8 @@ SUBDIR+= cc41 SUBDIR+= cc44 .ORDER: binutils222 cc44 +# maximum parallelism +# +SUBDIR_ORDERED= + .include diff --git a/libexec/Makefile b/libexec/Makefile index 035232761a..58cfa948ea 100644 --- a/libexec/Makefile +++ b/libexec/Makefile @@ -35,4 +35,8 @@ SUBDIR= atrun \ SUBDIR+=mail.local smrsh .endif +# maximum parallelism +# +SUBDIR_ORDERED= + .include diff --git a/sbin/Makefile b/sbin/Makefile index a5d58ad0ee..846ac44f73 100644 --- a/sbin/Makefile +++ b/sbin/Makefile @@ -115,4 +115,8 @@ SUBDIR+= ${MACHINE} SUBDIR+= ${MACHINE_ARCH} .endif +# maximum parallelism +# +SUBDIR_ORDERED= + .include diff --git a/secure/lib/Makefile b/secure/lib/Makefile index 8735d2031a..e95425fe09 100644 --- a/secure/lib/Makefile +++ b/secure/lib/Makefile @@ -9,4 +9,8 @@ SUBDIR+=libssh .endif .endif +# maximum parallelism +# +SUBDIR_ORDERED= + .include diff --git a/usr.bin/Makefile b/usr.bin/Makefile index a63a52f312..35bebcdec8 100644 --- a/usr.bin/Makefile +++ b/usr.bin/Makefile @@ -7,6 +7,11 @@ # XXX Use GNU versions: apropos diff grep ld man ptx uucp whatis # Moved to secure: bdes # + +# No ordering dependencies, everything can run in parallel +# +SUBDIR_ORDERED= + SUBDIR= alias \ apply \ asa \ diff --git a/usr.sbin/Makefile b/usr.sbin/Makefile index d3680dd8ed..9ecfdf7b7d 100644 --- a/usr.sbin/Makefile +++ b/usr.sbin/Makefile @@ -188,4 +188,8 @@ SUBDIR+=boot0cfg SUBDIR+=installer .endif +# maximum parallelism +# +SUBDIR_ORDERED= + .include -- 2.41.0