build - Significantly improve parallel buildworld times

* Change the serialized command sequence used for bootstrap tools,
  btools, and ctools into dependencies so buildworld can build them
  concurrently.

* Utilize the new SUBDIR concurrent build feature by specifying
  a SUBDIR_ORDERED make variable (usually empty) in various high-level
  Makefile's.

diff --git a/Makefile.inc1 b/Makefile.inc1
index 0b6e7d1..136ed90 100644 (file)
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -754,27 +754,38 @@ installmost:
 _strfile=      games/fortune/strfile
 .endif
 
-bootstrap-tools:
+BSTRAPDIRS= ${_strfile} \
+       usr.bin/patch \
+       bin/chmod bin/cp bin/cpdup bin/dd bin/mkdir bin/rm bin/echo \
+       bin/test bin/cat bin/ln bin/mv bin/csh bin/expr bin/sh \
+       bin/hostname bin/kill \
+       usr.bin/yacc usr.bin/colldef usr.bin/uudecode usr.bin/xinstall \
+       usr.bin/m4 usr.bin/rpcgen usr.bin/make usr.bin/awk usr.bin/stat \
+       usr.bin/find usr.bin/lex usr.bin/sed usr.bin/uname usr.bin/touch \
+       usr.bin/mkdep usr.bin/mktemp usr.bin/lorder usr.bin/file2c \
+       usr.bin/tsort usr.bin/tr usr.bin/join usr.bin/wc usr.bin/basename \
+       usr.bin/gencat usr.bin/chflags usr.bin/expand usr.bin/paste \
+       usr.bin/mklocale usr.bin/uuencode usr.bin/compile_et usr.bin/hexdump \
+       usr.bin/cap_mkdb usr.bin/true usr.bin/false \
+       usr.bin/cmp usr.bin/xargs usr.bin/id usr.bin/env usr.bin/dirname \
+       usr.bin/tail usr.bin/unifdef \
+       usr.sbin/chown usr.sbin/mtree usr.sbin/config \
+       usr.sbin/btxld usr.sbin/zic usr.sbin/makewhatis \
+       gnu/usr.bin/texinfo gnu/usr.bin/grep usr.bin/sort \
+       usr.bin/gzip usr.bin/bzip2 usr.bin/mkcsmapper usr.bin/mkesdb
+
+bootstrap-tools: bootstrap-tools-before bootstrap-tools-targets
+       touch ${BTOOLSDEST}/.bootstrap_done
+
+bootstrap-tools-before:
        ${LN} -fs /bin/date ${BTOOLSDEST}/bin/date
-.for _tool in ${_strfile} \
-    usr.bin/patch \
-    bin/chmod bin/cp bin/cpdup bin/dd bin/mkdir bin/rm bin/echo bin/test \
-    bin/cat bin/ln bin/mv bin/csh bin/expr bin/sh \
-    bin/hostname bin/kill \
-    usr.bin/yacc usr.bin/colldef usr.bin/uudecode usr.bin/xinstall \
-    usr.bin/m4 usr.bin/rpcgen usr.bin/make usr.bin/awk usr.bin/stat \
-    usr.bin/find usr.bin/lex usr.bin/sed usr.bin/uname usr.bin/touch \
-    usr.bin/mkdep usr.bin/mktemp usr.bin/lorder usr.bin/file2c \
-    usr.bin/tsort usr.bin/tr usr.bin/join usr.bin/wc usr.bin/basename \
-    usr.bin/gencat usr.bin/chflags usr.bin/expand usr.bin/paste \
-    usr.bin/mklocale usr.bin/uuencode usr.bin/compile_et usr.bin/hexdump \
-    usr.bin/cap_mkdb usr.bin/true usr.bin/false \
-    usr.bin/cmp usr.bin/xargs usr.bin/id usr.bin/env usr.bin/dirname \
-    usr.bin/tail usr.bin/unifdef \
-    usr.sbin/chown usr.sbin/mtree usr.sbin/config \
-    usr.sbin/btxld usr.sbin/zic usr.sbin/makewhatis \
-    gnu/usr.bin/texinfo gnu/usr.bin/grep usr.bin/sort \
-    usr.bin/gzip usr.bin/bzip2 usr.bin/mkcsmapper usr.bin/mkesdb
+
+bootstrap-tools-targets: ${BSTRAPDIRS:S/^/bstrap-/}
+
+.ORDER: bootstrap-tools-before bootstrap-tools-targets
+
+.for _tool in ${BSTRAPDIRS}
+bstrap-${_tool}!
        ${ECHODIR} "===> ${_tool} (bootstrap-tools)"; \
                cd ${.CURDIR}/${_tool}; \
                ${MAKE} DIRPRFX=${_tool}/ obj; \
@@ -782,7 +793,6 @@ bootstrap-tools:
                ${MAKE} DIRPRFX=${_tool}/ all; \
                ${MAKE} DIRPRFX=${_tool}/ DESTDIR=${BTOOLSDEST} install
 .endfor
-       touch ${BTOOLSDEST}/.bootstrap_done
 
 # build-tools: Build special purpose build tools.
 #
@@ -802,8 +812,15 @@ _gcc44_tools= gnu/usr.bin/cc44/cc_prep gnu/usr.bin/cc44/cc_tools
 _custom_cross= libexec/customcc
 _binutils= gnu/usr.bin/${WORLD_BINUTILSVER}
 
-build-tools:
-.for _tool in ${_gcc41_tools} ${_gcc44_tools} ${_libkrb5} ${_share}
+BTOOLSDIRS= ${_gcc41_tools} ${_gcc44_tools} ${_libkrb5} ${_share}
+
+build-tools: build-tools-targets
+       touch ${BTOOLSDEST}/.build_done
+
+build-tools-targets: ${BTOOLSDIRS:S/^/btools-/}
+
+.for _tool in ${BTOOLSDIRS}
+btools-${_tool}!
        ${ECHODIR} "===> ${_tool} (build-tools)"; \
                cd ${.CURDIR}/${_tool}; \
                ${MAKE} DIRPRFX=${_tool}/ obj; \
@@ -811,7 +828,6 @@ build-tools:
                ${MAKE} DIRPRFX=${_tool}/ all; \
                ${MAKE} DIRPRFX=${_tool}/ DESTDIR=${BTOOLSDEST} install
 .endfor
-       touch ${BTOOLSDEST}/.build_done
 
 #
 # cross-tools: Build cross-building tools
@@ -820,10 +836,16 @@ build-tools:
 _btxld=        usr.sbin/btxld
 .endif
 
-cross-tools:
-.for _tool in ${_btxld} ${_binutils} \
-             usr.bin/objformat \
-             ${_gcc41_cross} ${_gcc44_cross} ${_custom_cross}
+CTOOLSDIRS= ${_btxld} ${_binutils} usr.bin/objformat \
+               ${_gcc41_cross} ${_gcc44_cross} ${_custom_cross}
+
+cross-tools: cross-tools-targets
+       touch ${CTOOLSDEST}/.cross_done
+
+cross-tools-targets: ${CTOOLSDIRS:S/^/ctools-/}
+
+.for _tool in ${CTOOLSDIRS}
+ctools-${_tool}!
        ${ECHODIR} "===> ${_tool} (cross-tools)"; \
                cd ${.CURDIR}/${_tool}; \
                ${MAKE} DIRPRFX=${_tool}/ obj; \
@@ -831,7 +853,6 @@ cross-tools:
                ${MAKE} DIRPRFX=${_tool}/ all; \
                ${MAKE} DIRPRFX=${_tool}/ DESTDIR=${CTOOLSDEST} install
 .endfor
-       touch ${CTOOLSDEST}/.cross_done
 
 #
 # hierarchy - ensure that all the needed directories are present

diff --git a/bin/Makefile b/bin/Makefile
index 93efd52..8befa98 100644 (file)
--- a/bin/Makefile
+++ b/bin/Makefile
@@ -40,4 +40,8 @@ SUBDIR= cat \
 SUBDIR+=rmail
 .endif
 
+# maximum parallelism
+#
+SUBDIR_ORDERED=
+
 .include <bsd.subdir.mk>

diff --git a/games/Makefile b/games/Makefile
index a07b6ae..017d088 100644 (file)
--- a/games/Makefile
+++ b/games/Makefile
@@ -44,4 +44,8 @@ SUBDIR= adventure \
        worms \
        wump
 
+# maximum parallelism
+#
+SUBDIR_ORDERED=
+
 .include <bsd.subdir.mk>

diff --git a/gnu/lib/Makefile b/gnu/lib/Makefile
index 2ff12bd..fba8be1 100644 (file)
--- a/gnu/lib/Makefile
+++ b/gnu/lib/Makefile
@@ -6,4 +6,8 @@ SUBDIR= gcc44 libcryptsetup libdevmapper libdialog libluks liblvm
 SUBDIR+= gcc41
 .endif
 
+# maximum parallelism
+#
+SUBDIR_ORDERED=
+
 .include <bsd.subdir.mk>

diff --git a/gnu/usr.bin/Makefile b/gnu/usr.bin/Makefile
index b79a7ce..db6f6c5 100644 (file)
--- a/gnu/usr.bin/Makefile
+++ b/gnu/usr.bin/Makefile
@@ -21,4 +21,8 @@ SUBDIR+= cc41
 SUBDIR+= cc44
 .ORDER: binutils222 cc44
 
+# maximum parallelism
+#
+SUBDIR_ORDERED=
+
 .include <bsd.subdir.mk>

diff --git a/libexec/Makefile b/libexec/Makefile
index 0352327..58cfa94 100644 (file)
--- a/libexec/Makefile
+++ b/libexec/Makefile
@@ -35,4 +35,8 @@ SUBDIR=       atrun \
 SUBDIR+=mail.local smrsh
 .endif
 
+# maximum parallelism
+#
+SUBDIR_ORDERED=
+
 .include <bsd.subdir.mk>

diff --git a/sbin/Makefile b/sbin/Makefile
index a5d58ad..846ac44 100644 (file)
--- a/sbin/Makefile
+++ b/sbin/Makefile
@@ -115,4 +115,8 @@ SUBDIR+= ${MACHINE}
 SUBDIR+= ${MACHINE_ARCH}
 .endif
 
+# maximum parallelism
+#
+SUBDIR_ORDERED=
+
 .include <bsd.subdir.mk>

diff --git a/secure/lib/Makefile b/secure/lib/Makefile
index 8735d20..e95425f 100644 (file)
--- a/secure/lib/Makefile
+++ b/secure/lib/Makefile
@@ -9,4 +9,8 @@ SUBDIR+=libssh
 .endif
 .endif
 
+# maximum parallelism
+#
+SUBDIR_ORDERED=
+
 .include <bsd.subdir.mk>

diff --git a/usr.bin/Makefile b/usr.bin/Makefile
index a63a52f..35bebcd 100644 (file)
--- a/usr.bin/Makefile
+++ b/usr.bin/Makefile
@@ -7,6 +7,11 @@
 # XXX Use GNU versions: apropos diff grep ld man ptx uucp whatis
 # Moved to secure: bdes
 #
+
+# No ordering dependencies, everything can run in parallel
+#
+SUBDIR_ORDERED=
+
 SUBDIR=        alias \
        apply \
        asa \

diff --git a/usr.sbin/Makefile b/usr.sbin/Makefile
index d3680dd..9ecfdf7 100644 (file)
--- a/usr.sbin/Makefile
+++ b/usr.sbin/Makefile
@@ -188,4 +188,8 @@ SUBDIR+=boot0cfg
 SUBDIR+=installer
 .endif
 
+# maximum parallelism
+#
+SUBDIR_ORDERED=
+
 .include <bsd.subdir.mk>