fastbulk - Rework and clean up the Makefile

* Rewrite the Makefile.  'make' with no arguments for more information.

* Proactively check and remove a dead /var/db/pkg directory after
  each build.

diff --git a/test/fastbulk/Makefile b/test/fastbulk/Makefile
index e691d4e..01a46e0 100644 (file)
--- a/test/fastbulk/Makefile
+++ b/test/fastbulk/Makefile
@@ -14,13 +14,15 @@ NPARALLEL?= 8
 
 all:
        @echo "fastbulk build options"
-       @echo "    make scratch      - cleanup, setup, cpdup /etc, bootstrap"
-       @echo "    make build        - iterate all dirs / build package"
-       @echo "    make setup        - setup/mount the build environment"
-       @echo "    make cleanup      - unmount the build environment"
-       @echo "    make realclean    - rm -rf the build env except for"
-       @echo "                        packages and distfiles"
+       @echo "    make init         - create/initialize topology"
+       @echo "    make build        - build all packages from sources"
+       @echo "    make quick        - incremental build restart"
+       @echo "    make umount       - umount the build environment"
+       @echo "    make cleanup      - umount, rm binpkgs and pkgs"
+       @echo ""
        @echo "    make changes      - deinstall modified packages"
+       @echo "    make destroy      - destroy the fastbulk setup"
+       @echo ""
        @echo "make variables (defaults shown):"
        @echo ""
        @echo "    BUILDBASE=/build/fastbulk     all fastbulk-generated files"
@@ -29,54 +31,58 @@ all:
        @echo "    NPARALLEL=8                   maximum concurrency"
        @echo ""
        @echo "NOTE: ${BUILDBASE}/{distfiles,packages} are normally retained"
-       @echo "      to avoid having to re-pull and for incremental operation"
+       @echo "      to avoid having to re-pull and for incremental operation."
+       @echo "      'build' will delete all packages, 'quick' will not."
        @echo ""
-       @echo "NOTE: When building with 'make build' the current progress"
-       @echo "      is in ${BUILDROOT}/tmp/logs/{run,good,bad}, each build"
-       @echo "      has its own log file named after the pkgsrc path"
-
-scratch: cleanup
-       rm -rf ${BUILDBASE}/var.db
-       rm -rf ${BUILDBASE}/var.db.pkg
-       rm -rf ${BUILDBASE}/var.db.pkg.refcount
-       rm -rf ${BUILDBASE}/etc
-       rm -rf ${BUILDBASE}/usr.pkg
+       @echo "NOTE: When building, the current progress is in "
+       @echo "      ${BUILDROOT}/tmp/logs/{run,good,bad}.  Each build"
+       @echo "      has its own log file named after the pkgsrc path."
+
+init:  umount initialize mount
+
+build: umount mount bootstrap
+       ${CHROOTENV} "cd /tmp/track; ./fastbulk -j ${NPARALLEL} /usr/pkgsrc"
+
+quick: umount mount quickstrap
+       ${CHROOTENV} "cd /tmp/track; ./fastbulk -j ${NPARALLEL} /usr/pkgsrc"
+
+cleanup: umount
+       cpdup -i0 -s0 -o ${BUILDBASE}/packages ${BUILDBASE}/packages.save
+       find -x ${BUILDBASE}/packages -depth -depth +0 -delete
+       find -x ${BUILDBASE}/track -depth -depth +0 -delete
+       find -x ${BUILDBASE}/bin -depth -depth +0 -delete
+       find -x ${BUILDBASE}/tmp/logs -depth -depth +0 -delete
+       find -x ${BUILDBASE}/var.db.pkg -depth -depth +0 -delete
+       find -x ${BUILDBASE}/var.db.pkg.refcount -depth -depth +0 -delete
+       find -x ${BUILDBASE}/usr.pkg -depth -depth +0 -delete
        rm -rf ${BUILDBASE}/usr.obj/work
-       rm -rf ${BUILDBASE}/tmp/logs
-       rm -rf ${BUILDBASE}/tmp/track
-       cpdup ${SYSROOT}/etc ${BUILDBASE}/etc
-       cpdup ${SYSROOT}/var/db ${BUILDBASE}/var.db
-       rm -rf ${BUILDBASE}/var.db/pkg
-       rm -rf ${BUILDBASE}/var.db/pkg.refcount
-       ${MAKE} setup bootstrap
-       #${MAKE} build
-
-setup: cleanup
-       mkdir -p ${SYSROOT}/var/db/pkg
-       mkdir -p ${SYSROOT}/var/db/pkg.refcount
-       mkdir -p ${SYSROOT}/usr/pkgsrc
-       mkdir -p ${SYSROOT}/usr/pkg
-       mkdir -p ${BUILDBASE}
-       mkdir -p ${BUILDBASE}/packages
-       mkdir -p ${BUILDBASE}/distfiles
-       mkdir -p ${BUILDBASE}/bin
-       mkdir -p ${BUILDBASE}/tmp
-       mkdir -p ${BUILDBASE}/tmp/track
-       mkdir -p ${BUILDBASE}/tmp/logs
        mkdir -p ${BUILDBASE}/tmp/logs/run
        mkdir -p ${BUILDBASE}/tmp/logs/good
        mkdir -p ${BUILDBASE}/tmp/logs/bad
-       mkdir -p ${BUILDBASE}/etc
-       mkdir -p ${BUILDBASE}/usr.pkg
-       mkdir -p ${BUILDBASE}/usr.pkgsrc
-       mkdir -p ${BUILDBASE}/usr.obj
-       mkdir -p ${BUILDBASE}/var.db
-       mkdir -p ${BUILDBASE}/var.db/pkg
-       mkdir -p ${BUILDBASE}/var.db/pkg.refcount
-       mkdir -p ${BUILDBASE}/var.db.pkg
-       mkdir -p ${BUILDBASE}/var.db.pkg.refcount
-       mkdir -p ${BUILDBASE}/track
-       mkdir -p ${BUILDROOT}
+
+destroy: umount
+       rm -rf ${BUILDBASE}
+
+umount:
+       -resident -d ${BUILDROOT}/usr/pkg/bin/bmake
+       -umount ${BUILDROOT}/usr/pkgsrc/distfiles > /dev/null 2>&1
+       -umount ${BUILDROOT}/usr/pkgsrc/packages > /dev/null 2>&1
+       -umount ${BUILDROOT}/usr/pkgsrc > /dev/null 2>&1
+       -umount ${BUILDROOT}/dev > /dev/null 2>&1
+       -umount ${BUILDROOT}/var/db/pkg.refcount > /dev/null 2>&1
+       -umount ${BUILDROOT}/var/db/pkg > /dev/null 2>&1
+       -umount ${BUILDROOT}/var/db > /dev/null 2>&1
+       -umount ${BUILDROOT}/usr/obj > /dev/null 2>&1
+       -umount ${BUILDROOT}/usr/pkg > /dev/null 2>&1
+       -umount ${BUILDROOT}/etc > /dev/null 2>&1
+       -umount ${BUILDROOT}/var/tmp > /dev/null 2>&1
+       -umount ${BUILDROOT}/tmp/track > /dev/null 2>&1
+       -umount ${BUILDROOT}/tmp > /dev/null 2>&1
+       -umount ${BUILDROOT}/var > /dev/null 2>&1
+       -umount ${BUILDROOT}/usr > /dev/null 2>&1
+       -umount ${BUILDROOT} > /dev/null 2>&1
+
+mount:
        mount_null -o ro ${SYSROOT} ${BUILDROOT}
        mount_null -o ro ${SYSROOT}/usr ${BUILDROOT}/usr
        mount_null -o ro ${SYSROOT}/var ${BUILDROOT}/var
@@ -96,52 +102,92 @@ setup: cleanup
        mount_null ${BUILDBASE}/var.db.pkg.refcount \
                        ${BUILDROOT}/var/db/pkg.refcount
        mount_null ${SYSROOT}/dev ${BUILDROOT}/dev
-       cpdup -s0 -i0 -X ${.CURDIR}/pkgsrc.ign \
-                       ${SYSPKGSRC} ${BUILDROOT}/usr/pkgsrc
-       find ${BUILDROOT}/usr/pkgsrc -name work | xargs rm -rf
-       mkdir -p ${BUILDROOT}/usr/pkgsrc/packages
-       mkdir -p ${BUILDROOT}/usr/pkgsrc/distfiles
        mount_null ${BUILDBASE}/packages ${BUILDROOT}/usr/pkgsrc/packages
        mount_null ${BUILDBASE}/distfiles ${BUILDROOT}/usr/pkgsrc/distfiles
-       echo "finding and removing work directories"
+       -resident ${BUILDROOT}/usr/pkg/bin/bmake > /dev/null 2>&1
 
-cleanup:
-       -resident -d ${BUILDROOT}/usr/pkg/bin/bmake
-       -umount ${BUILDROOT}/usr/pkgsrc/distfiles > /dev/null 2>&1
-       -umount ${BUILDROOT}/usr/pkgsrc/packages > /dev/null 2>&1
-       -umount ${BUILDROOT}/usr/pkgsrc > /dev/null 2>&1
-       -umount ${BUILDROOT}/dev > /dev/null 2>&1
-       -umount ${BUILDROOT}/var/db/pkg.refcount > /dev/null 2>&1
-       -umount ${BUILDROOT}/var/db/pkg > /dev/null 2>&1
-       -umount ${BUILDROOT}/var/db > /dev/null 2>&1
-       -umount ${BUILDROOT}/usr/obj > /dev/null 2>&1
-       -umount ${BUILDROOT}/usr/pkg > /dev/null 2>&1
-       -umount ${BUILDROOT}/etc > /dev/null 2>&1
-       -umount ${BUILDROOT}/var/tmp > /dev/null 2>&1
-       -umount ${BUILDROOT}/tmp/track > /dev/null 2>&1
-       -umount ${BUILDROOT}/tmp > /dev/null 2>&1
-       -umount ${BUILDROOT}/var > /dev/null 2>&1
-       -umount ${BUILDROOT}/usr > /dev/null 2>&1
-       -umount ${BUILDROOT} > /dev/null 2>&1
+initialize:
+       #
+       # The real root must contain these directories so
+       # we can do NULL mounts on them.
+       #
+       mkdir -p ${SYSROOT}/var/db/pkg
+       mkdir -p ${SYSROOT}/var/db/pkg.refcount
+       mkdir -p ${SYSROOT}/usr/pkgsrc
+       mkdir -p ${SYSROOT}/usr/pkg
+       mkdir -p ${BUILDBASE}
+       #
+       # buildbase directory structure (this is not the chroot topology)
+       #
+       mkdir -p ${BUILDBASE}
+       mkdir -p ${BUILDBASE}/packages
+       mkdir -p ${BUILDBASE}/packages.save
+       mkdir -p ${BUILDBASE}/distfiles
+       mkdir -p ${BUILDBASE}/bin
+       mkdir -p ${BUILDBASE}/tmp
+       mkdir -p ${BUILDBASE}/tmp/track
+       mkdir -p ${BUILDBASE}/tmp/logs
+       mkdir -p ${BUILDBASE}/tmp/logs/run
+       mkdir -p ${BUILDBASE}/tmp/logs/good
+       mkdir -p ${BUILDBASE}/tmp/logs/bad
+       mkdir -p ${BUILDBASE}/etc
+       mkdir -p ${BUILDBASE}/usr.pkg
+       mkdir -p ${BUILDBASE}/usr.pkgsrc
+       mkdir -p ${BUILDBASE}/usr.obj
+       mkdir -p ${BUILDBASE}/var.db
+       mkdir -p ${BUILDBASE}/var.db/pkg
+       mkdir -p ${BUILDBASE}/var.db/pkg.refcount
+       mkdir -p ${BUILDBASE}/var.db.pkg
+       mkdir -p ${BUILDBASE}/var.db.pkg.refcount
+       mkdir -p ${BUILDBASE}/track
+       #
+       # Copy stuff we need, make sure the pkgsrc copy we
+       # make has the needed mount points for packages and distfiles.
+       #
+       cpdup -s0 -i0 ${SYSROOT}/etc ${BUILDBASE}/etc
+       cpdup -s0 -i0 -X ${.CURDIR}/vardb.ign \
+                       ${SYSROOT}/var/db ${BUILDBASE}/var.db
+       cpdup -s0 -i0 -X ${.CURDIR}/pkgsrc.ign \
+                       ${SYSPKGSRC} ${BUILDBASE}/usr.pkgsrc
+       mkdir -p ${BUILDBASE}/usr.pkgsrc/packages
+       mkdir -p ${BUILDBASE}/usr.pkgsrc/distfiles
+       mkdir -p ${BUILDROOT}
+       #find -x -X ${BUILDROOT}/usr/pkgsrc -name work | xargs rm -rf
 
-# Cleans everything except packages and distfiles
+# quickstrap - get ready to start a fastbulk build
 #
-realclean: cleanup
-       rm -rf ${BUILDBASE}/bin
-       rm -rf ${BUILDBASE}/tmp
-       rm -rf ${BUILDBASE}/track
-       rm -rf ${BUILDBASE}/usr.pkg
-       rm -rf ${BUILDBASE}/usr.pkgsrc
-       rm -rf ${BUILDBASE}/usr.obj
-       rm -rf ${BUILDBASE}/var.db.pkg
-       rm -rf ${BUILDBASE}/var.db.pkg.refcount
-
-# Bootstrap an empty chroot setup
+# NOTE: ${BUILDROOT}/usr/obj might be mounted from a tmpfs and not
+#      from ${BUILDBASE}/usr.obj
 #
-bootstrap:
+quickstrap: prestage
        rm -rf ${BUILDBASE}/usr.obj/work
+       rm -rf ${BUILDROOT}/usr/obj/work
+       rm -rf ${BUILDROOT}/tmp/logs/run
+       rm -rf ${BUILDROOT}/tmp/logs/bad
+       mkdir -p ${BUILDROOT}/tmp/logs/run
+       mkdir -p ${BUILDROOT}/tmp/logs/bad
+       mkdir -p ${BUILDROOT}/usr/obj/work
+       #
+       # Sometimes we get blah.core files in /var/db/pkg, which
+       # interferes with pkg_add.  Make sure they are gone.
+       #
+       find ${BUILDROOT}/var/db/pkg -name '*.core' -delete
+       find ${BUILDROOT}/var/db/pkg -type d -depth 1 -delete
+
+bootstrap: quickstrap
+       -resident -d ${BUILDROOT}/usr/pkg/bin/bmake
+       cpdup -i0 -s0 -o ${BUILDBASE}/packages ${BUILDBASE}/packages.save
+       find -x ${BUILDBASE}/packages -depth -depth +0 -delete
+       find -x ${BUILDBASE}/usr.pkgsrc/packages -depth -depth +0 -delete
+       find -x ${BUILDBASE}/var.db.pkg -depth -depth +0 -delete
+       find -x ${BUILDBASE}/var.db.pkg.refcount -depth -depth +0 -delete
+       find -x ${BUILDBASE}/usr.pkg -depth -depth +0 -delete
+       # note: work dir might be either tmpfs or null mount
+       #${CHROOTENV} "pkg_admin -K /var/db/pkg fetch-pkg-vulnerabilities"
+       #${CHROOTENV} "pkg_admin rebuild"
+       rm -rf ${BUILDROOT}/usr/obj/work
        ${CHROOTENV} "cd /usr/pkgsrc/bootstrap; ./bootstrap --workdir=/usr/obj/work"
-       resident ${BUILDROOT}/usr/pkg/bin/bmake
+       -resident ${BUILDROOT}/usr/pkg/bin/bmake > /dev/null 2>&1
        cat mk.conf.add >> ${BUILDROOT}/usr/pkg/etc/mk.conf
 
 # Figure out what has changed and deinstall the related packages
@@ -154,6 +200,8 @@ PRESTAGE+= ${BUILDBASE}/track/dochanges
 PRESTAGE+= ${BUILDBASE}/track/dobuild
 PRESTAGE+= ${BUILDBASE}/track/fastbulk
 
+prestage: ${PRESTAGE}
+
 changes: ${PRESTAGE}
        @if [ ! -f ${BUILDBASE}/track/dochanges ]; then \
                touch ${BUILDBASE}/track/dochanges; fi
@@ -170,17 +218,6 @@ changes: ${PRESTAGE}
        cp Makefile ${BUILDBASE}/track/Makefile
        ${CHROOTENV} "cd /tmp/track; ./dochanges"
 
-build: ${PRESTAGE}
-       # note: work dir might be either tmpfs or null mount
-       rm -rf ${BUILDROOT}/usr/obj/work
-       mkdir -p ${BUILDROOT}/usr/obj/work
-       -rm ${BUILDROOT}/tmp/logs/run/*
-       -rm ${BUILDROOT}/tmp/logs/bad/*
-       find ${BUILDROOT}/var/db/pkg -name '*.core' -delete
-       find ${BUILDROOT}/var/db/pkg -type d -depth 1 -delete
-       #${CHROOTENV} "pkg_admin -K /var/db/pkg fetch-pkg-vulnerabilities"
-       #${CHROOTENV} "pkg_admin rebuild"
-       ${CHROOTENV} "cd /tmp/track; ./fastbulk -j ${NPARALLEL} /usr/pkgsrc"
 
 ${BUILDBASE}/bin/getpkgsrcdir: getpkgsrcdir.c
        cc getpkgsrcdir.c -o ${BUILDBASE}/bin/getpkgsrcdir

diff --git a/test/fastbulk/dobuild b/test/fastbulk/dobuild
index 8d09364..55f1bea 100755 (executable)
--- a/test/fastbulk/dobuild
+++ b/test/fastbulk/dobuild
@@ -10,6 +10,7 @@
 #
 if ( $#argv == 1 ) then
        set pkgwild = "`bmake show-var VARNAME=PKGWILDCARD`.lck"
+       set pkgname = "`bmake show-var VARNAME=PKGNAME`"
        mkdir -p /tmp/track/locks
        echo "FASTBULK START ${argv[1]}: `date`"
        lockf "/tmp/track/locks/$pkgwild" /tmp/track/dobuild ${argv[1]} "$pkgwild"
@@ -18,6 +19,8 @@ if ( $#argv == 1 ) then
                exit 0
        else
                echo "FASTBULK FAILURE ${argv[1]}: `date`"
+               find /var/db/pkg/$pkgname -name '*.core' -delete
+               rmdir /var/db/pkg/$pkgname >& /dev/null
                exit 1
        endif
 endif
@@ -30,6 +33,12 @@ set logname = "$argv[1]"
 set pkgwild = "$argv[2]"
 unsetenv PKG_PATH
 
+# Make sure /var/db/pkg/ does not have any .core files messing it up,
+# remove the recorded installation if it becomes empty.
+#
+find /var/db/pkg/$pkgname -name '*.core' -delete
+rmdir /var/db/pkg/$pkgname >& /dev/null
+
 # Check if already installed, prevent overwrite of
 # previous run's logfile if it exists, by deleting
 # the current run file.  Else put in a simple logfile.

diff --git a/test/fastbulk/vardb.ign b/test/fastbulk/vardb.ign
new file mode 100644 (file)
index 0000000..51178dd
--- /dev/null
+++ b/test/fastbulk/vardb.ign
@@ -0,0 +1,2 @@
+pkg
+pkg.refcount