test/fastbulk/Makefile

   1 # Makefile for fastbulk - fast incremental pkgsrc bulk [re]build
   2 #
   3 # deinstall _UPDATE_RUNNING=YES DEINSTALLDEPENDS=ALL
   4 # package
   5 # show-depends  (output format wildcard:relative_path_to_pkgsrc_dir)
   6
   7 BUILDBASE=/build/fastbulk
   8 BUILDROOT=${BUILDBASE}/root
   9 SYSPKGSRC=/usr/pkgsrc
  10 SYSROOT=/
  11 CHROOTENV=chroot ${BUILDROOT} /bin/sh -c
  12 NPARALLEL?= 8
  13 # -DNOTMPFS
  14
  15 all:
  16         @echo "fastbulk build options"
  17         @echo "    make init         - create/initialize topology"
  18         @echo "    make build        - build all packages from sources"
  19         @echo "    make quick        - incremental build restart"
  20         @echo "    make umount       - umount the build environment"
  21         @echo "    make cleanup      - umount, rm binpkgs and pkgs"
  22         @echo ""
  23         @echo "    make changes      - deinstall modified packages"
  24         @echo "    make destroy      - destroy the fastbulk setup"
  25         @echo ""
  26         @echo "make variables (defaults shown):"
  27         @echo ""
  28         @echo "    BUILDBASE=/build/fastbulk     all fastbulk-generated files"
  29         @echo "    SYSPKGSRC=/usr/pkgsrc         where to get pkgsrc from"
  30         @echo "    SYSROOT=/                     rootenv to replicate"
  31         @echo "    NPARALLEL=8                   maximum concurrency"
  32         @echo ""
  33         @echo "NOTE: ${BUILDBASE}/{distfiles,packages} are normally retained"
  34         @echo "      to avoid having to re-pull and for incremental operation."
  35         @echo "      'build' will delete all packages, 'quick' will not."
  36         @echo ""
  37         @echo "NOTE: When building, the current progress is in "
  38         @echo "      ${BUILDROOT}/tmp/logs/{run,good,bad}.  Each build"
  39         @echo "      has its own log file named after the pkgsrc path."
  40
  41 init:   umount initialize mount
  42
  43 build:  umount mount bootstrap
  44         ${CHROOTENV} "cd /tmp/track; ./fastbulk -j ${NPARALLEL} /usr/pkgsrc"
  45
  46 quick:  umount mount quickstrap
  47         ${CHROOTENV} "cd /tmp/track; ./fastbulk -j ${NPARALLEL} /usr/pkgsrc"
  48
  49 cleanup: umount
  50         cpdup -i0 -s0 -o ${BUILDBASE}/packages ${BUILDBASE}/packages.save
  51         find -x ${BUILDBASE}/packages -depth -depth +0 -delete
  52         find -x ${BUILDBASE}/track -depth -depth +0 -delete
  53         find -x ${BUILDBASE}/bin -depth -depth +0 -delete
  54         find -x ${BUILDBASE}/tmp/logs -depth -depth +0 -delete
  55         find -x ${BUILDBASE}/var.db.pkg -depth -depth +0 -delete
  56         find -x ${BUILDBASE}/var.db.pkg.refcount -depth -depth +0 -delete
  57         find -x ${BUILDBASE}/usr.pkg -depth -depth +0 -delete
  58         rm -rf ${BUILDBASE}/usr.obj/work
  59         mkdir -p ${BUILDBASE}/tmp/logs/run
  60         mkdir -p ${BUILDBASE}/tmp/logs/good
  61         mkdir -p ${BUILDBASE}/tmp/logs/bad
  62
  63 destroy: umount
  64         rm -rf ${BUILDBASE}
  65
  66 umount:
  67         -resident -d ${BUILDROOT}/usr/pkg/bin/bmake
  68         -umount ${BUILDROOT}/usr/pkgsrc/distfiles > /dev/null 2>&1
  69         -umount ${BUILDROOT}/usr/pkgsrc/packages > /dev/null 2>&1
  70         -umount ${BUILDROOT}/usr/pkgsrc > /dev/null 2>&1
  71         -umount ${BUILDROOT}/dev > /dev/null 2>&1
  72         -umount ${BUILDROOT}/var/db/pkg.refcount > /dev/null 2>&1
  73         -umount ${BUILDROOT}/var/db/pkg > /dev/null 2>&1
  74         -umount ${BUILDROOT}/var/db > /dev/null 2>&1
  75         -umount ${BUILDROOT}/usr/obj > /dev/null 2>&1
  76         -umount ${BUILDROOT}/usr/pkg > /dev/null 2>&1
  77         -umount ${BUILDROOT}/etc > /dev/null 2>&1
  78         -umount ${BUILDROOT}/var/tmp > /dev/null 2>&1
  79         -umount ${BUILDROOT}/tmp/track > /dev/null 2>&1
  80         -umount ${BUILDROOT}/tmp > /dev/null 2>&1
  81         -umount ${BUILDROOT}/var > /dev/null 2>&1
  82         -umount ${BUILDROOT}/usr > /dev/null 2>&1
  83         -umount ${BUILDROOT} > /dev/null 2>&1
  84
  85 mount:
  86         mount_null -o ro ${SYSROOT} ${BUILDROOT}
  87         mount_null -o ro ${SYSROOT}/usr ${BUILDROOT}/usr
  88         mount_null -o ro ${SYSROOT}/var ${BUILDROOT}/var
  89         mount_null ${BUILDBASE}/tmp ${BUILDROOT}/tmp
  90         mount_null ${BUILDBASE}/tmp ${BUILDROOT}/var/tmp
  91         mount_null ${BUILDBASE}/track ${BUILDROOT}/tmp/track
  92         mount_null ${BUILDBASE}/etc ${BUILDROOT}/etc
  93         mount_null ${BUILDBASE}/usr.pkg ${BUILDROOT}/usr/pkg
  94         mount_null ${BUILDBASE}/usr.pkgsrc ${BUILDROOT}/usr/pkgsrc
  95 .if defined(NOTMPFS)
  96         mount_null ${BUILDBASE}/usr.obj ${BUILDROOT}/usr/obj
  97 .else
  98         mount_tmpfs dummy ${BUILDROOT}/usr/obj
  99 .endif
 100         mount_null ${BUILDBASE}/var.db ${BUILDROOT}/var/db
 101         mount_null ${BUILDBASE}/var.db.pkg ${BUILDROOT}/var/db/pkg
 102         mount_null ${BUILDBASE}/var.db.pkg.refcount \
 103                         ${BUILDROOT}/var/db/pkg.refcount
 104         mount_null ${SYSROOT}/dev ${BUILDROOT}/dev
 105         mount_null ${BUILDBASE}/packages ${BUILDROOT}/usr/pkgsrc/packages
 106         mount_null ${BUILDBASE}/distfiles ${BUILDROOT}/usr/pkgsrc/distfiles
 107         -resident ${BUILDROOT}/usr/pkg/bin/bmake > /dev/null 2>&1
 108
 109 initialize:
 110         #
 111         # The real root must contain these directories so
 112         # we can do NULL mounts on them.
 113         #
 114         mkdir -p ${SYSROOT}/var/db/pkg
 115         mkdir -p ${SYSROOT}/var/db/pkg.refcount
 116         mkdir -p ${SYSROOT}/usr/pkgsrc
 117         mkdir -p ${SYSROOT}/usr/pkg
 118         mkdir -p ${BUILDBASE}
 119         #
 120         # buildbase directory structure (this is not the chroot topology)
 121         #
 122         mkdir -p ${BUILDBASE}
 123         mkdir -p ${BUILDBASE}/packages
 124         mkdir -p ${BUILDBASE}/packages.save
 125         mkdir -p ${BUILDBASE}/distfiles
 126         mkdir -p ${BUILDBASE}/bin
 127         mkdir -p ${BUILDBASE}/tmp
 128         mkdir -p ${BUILDBASE}/tmp/track
 129         mkdir -p ${BUILDBASE}/tmp/logs
 130         mkdir -p ${BUILDBASE}/tmp/logs/run
 131         mkdir -p ${BUILDBASE}/tmp/logs/good
 132         mkdir -p ${BUILDBASE}/tmp/logs/bad
 133         mkdir -p ${BUILDBASE}/etc
 134         mkdir -p ${BUILDBASE}/usr.pkg
 135         mkdir -p ${BUILDBASE}/usr.pkgsrc
 136         mkdir -p ${BUILDBASE}/usr.obj
 137         mkdir -p ${BUILDBASE}/var.db
 138         mkdir -p ${BUILDBASE}/var.db/pkg
 139         mkdir -p ${BUILDBASE}/var.db/pkg.refcount
 140         mkdir -p ${BUILDBASE}/var.db.pkg
 141         mkdir -p ${BUILDBASE}/var.db.pkg.refcount
 142         mkdir -p ${BUILDBASE}/track
 143         #
 144         # Copy stuff we need, make sure the pkgsrc copy we
 145         # make has the needed mount points for packages and distfiles.
 146         #
 147         cpdup -s0 -i0 ${SYSROOT}/etc ${BUILDBASE}/etc
 148         cpdup -s0 -i0 -X ${.CURDIR}/vardb.ign \
 149                         ${SYSROOT}/var/db ${BUILDBASE}/var.db
 150         cpdup -s0 -i0 -X ${.CURDIR}/pkgsrc.ign \
 151                         ${SYSPKGSRC} ${BUILDBASE}/usr.pkgsrc
 152         mkdir -p ${BUILDBASE}/usr.pkgsrc/packages
 153         mkdir -p ${BUILDBASE}/usr.pkgsrc/distfiles
 154         mkdir -p ${BUILDROOT}
 155         #find -x -X ${BUILDROOT}/usr/pkgsrc -name work | xargs rm -rf
 156
 157 # quickstrap - get ready to start a fastbulk build
 158 #
 159 # NOTE: ${BUILDROOT}/usr/obj might be mounted from a tmpfs and not
 160 #       from ${BUILDBASE}/usr.obj
 161 #
 162 quickstrap: prestage
 163         rm -rf ${BUILDBASE}/usr.obj/work
 164         rm -rf ${BUILDROOT}/usr/obj/work
 165         rm -rf ${BUILDROOT}/tmp/logs/run
 166         rm -rf ${BUILDROOT}/tmp/logs/bad
 167         mkdir -p ${BUILDROOT}/tmp/logs/run
 168         mkdir -p ${BUILDROOT}/tmp/logs/bad
 169         mkdir -p ${BUILDROOT}/usr/obj/work
 170         #
 171         # Sometimes we get blah.core files in /var/db/pkg, which
 172         # interferes with pkg_add.  Make sure they are gone.
 173         #
 174         find ${BUILDROOT}/var/db/pkg -name '*.core' -delete
 175         find ${BUILDROOT}/var/db/pkg -type d -depth 1 -delete
 176
 177 bootstrap: quickstrap
 178         -resident -d ${BUILDROOT}/usr/pkg/bin/bmake
 179         cpdup -i0 -s0 -o ${BUILDBASE}/packages ${BUILDBASE}/packages.save
 180         find -x ${BUILDBASE}/packages -depth -depth +0 -delete
 181         find -x ${BUILDBASE}/usr.pkgsrc/packages -depth -depth +0 -delete
 182         find -x ${BUILDBASE}/var.db.pkg -depth -depth +0 -delete
 183         find -x ${BUILDBASE}/var.db.pkg.refcount -depth -depth +0 -delete
 184         find -x ${BUILDBASE}/usr.pkg -depth -depth +0 -delete
 185         # note: work dir might be either tmpfs or null mount
 186         #${CHROOTENV} "pkg_admin -K /var/db/pkg fetch-pkg-vulnerabilities"
 187         #${CHROOTENV} "pkg_admin rebuild"
 188         rm -rf ${BUILDROOT}/usr/obj/work
 189         ${CHROOTENV} "cd /usr/pkgsrc/bootstrap; ./bootstrap --workdir=/usr/obj/work"
 190         -resident ${BUILDROOT}/usr/pkg/bin/bmake > /dev/null 2>&1
 191         cat mk.conf.add >> ${BUILDROOT}/usr/pkg/etc/mk.conf
 192
 193 # Figure out what has changed and deinstall the related packages
 194 #
 195
 196 PRESTAGE= ${BUILDBASE}/bin/getpkgsrcdir
 197 PRESTAGE+= ${BUILDBASE}/bin/fastbulk
 198 PRESTAGE+= ${BUILDBASE}/track/Makefile
 199 PRESTAGE+= ${BUILDBASE}/track/dochanges
 200 PRESTAGE+= ${BUILDBASE}/track/dobuild
 201 PRESTAGE+= ${BUILDBASE}/track/fastbulk
 202
 203 prestage: ${PRESTAGE}
 204
 205 changes: ${PRESTAGE}
 206         @if [ ! -f ${BUILDBASE}/track/dochanges ]; then \
 207                 touch ${BUILDBASE}/track/dochanges; fi
 208         @if [ ! -f ${BUILDBASE}/track/changes.raw ]; then \
 209                 touch ${BUILDBASE}/track/changes.raw; fi
 210         ( cat ${BUILDBASE}/track/changes.raw; \
 211           find ${SYSPKGSRC} -mnewer ${BUILDBASE}/track/dochanges ) | \
 212             sort | uniq | ${BUILDBASE}/bin/getpkgsrcdir ${SYSPKGSRC} | \
 213             sort | uniq > ${BUILDBASE}/track/changes.raw.new
 214         mv ${BUILDBASE}/track/changes.raw.new ${BUILDBASE}/track/changes.raw
 215         ${BUILDBASE}/bin/getpkgsrcdir -s ${SYSPKGSRC} < \
 216                 ${BUILDBASE}/track/changes.raw > \
 217                 ${BUILDBASE}/track/changes.txt
 218         cp Makefile ${BUILDBASE}/track/Makefile
 219         ${CHROOTENV} "cd /tmp/track; ./dochanges"
 220
 221
 222 ${BUILDBASE}/bin/getpkgsrcdir: getpkgsrcdir.c
 223         cc getpkgsrcdir.c -o ${BUILDBASE}/bin/getpkgsrcdir
 224
 225 ${BUILDBASE}/bin/fastbulk: fastbulk.c
 226         cc fastbulk.c -o ${BUILDBASE}/bin/fastbulk
 227
 228 ${BUILDBASE}/track/Makefile: ${.CURDIR}/Makefile
 229         cp -p ${.CURDIR}/Makefile ${BUILDBASE}/track/Makefile
 230
 231 ${BUILDBASE}/track/dochanges: ${.CURDIR}/dochanges
 232         cp -p ${.CURDIR}/dochanges ${BUILDBASE}/track/dochanges
 233
 234 ${BUILDBASE}/track/dobuild: ${.CURDIR}/dobuild
 235         cp -p ${.CURDIR}/dobuild ${BUILDBASE}/track/dobuild
 236
 237 ${BUILDBASE}/track/fastbulk: ${BUILDBASE}/bin/fastbulk
 238         cp -p ${BUILDBASE}/bin/fastbulk ${BUILDBASE}/track/fastbulk