fastbulk - Rework and clean up the Makefile
[dragonfly.git] / test / fastbulk / Makefile
1 # Makefile for fastbulk - fast incremental pkgsrc bulk [re]build
2 #
3 # deinstall _UPDATE_RUNNING=YES DEINSTALLDEPENDS=ALL
4 # package
5 # show-depends  (output format wildcard:relative_path_to_pkgsrc_dir)
6
7 BUILDBASE=/build/fastbulk
8 BUILDROOT=${BUILDBASE}/root
9 SYSPKGSRC=/usr/pkgsrc
10 SYSROOT=/
11 CHROOTENV=chroot ${BUILDROOT} /bin/sh -c
12 NPARALLEL?= 8
13 # -DNOTMPFS
14
15 all:
16         @echo "fastbulk build options"
17         @echo "    make init         - create/initialize topology"
18         @echo "    make build        - build all packages from sources"
19         @echo "    make quick        - incremental build restart"
20         @echo "    make umount       - umount the build environment"
21         @echo "    make cleanup      - umount, rm binpkgs and pkgs"
22         @echo ""
23         @echo "    make changes      - deinstall modified packages"
24         @echo "    make destroy      - destroy the fastbulk setup"
25         @echo ""
26         @echo "make variables (defaults shown):"
27         @echo ""
28         @echo "    BUILDBASE=/build/fastbulk     all fastbulk-generated files"
29         @echo "    SYSPKGSRC=/usr/pkgsrc         where to get pkgsrc from"
30         @echo "    SYSROOT=/                     rootenv to replicate"
31         @echo "    NPARALLEL=8                   maximum concurrency"
32         @echo ""
33         @echo "NOTE: ${BUILDBASE}/{distfiles,packages} are normally retained"
34         @echo "      to avoid having to re-pull and for incremental operation."
35         @echo "      'build' will delete all packages, 'quick' will not."
36         @echo ""
37         @echo "NOTE: When building, the current progress is in "
38         @echo "      ${BUILDROOT}/tmp/logs/{run,good,bad}.  Each build"
39         @echo "      has its own log file named after the pkgsrc path."
40
41 init:   umount initialize mount
42
43 build:  umount mount bootstrap
44         ${CHROOTENV} "cd /tmp/track; ./fastbulk -j ${NPARALLEL} /usr/pkgsrc"
45
46 quick:  umount mount quickstrap
47         ${CHROOTENV} "cd /tmp/track; ./fastbulk -j ${NPARALLEL} /usr/pkgsrc"
48
49 cleanup: umount
50         cpdup -i0 -s0 -o ${BUILDBASE}/packages ${BUILDBASE}/packages.save
51         find -x ${BUILDBASE}/packages -depth -depth +0 -delete
52         find -x ${BUILDBASE}/track -depth -depth +0 -delete
53         find -x ${BUILDBASE}/bin -depth -depth +0 -delete
54         find -x ${BUILDBASE}/tmp/logs -depth -depth +0 -delete
55         find -x ${BUILDBASE}/var.db.pkg -depth -depth +0 -delete
56         find -x ${BUILDBASE}/var.db.pkg.refcount -depth -depth +0 -delete
57         find -x ${BUILDBASE}/usr.pkg -depth -depth +0 -delete
58         rm -rf ${BUILDBASE}/usr.obj/work
59         mkdir -p ${BUILDBASE}/tmp/logs/run
60         mkdir -p ${BUILDBASE}/tmp/logs/good
61         mkdir -p ${BUILDBASE}/tmp/logs/bad
62
63 destroy: umount
64         rm -rf ${BUILDBASE}
65
66 umount:
67         -resident -d ${BUILDROOT}/usr/pkg/bin/bmake
68         -umount ${BUILDROOT}/usr/pkgsrc/distfiles > /dev/null 2>&1
69         -umount ${BUILDROOT}/usr/pkgsrc/packages > /dev/null 2>&1
70         -umount ${BUILDROOT}/usr/pkgsrc > /dev/null 2>&1
71         -umount ${BUILDROOT}/dev > /dev/null 2>&1
72         -umount ${BUILDROOT}/var/db/pkg.refcount > /dev/null 2>&1
73         -umount ${BUILDROOT}/var/db/pkg > /dev/null 2>&1
74         -umount ${BUILDROOT}/var/db > /dev/null 2>&1
75         -umount ${BUILDROOT}/usr/obj > /dev/null 2>&1
76         -umount ${BUILDROOT}/usr/pkg > /dev/null 2>&1
77         -umount ${BUILDROOT}/etc > /dev/null 2>&1
78         -umount ${BUILDROOT}/var/tmp > /dev/null 2>&1
79         -umount ${BUILDROOT}/tmp/track > /dev/null 2>&1
80         -umount ${BUILDROOT}/tmp > /dev/null 2>&1
81         -umount ${BUILDROOT}/var > /dev/null 2>&1
82         -umount ${BUILDROOT}/usr > /dev/null 2>&1
83         -umount ${BUILDROOT} > /dev/null 2>&1
84
85 mount:
86         mount_null -o ro ${SYSROOT} ${BUILDROOT}
87         mount_null -o ro ${SYSROOT}/usr ${BUILDROOT}/usr
88         mount_null -o ro ${SYSROOT}/var ${BUILDROOT}/var
89         mount_null ${BUILDBASE}/tmp ${BUILDROOT}/tmp
90         mount_null ${BUILDBASE}/tmp ${BUILDROOT}/var/tmp
91         mount_null ${BUILDBASE}/track ${BUILDROOT}/tmp/track
92         mount_null ${BUILDBASE}/etc ${BUILDROOT}/etc
93         mount_null ${BUILDBASE}/usr.pkg ${BUILDROOT}/usr/pkg
94         mount_null ${BUILDBASE}/usr.pkgsrc ${BUILDROOT}/usr/pkgsrc
95 .if defined(NOTMPFS)
96         mount_null ${BUILDBASE}/usr.obj ${BUILDROOT}/usr/obj
97 .else
98         mount_tmpfs dummy ${BUILDROOT}/usr/obj
99 .endif
100         mount_null ${BUILDBASE}/var.db ${BUILDROOT}/var/db
101         mount_null ${BUILDBASE}/var.db.pkg ${BUILDROOT}/var/db/pkg
102         mount_null ${BUILDBASE}/var.db.pkg.refcount \
103                         ${BUILDROOT}/var/db/pkg.refcount
104         mount_null ${SYSROOT}/dev ${BUILDROOT}/dev
105         mount_null ${BUILDBASE}/packages ${BUILDROOT}/usr/pkgsrc/packages
106         mount_null ${BUILDBASE}/distfiles ${BUILDROOT}/usr/pkgsrc/distfiles
107         -resident ${BUILDROOT}/usr/pkg/bin/bmake > /dev/null 2>&1
108
109 initialize:
110         #
111         # The real root must contain these directories so
112         # we can do NULL mounts on them.
113         #
114         mkdir -p ${SYSROOT}/var/db/pkg
115         mkdir -p ${SYSROOT}/var/db/pkg.refcount
116         mkdir -p ${SYSROOT}/usr/pkgsrc
117         mkdir -p ${SYSROOT}/usr/pkg
118         mkdir -p ${BUILDBASE}
119         #
120         # buildbase directory structure (this is not the chroot topology)
121         #
122         mkdir -p ${BUILDBASE}
123         mkdir -p ${BUILDBASE}/packages
124         mkdir -p ${BUILDBASE}/packages.save
125         mkdir -p ${BUILDBASE}/distfiles
126         mkdir -p ${BUILDBASE}/bin
127         mkdir -p ${BUILDBASE}/tmp
128         mkdir -p ${BUILDBASE}/tmp/track
129         mkdir -p ${BUILDBASE}/tmp/logs
130         mkdir -p ${BUILDBASE}/tmp/logs/run
131         mkdir -p ${BUILDBASE}/tmp/logs/good
132         mkdir -p ${BUILDBASE}/tmp/logs/bad
133         mkdir -p ${BUILDBASE}/etc
134         mkdir -p ${BUILDBASE}/usr.pkg
135         mkdir -p ${BUILDBASE}/usr.pkgsrc
136         mkdir -p ${BUILDBASE}/usr.obj
137         mkdir -p ${BUILDBASE}/var.db
138         mkdir -p ${BUILDBASE}/var.db/pkg
139         mkdir -p ${BUILDBASE}/var.db/pkg.refcount
140         mkdir -p ${BUILDBASE}/var.db.pkg
141         mkdir -p ${BUILDBASE}/var.db.pkg.refcount
142         mkdir -p ${BUILDBASE}/track
143         #
144         # Copy stuff we need, make sure the pkgsrc copy we
145         # make has the needed mount points for packages and distfiles.
146         #
147         cpdup -s0 -i0 ${SYSROOT}/etc ${BUILDBASE}/etc
148         cpdup -s0 -i0 -X ${.CURDIR}/vardb.ign \
149                         ${SYSROOT}/var/db ${BUILDBASE}/var.db
150         cpdup -s0 -i0 -X ${.CURDIR}/pkgsrc.ign \
151                         ${SYSPKGSRC} ${BUILDBASE}/usr.pkgsrc
152         mkdir -p ${BUILDBASE}/usr.pkgsrc/packages
153         mkdir -p ${BUILDBASE}/usr.pkgsrc/distfiles
154         mkdir -p ${BUILDROOT}
155         #find -x -X ${BUILDROOT}/usr/pkgsrc -name work | xargs rm -rf
156
157 # quickstrap - get ready to start a fastbulk build
158 #
159 # NOTE: ${BUILDROOT}/usr/obj might be mounted from a tmpfs and not
160 #       from ${BUILDBASE}/usr.obj
161 #
162 quickstrap: prestage
163         rm -rf ${BUILDBASE}/usr.obj/work
164         rm -rf ${BUILDROOT}/usr/obj/work
165         rm -rf ${BUILDROOT}/tmp/logs/run
166         rm -rf ${BUILDROOT}/tmp/logs/bad
167         mkdir -p ${BUILDROOT}/tmp/logs/run
168         mkdir -p ${BUILDROOT}/tmp/logs/bad
169         mkdir -p ${BUILDROOT}/usr/obj/work
170         #
171         # Sometimes we get blah.core files in /var/db/pkg, which
172         # interferes with pkg_add.  Make sure they are gone.
173         #
174         find ${BUILDROOT}/var/db/pkg -name '*.core' -delete
175         find ${BUILDROOT}/var/db/pkg -type d -depth 1 -delete
176
177 bootstrap: quickstrap
178         -resident -d ${BUILDROOT}/usr/pkg/bin/bmake
179         cpdup -i0 -s0 -o ${BUILDBASE}/packages ${BUILDBASE}/packages.save
180         find -x ${BUILDBASE}/packages -depth -depth +0 -delete
181         find -x ${BUILDBASE}/usr.pkgsrc/packages -depth -depth +0 -delete
182         find -x ${BUILDBASE}/var.db.pkg -depth -depth +0 -delete
183         find -x ${BUILDBASE}/var.db.pkg.refcount -depth -depth +0 -delete
184         find -x ${BUILDBASE}/usr.pkg -depth -depth +0 -delete
185         # note: work dir might be either tmpfs or null mount
186         #${CHROOTENV} "pkg_admin -K /var/db/pkg fetch-pkg-vulnerabilities"
187         #${CHROOTENV} "pkg_admin rebuild"
188         rm -rf ${BUILDROOT}/usr/obj/work
189         ${CHROOTENV} "cd /usr/pkgsrc/bootstrap; ./bootstrap --workdir=/usr/obj/work"
190         -resident ${BUILDROOT}/usr/pkg/bin/bmake > /dev/null 2>&1
191         cat mk.conf.add >> ${BUILDROOT}/usr/pkg/etc/mk.conf
192
193 # Figure out what has changed and deinstall the related packages
194 #
195
196 PRESTAGE= ${BUILDBASE}/bin/getpkgsrcdir
197 PRESTAGE+= ${BUILDBASE}/bin/fastbulk
198 PRESTAGE+= ${BUILDBASE}/track/Makefile
199 PRESTAGE+= ${BUILDBASE}/track/dochanges
200 PRESTAGE+= ${BUILDBASE}/track/dobuild
201 PRESTAGE+= ${BUILDBASE}/track/fastbulk
202
203 prestage: ${PRESTAGE}
204
205 changes: ${PRESTAGE}
206         @if [ ! -f ${BUILDBASE}/track/dochanges ]; then \
207                 touch ${BUILDBASE}/track/dochanges; fi
208         @if [ ! -f ${BUILDBASE}/track/changes.raw ]; then \
209                 touch ${BUILDBASE}/track/changes.raw; fi
210         ( cat ${BUILDBASE}/track/changes.raw; \
211           find ${SYSPKGSRC} -mnewer ${BUILDBASE}/track/dochanges ) | \
212             sort | uniq | ${BUILDBASE}/bin/getpkgsrcdir ${SYSPKGSRC} | \
213             sort | uniq > ${BUILDBASE}/track/changes.raw.new
214         mv ${BUILDBASE}/track/changes.raw.new ${BUILDBASE}/track/changes.raw
215         ${BUILDBASE}/bin/getpkgsrcdir -s ${SYSPKGSRC} < \
216                 ${BUILDBASE}/track/changes.raw > \
217                 ${BUILDBASE}/track/changes.txt
218         cp Makefile ${BUILDBASE}/track/Makefile
219         ${CHROOTENV} "cd /tmp/track; ./dochanges"
220
221
222 ${BUILDBASE}/bin/getpkgsrcdir: getpkgsrcdir.c
223         cc getpkgsrcdir.c -o ${BUILDBASE}/bin/getpkgsrcdir
224
225 ${BUILDBASE}/bin/fastbulk: fastbulk.c
226         cc fastbulk.c -o ${BUILDBASE}/bin/fastbulk
227
228 ${BUILDBASE}/track/Makefile: ${.CURDIR}/Makefile
229         cp -p ${.CURDIR}/Makefile ${BUILDBASE}/track/Makefile
230
231 ${BUILDBASE}/track/dochanges: ${.CURDIR}/dochanges
232         cp -p ${.CURDIR}/dochanges ${BUILDBASE}/track/dochanges
233
234 ${BUILDBASE}/track/dobuild: ${.CURDIR}/dobuild
235         cp -p ${.CURDIR}/dobuild ${BUILDBASE}/track/dobuild
236
237 ${BUILDBASE}/track/fastbulk: ${BUILDBASE}/bin/fastbulk
238         cp -p ${BUILDBASE}/bin/fastbulk ${BUILDBASE}/track/fastbulk