3 # $NetBSD: pkg_rolling-replace.sh,v 1.31 2011/03/22 04:04:58 obache Exp $
5 # Copyright (c) 2006 BBN Technologies Corp. All rights reserved.
7 # Redistribution and use in source and binary forms, with or without
8 # modification, are permitted provided that the following conditions
10 # 1. Redistributions of source code must retain the above copyright
11 # notice, this list of conditions and the following disclaimer.
12 # 2. Redistributions in binary form must reproduce the above copyright
13 # notice, this list of conditions and the following disclaimer in the
14 # documentation and/or other materials provided with the distribution.
15 # 3. Neither the name of BBN Technologies nor the names of its contributors
16 # may be used to endorse or promote products derived from this software
17 # without specific prior written permission.
19 # THIS SOFTWARE IS PROVIDED BY BBN TECHNOLOGIES AND CONTRIBUTORS ``AS IS''
20 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 # ARE DISCLAIMED. IN NO EVENT SHALL BBN TECHNOLOGIES OR CONTRIBUTORS BE
23 # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 # POSSIBILITY OF SUCH DAMAGE.
31 # Effort sponsored in part by the Defense Advanced Research Projects
32 # Agency (DARPA) and the Department of the Interior National Business
33 # Center under agreement number NBCHC050166.
36 # REVIEW NOTATION LEGEND
37 # XXX marks things that are broken/kludgy
38 # TODO marks things that need to be done
39 # FIXED marks items fixed, needs review and deletion of FIXED comment
41 # Items to be completed for initial release:
43 # TODO: move bulk of help into a real man page.
45 # Items for future work (some imply changes in pkgsrc, not necessarily here)
47 # Make a pass over all packages for build depends, and rememeber
48 # which packages have been so checked. Check newly unsafe packages.
49 # The current lazy evaluation can perhaps fail to do things in the
50 # right order. Alternatively, explain why the current scheme is safe
53 # Consider expansion to update via binary packages. This raises
54 # interesting questions about ABIs and binary builds
56 # Deal with conflicts in a smarter way, perhaps more automatically.
57 # This is hard because just removing packages is not ok.
59 # Handle foo being split into foo and foo-share. Now, make replace
60 # of foo pulls in foo-share which conflicts with foo. This needlessly
64 # Substituted by pkgsrc at pre-configure time.
68 if [ -z "$MAKECONF" ] ; then
69 for mkconf in "@MAKECONF@" "@PREFIX@/etc/mk.conf" /etc/mk.conf ; do
70 if [ -f "$mkconf" ] ; then
76 if [ -z "$MAKECONF" -o ! -f "$MAKECONF" ] ; then
79 test -f "$MAKECONF" && test -z "$PKGSRCDIR" && PKGSRCDIR="` \
80 printf '.include "%s"\n_print_pkgsrcdir:\n\t@echo "${PKGSRCDIR}"\n' \
81 "$MAKECONF" | "$MAKE" -f - BSD_PKG_MK=1 _print_pkgsrcdir`"
82 if [ -z "$PKGSRCDIR" ] ; then
83 for dir in . .. ../.. /usr/pkgsrc ; do
84 if [ -f "${dir}/mk/bsd.pkg.mk" ]; then
86 /*) PKGSRCDIR="${dir}" ;;
87 *) PKGSRCDIR="$( cd "${dir}" >/dev/null 2>&1 && pwd )" ;;
93 test -z "$PKGSRCDIR" && echo >&2 "Please set PKGSRCDIR" && exit 1
94 test -z "$PKG_CHK" && PKG_CHK="@PKG_CHK@"
95 test -z "$PKG_INFO" && PKG_INFO="@PKG_INFO_CMD@"
99 unset PKG_PATH || true #or pkgsrc makefiles will complain
103 echo "Usage: pkg_rolling-replace [opts]
105 -F Fetch sources (including depends) only, don't build
106 -k Keep running, even on error
107 -n Don't actually do make replace
108 -r Just replace, don't create binary packages
109 -s Replace even if the ABIs are still compatible ("strict")
110 -u Update outdated packages
112 -D VAR=VAL Passes given variables and values to make
113 -L <path> Log to path (<path>/pkgdir/pkg)
114 -X <pkg> exclude <pkg> from being rebuilt
115 -x <pkg> exclude <pkg> from outdated check
117 pkg_rolling-replace does 'make replace' on one package at a time,
118 tsorting the packages being replaced according to their
119 interdependencies, which avoids most duplicate rebuilds.
121 pkg_rolling-replace can be used in one of two ways:
123 - 'make replace' is unsafe in that, if the replaced package's ABI
124 changes, its dependent packages may break. If this happens, run
125 'pkg_rolling-replace' (no arguments) to rebuild them against the
128 - 'pkg_chk -u' will delete all your mismatched (outdated)
129 packages, then reinstall them one at a time, leaving you without
130 those packages in the meantime. 'pkg_rolling-replace -u' will
131 instead upgrade them in place, allowing you to keep using your
132 system in the meantime (maybe...if you're lucky...because
133 pkg_rolling-replace replaces the \"deepest\" dependency first,
134 things could still break if that happens to be a fundamental
135 library whose ABI has changed).
140 # pkg_rolling-replace's output is intermingled with pkgsrc make
141 # output. Thus, we use a different prefix. pkgsrc uses '===>' for
142 # major steps, and '=>' for minor ones. Thus, we use 'RR>'.
145 OPC='rr>' # continuation
148 # Echo the names of packages needing updates, versions stripped. This
149 # has been tested with pkg_chk 1.76. Older versions are not
150 # supported. Newer versions may or may not work (patches welcome).
151 check_packages_mismatched()
153 ${PKG_CHK} -u -q | while read line; do
154 # duplicate output of pkg_chk to stderr (bypass $(...) or `...`)
155 echo "${OPC} $line" 1>&2
156 # Look for the first thing that looks like pkg-version rather
157 # than category/pkg and remove the version.
158 for word in $line; do
159 if [ "$(echo $word | egrep '^[^/]+-[0-9][^-/]*$')" ]; then
160 echo $word | sed 's/-[0-9][^-]*$//'
161 break #done with this line
167 # usage: check_packages_w_flag flag
168 # echo packages with flag=YES
169 check_packages_w_flag()
172 for pkgver in $(${PKG_INFO} -e '*'); do
173 if ${PKG_INFO} -Bq $pkgver \
174 | egrep "^$_flag=[Yy][Ee][Ss]" > /dev/null; then
175 echo $pkgver | sed 's/-[0-9][^-]*$//'
180 # echo dep->pkg edges for all installed packages
183 ${PKG_INFO} -N '*' | ${AWK} ' \
184 /^Information for/ { \
185 pkg=$3; sub("-[0-9][^-]*:$", "", pkg); \
189 /^\** PACKAGE MAY NOT BE DELETED \**$/ { state=3; } \
192 dep=$1; sub("-[0-9][^-]*$", "", dep); \
202 # usage: who_requires pkg --in-graph DEPGRAPH
206 test "$1" = '--in-graph' && shift || return $((false))
208 while [ $# -gt 0 ]; do
209 _dep=$1; _pkg=$2; shift 2;
210 if [ $_dep = $_target -a $_dep != $_pkg ]; then
216 # usage: is_member x list
217 # return true (0) if x appears in list, false (1) otherwise
221 while [ $# -gt 0 -a "$1" != "$_x" ]; do
228 # usage: exclude list1 --from list2
229 # echo list2 with items from list1 excluded
233 while [ $# -gt 0 -a $1 != "--from" ]; do
237 if [ $# -gt 0 ]; then
238 shift #discard '--from'
240 while [ $# -gt 0 ]; do
241 is_member $1 $_excl || echo $1
246 # usage: uniquify list
247 # just does sort -u on list (equiv. to sort | uniq)
248 # CAUTION: reorders list
251 for _x in $*; do echo $_x; done | sort -u
256 if [ -n "$opt_v" ]; then
263 if [ -n "$opt_v" ]; then
270 for a in $SUCCEEDED; do
282 echo "*** Please read the errors listed above, fix the problem,"
283 echo "*** then re-run pkg_rolling-replace to continue."
290 if [ -z "$opt_k" ]; then
298 SUCCEEDED=$(uniqify $SUCCEEDED $1)
303 FAILED=$(uniqify $FAILED $1)
308 if [ -z "$opt_F" ]; then
309 verbose "${OPI} Packages to rebuild:"
310 verbose "${OPC} MISMATCH_TODO=[$(echo $MISMATCH_TODO)]" #strip newlines
311 verbose "${OPC} REBUILD_TODO=[$(echo $REBUILD_TODO)]"
312 verbose "${OPC} UNSAFE_TODO=[$(echo $UNSAFE_TODO)]"
313 REPLACE_TODO=$(uniqify $MISMATCH_TODO $REBUILD_TODO $UNSAFE_TODO)
315 REPLACE_TODO=$(uniqify $MISMATCH_TODO)
316 verbose "${OPI} Packages to fetch:"
317 verbose "${OPC} MISMATCH_TODO=[$(echo $MISMATCH_TODO)]" #strip newlines
319 REPLACE_TODO=$(exclude $REALLYEXCLUDE --from $REPLACE_TODO)
320 if [ -n "$FAILED" ]; then
321 REPLACE_TODO=$(exclude $FAILED --from $REPLACE_TODO)
326 ######################################################################
332 MAKE_VAR="IN_PKG_ROLLING_REPLACE=1"
335 args=$(getopt FhknursvD:x:X:L: $*)
336 if [ $? -ne 0 ]; then
340 while [ $# -gt 0 ]; do
350 -D) MAKE_VAR="${MAKE_VAR}${MAKE_VAR_SEP}$2"; MAKE_VAR_SEP=" "; shift ;;
351 -x) EXCLUDE="$EXCLUDE $(echo $2 | sed 's/,/ /g')" ; shift ;;
352 -X) REALLYEXCLUDE="$REALLYEXCLUDE $(echo $2 | sed 's/,/ /g')" ; shift ;;
353 -L) LOGPATH="$2"; shift ;;
359 if [ -n "$opt_h" ]; then
363 if [ -n "$opt_s" ]; then
364 UNSAFE_VAR=unsafe_depends_strict
366 UNSAFE_VAR=unsafe_depends
369 MAKE="${MAKE}${MAKE_VAR_SEP}${MAKE_VAR}"
375 if [ -n "$opt_u" -o -n "$opt_F" ]; then
376 echo "${OPI} Checking for mismatched installed packages using pkg_chk"
377 MISMATCH_TODO=$(check_packages_mismatched)
378 echo "${OPI} Excluding the following mismatched packages:"
379 echo "${OPC} EXCLUDE=[$EXCLUDE]"
380 MISMATCH_TODO=$(exclude $EXCLUDE --from $MISMATCH_TODO)
383 if [ -z "$opt_F" ]; then
384 echo "${OPI} Checking for rebuild-requested installed packages (rebuild=YES)"
385 REBUILD_TODO=$(check_packages_w_flag 'rebuild')
387 echo "${OPI} Checking for unsafe installed packages (${UNSAFE_VAR}=YES)"
388 UNSAFE_TODO=$(check_packages_w_flag ${UNSAFE_VAR})
391 # DEPGRAPH_INSTALLED is rebuilt each round. DEPGRAPH_SRC will collect
392 # edges that we discover using 'make show-depends', but that weren't
393 # listed as depends by the installed version of a package, and
394 # DEPENDS_CHECKED lists packages for which we've already done that
404 while [ -n "$REPLACE_TODO" ]; do
407 # don't rebuild depgraph if we continued from new-depends step below
408 if [ "$depgraph_built" -eq 0 ]; then
409 echo "${OPI} Building dependency graph for installed packages"
410 DEPGRAPH_INSTALLED=$(depgraph_installed)
414 echo "${OPI} Tsorting dependency graph"
415 TSORTED=$(echo $DEPGRAPH_INSTALLED $DEPGRAPH_SRC | tsort)
417 for pkg in $TSORTED; do
418 if is_member $pkg $REPLACE_TODO; then
419 pkgdir=$(${PKG_INFO} -Q PKGPATH $pkg)
420 [ -n "$pkgdir" ] || abort "Couldn't extract PKGPATH from installed package $pkg"
424 # loop should never exit without selecting a package
425 [ -n "$pkgdir" ] || abort "pkg_chk reports the following packages need replacing, but they are not installed: $REPLACE_TODO"
427 echo "${OPI} Selecting $pkg ($pkgdir) as next package to replace"
430 # Newer versions in pkgsrc sometimes have more depends than are
431 # recorded for the installed version (this is entirely to be
432 # expected as packages are split, renamed, and add new features).
433 # When this happens, add the new edges to the graph and re-tsort.
434 # Don't re-tsort if the new edges were already installed once
435 # (otherwise we'll go into a loop).
437 # XXX I'm not sure that DEPENDS_CHECKED is necessary, or whether
438 # simply ignoring new deps that aren't in $REPLACE_TODO would have
439 # been sufficient. The DEPENDS_CHECKED approach causes one extra
440 # tsort in some cases, which isn't overly expensive.
442 # XXX After tsorting and choosing what to 'make replace', we
443 # filter out packages that aren't already installed and marked for
444 # replacement. This is safe, since uninstalled depends will be
445 # pulled in automatically by 'make replace'; but in rare cases it
446 # might be non-optimal, leading to a duplicate rebuild that we
447 # would have avoided had we selected the uninstalled package in
448 # tsorted order and run 'make install' on it. This seems like
449 # such a rare case that the added complexity isn't worth it.
451 # Set PKGNAME_REQD to give underlying make processes a chance to
452 # set options derived from the package name. For example,
453 # the appropriate version of Python can be derived from the
454 # package name (so, when building py25-foo, use python-2.5,
456 MAKE_SET_VARS="PKGNAME_REQD=${pkg}-*"
458 if ! is_member $pkg $DEPENDS_CHECKED; then
459 echo "${OPI} Checking if $pkg has new depends..."
460 OLD_DEPENDS=$(${PKG_INFO} -Nq $pkg | sed 's/-[0-9][^-]*$//')
462 cd "$PKGSRCDIR/$pkgdir"
463 bdeps=$(@SETENV@ ${MAKE_SET_VARS} ${MAKE} show-depends VARNAME=BUILD_DEPENDS)
464 rdeps=$(@SETENV@ ${MAKE_SET_VARS} ${MAKE} show-depends)
465 for depver in $bdeps $rdeps; do
466 dep=$(echo $depver | sed -e 's/[:[].*$/0/' -e 's/[<>]=/-/' \
467 -e 's/-[0-9][^-]*$//')
468 if ! is_member $dep $OLD_DEPENDS $NEW_DEPENDS; then
469 NEW_DEPENDS="$NEW_DEPENDS $dep"
470 DEPGRAPH_SRC="$DEPGRAPH_SRC $dep $pkg"
473 DEPENDS_CHECKED="$DEPENDS_CHECKED $pkg"
474 if [ -n "$NEW_DEPENDS" ]; then
475 echo "${OPI} $pkg has the following new depends (need to re-tsort):"
476 echo "${OPC} [$(echo $NEW_DEPENDS)]"
482 pkgname=$(${PKG_INFO} -e $pkg)
483 if [ -n "$LOGPATH" ]; then
484 logdir="$LOGPATH/`dirname $pkgdir`"
485 logfile="$logdir/$pkgname"
486 @MKDIR@ "$logdir" || abort "mkdir -p '$logdir' failed"
492 # Do make replace, with clean before, and package and clean afterwards.
494 if [ -d "$PKGSRCDIR/$pkgdir" ]; then
495 cd "$PKGSRCDIR/$pkgdir";
498 error "No package directory '$pkgdir' for $pkg."
501 if [ -z "$fail" ]; then
502 if [ -z "$opt_F" ]; then
503 echo "${OPI} Replacing $pkgname"
504 cmd="@SETENV@ ${MAKE_SET_VARS} ${MAKE} clean || fail=1"
505 if [ -n "$opt_n" ]; then
506 echo "${OPI} Would run: $cmd"
508 if [ -n "$logfile" ]; then
513 if [ -n "$fail" ]; then
515 error "'make clean' failed for package $pkg."
518 cmd="@SETENV@ ${MAKE_SET_VARS} ${MAKE} replace || fail=1" # XXX OLDNAME= support? xmlrpc-c -> xmlrpc-c-ss
520 echo "${OPI} Fetching $pkgname"
521 cmd="@SETENV@ ${MAKE_SET_VARS} ${MAKE} fetch depends-fetch || fail=1"
525 if [ -n "$opt_n" -a -z "$fail" ]; then
526 echo "${OPI} Would run: $cmd"
527 elif [ -z "$fail" ]; then
528 if [ -n "$logfile" ]; then
533 if [ -n "$fail" ]; then
535 error "'make replace' failed for package $pkg."
538 if [ -z "$opt_n" -a -z "$opt_k" -a -z "$opt_F" ]; then
539 [ -z "$(${PKG_INFO} -Q unsafe_depends_strict $pkg)" ] || \
540 abort "package $pkg still has unsafe_depends_strict."
541 [ -z "$(${PKG_INFO} -Q unsafe_depends $pkg)" ] || \
542 abort "package $pkg still has unsafe_depends."
543 [ -z "$(${PKG_INFO} -Q rebuild $pkg)" ] || \
544 abort "package $pkg is still requested to be rebuilt."
546 # If -r not given, make a binary package.
547 if [ -z "$opt_r" -a -z "$fail" -a -z "$opt_F" ]; then
548 echo "${OPI} Packaging $(${PKG_INFO} -e $pkg)"
549 cmd="@SETENV@ ${MAKE_SET_VARS} ${MAKE} package || fail=1"
550 if [ -n "$opt_n" -a -z "$fail" ]; then
551 echo "${OPI} Would run: $cmd"
552 elif [ -z "$fail" ]; then
553 if [ -n "$logfile" ]; then
558 if [ -n "$fail" ]; then
560 error "'make package' failed for package $pkg."
565 if [ -z "$opt_n" -a -z "$fail" -a -z "$opt_F" ]; then
566 cmd="@SETENV@ ${MAKE_SET_VARS} ${MAKE} clean || fail=1"
567 if [ -n "$logfile" ]; then
572 if [ -n "$fail" ]; then
574 error "'make clean' failed for package $pkg."
578 if [ -z "$fail" ]; then
579 mark_as_succeeded $pkg
584 # remove just-replaced package from all *_TODO lists
585 MISMATCH_TODO=$(exclude $pkg --from $MISMATCH_TODO)
586 REBUILD_TODO=$(exclude $pkg --from $REBUILD_TODO)
587 UNSAFE_TODO=$(exclude $pkg --from $UNSAFE_TODO)
589 echo "${OPI} Re-checking for unsafe installed packages (${UNSAFE_VAR}=YES)"
590 if [ -n "$opt_n" ]; then
591 # With -n, the replace didn't happen, and thus the packages that would
592 # have been marked unsafe_depends=YES were not. Add the set that
593 # would have been marked so we can watch what pkg_rolling-replace
594 # would have done (approximately).
595 UNSAFE_TODO=$(uniqify $UNSAFE_TODO \
596 $(who_requires $pkg --in-graph $DEPGRAPH_INSTALLED))
599 UNSAFE_TODO=$(check_packages_w_flag ${UNSAFE_VAR})
607 if [ -n "$logfile" ]; then
613 echo "${OPI} No more packages to replace; done."