mk/scripts/binpkg-cache

   1 #!/bin/sh
   2 #
   3 # $NetBSD: binpkg-cache,v 1.16 2007/10/13 17:29:26 rillig Exp $
   4 #
   5 # Script for generating a cache file with information about
   6 # all binary packages contained in a directory.
   7 #
   8 # Copyright (c) 2005, 2006 The NetBSD Foundation, Inc.
   9 # All rights reserved.
  10 #
  11 # This code is derived from software contributed to The NetBSD Foundation
  12 # by Dan McMahill.
  13 #
  14 # Redistribution and use in source and binary forms, with or without
  15 # modification, are permitted provided that the following conditions
  16 # are met:
  17 # 1. Redistributions of source code must retain the above copyright
  18 #    notice, this list of conditions and the following disclaimer.
  19 # 2. Redistributions in binary form must reproduce the above copyright
  20 #    notice, this list of conditions and the following disclaimer in the
  21 #    documentation and/or other materials provided with the distribution.
  22 # 3. All advertising materials mentioning features or use of this software
  23 #    must display the following acknowledgement:
  24 #        This product includes software developed by the NetBSD
  25 #        Foundation, Inc. and its contributors.
  26 # 4. Neither the name of The NetBSD Foundation nor the names of its
  27 #    contributors may be used to endorse or promote products derived
  28 #    from this software without specific prior written permission.
  29 #
  30 # THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  31 # ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  32 # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  33 # PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  34 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  35 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  36 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  37 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  38 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  39 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  40 # POSSIBILITY OF SUCH DAMAGE.
  41 #
  42
  43
  44 TMPDIR=${TMPDIR:-/tmp}
  45 PACKAGES=${PACKAGES:-/usr/pkgsrc/packages/}
  46 AWK=${AWK:-awk}
  47 CMP=${CMP:-cmp}
  48 FIND=${FIND:-find}
  49 GREP=${GREP:-grep}
  50 GZIP_CMD=${GZIP_CMD:-gzip}
  51 PKG_INFO=${PKG_INFO:-pkg_info}
  52 PKG_SUFX=${PKG_SUFX:-.tgz}
  53 SED=${SED:-sed}
  54 SORT=${SORT:-sort}
  55 STAT=${STAT:-stat}
  56
  57 cachefile=.pkgcache
  58 summaryfile=pkg_summary
  59 cacheversion=20050428
  60
  61 prompt="----> "
  62 tab="      "
  63
  64 tmpd=${TMPDIR}/pkg-cache.$$
  65 mkdir -m 0700 ${tmpd}
  66 if test $? -ne 0 ; then
  67         echo "ERROR:  Could not create temporary directory ${tmpd}"
  68         echo "Either you do not have write permission to ${tmpd} or"
  69         echo "${tmpd} already exists"
  70         exit 1
  71 fi
  72 all_dirs=${tmpd}/all_dirs
  73
  74 prog=$0
  75
  76 usage(){
  77         echo "$prog - Generates cache files for each directory containing binary"
  78         echo "        packages.  This cache file can then be used by the README.html"
  79         echo "        generation code to avoid having to call pkg_info(1) over and over"
  80         echo "        on the same binary package.  In addition, pkg_summary.gz files for"
  81         echo "        use by the pkgtools/pkg_chk pacakge may be generated."
  82         echo " "
  83         echo "Usage:      $prog [-d|--debug] [-v|--verbose] [-s|--summary] [-p|--packages <dir>]"
  84         echo " "
  85         echo "            $prog -h|--help"
  86         echo " "
  87         echo "            $prog -V|--version"
  88         echo " "
  89         echo "The options supported by $prog are: "
  90         echo " "
  91         echo "  -d|--debug            Enables debugging output"
  92         echo " "
  93         echo "  -F|--force            Forces a rebuild of the cache files even if they are"
  94         echo "                        otherwise up to date."
  95         echo " "
  96         echo "  -h|--help             Displays this help message"
  97         echo " "
  98         echo "  -p|--packages <dir>   Specifies the top level directory to be searched"
  99         echo "                        for binary packages."
 100         echo " "
 101         echo "  -s|--summary          Enables the creation of pkg_summary.gz files in each"
 102         echo "                        directory containing binary packages.  The pkg_summary.gz"
 103         echo "                        file is used by the pkgtools/pkg_chk package."
 104         echo " "
 105         echo "  -v|--verbose          Enables verbose output from the script."
 106         echo " "
 107         echo "  -V|--version          Displays the version of this script and exits."
 108         echo " "
 109         echo "Returns 0 on success, 1 on errors, 2 if the packages"
 110         echo "directory does not exist."
 111         echo " "
 112         echo "Example:    $prog -v --packages /usr/pkgsrc/packages"
 113         echo " "
 114 }
 115
 116 clean_and_exit0(){
 117         rm -fr ${tmpd}
 118         exit 0
 119 }
 120
 121 clean_and_exit1(){
 122         rm -fr ${tmpd}
 123         exit 1
 124 }
 125
 126 clean_and_exit2(){
 127         rm -fr ${tmpd}
 128         exit 2
 129 }
 130
 131 all_cache_files=""
 132
 133 #############################################################################
 134 #
 135 # process_binpkg_dir()
 136 #
 137 # Process a directory by checking to see if a cache file exists.  If the
 138 # cache file exists, make sure it is up to date.  If the file does not
 139 # exist, create one.
 140 #
 141 # also keep track of this directory so it can be added to the master
 142 # cache.
 143 #
 144
 145 process_binpkg_dir(){
 146         rdir=`${GREP} "^${d} " ${all_dirs} | ${AWK} '{print $2}'`
 147         need_update=no
 148         if test -f ${d}/${cachefile} ; then
 149                 stale_entries=`${FIND} ${d} -type f -name \*${PKG_SUFX} -newer ${d}/${cachefile} -print`
 150
 151                 # FIX_ME
 152                 #
 153                 # We also should find cache entries for files which no longer exist
 154                 # and nuke them.  Right now we simply declare the entire cache out
 155                 # of date.  Once we implement incremental updates to the cache,
 156                 # we need to remove the entries but not mark the entire cache as
 157                 # bad.
 158                 $if_debug echo "      Checking for cache entries with no corresponding pkg."
 159                 # get the list of what pkgs belong in the cache
 160                 rm -f ${tmpd}/pkg_list ${tmpd}/cache_pkg_list
 161                 ${FIND} ${d}/ -name \*${PKG_SUFX} -print | \
 162                         ${SED} -e "s;^${d}/*;${rdir}/;g" -e 's;//;/;g' | \
 163                         ${SORT} > ${tmpd}/pkg_list
 164
 165                 # and get the list of what is in the cache
 166                 ${AWK} '/pkgcache_begin/ {gsub(/pkgcache_begin[ \t]*/, ""); print}' \
 167                         ${d}/${cachefile} | ${SORT} > ${tmpd}/cache_pkg_list
 168
 169                 if ${CMP} -s ${tmpd}/pkg_list ${tmpd}/cache_pkg_list ; then
 170                         $if_debug echo "      No extra cache entries in ${d}/${cachefile}"
 171                 else
 172                         $if_debug echo "Package list:"
 173                         $if_debug cat  ${tmpd}/pkg_list
 174                         $if_debug echo "Cache list:"
 175                         $if_debug cat ${tmpd}/cache_pkg_list
 176                         echo "      Entries found in ${d}/${cachefile} but no packages found"
 177                         need_update=yes
 178                 fi
 179         else
 180                 stale_entries=""
 181         fi
 182
 183         if test "X${force}" = "Xyes" -o "X${need_update}" = "Xyes" ; then
 184                 need_update=yes
 185                 echo "${tab}Forcing rebuild of cache ${d}/${cachefile}."
 186         elif test ! -f ${d}/${cachefile} ; then
 187                 need_update=yes
 188                 echo "${tab}Missing cache file.  ${d}/${cachefile} will be generated."
 189         elif test -n "${stale_entries}" ; then
 190                 need_update=yes
 191                 echo "${tab}Stale cache file.  ${d}/${cachefile} will be regenerated."
 192         else
 193                 ${GREP} "pkgcache_version ${cacheversion}" ${d}/${cachefile} >/dev/null 2>&1
 194                 if test $? -ne 0 ; then
 195                         need_update=yes
 196                         echo "${tab}Invalid version cache file.  ${d}/${cachefile} will be regenerated."
 197                         echo "Need version ${cacheversion} but the file has"
 198                         ${GREP} "^pkgcache_version " ${d}/${cachefile}
 199                 else
 200                         echo "${tab}Cache file ${d}/${cachefile} is up to date."
 201                 fi
 202         fi
 203
 204         if test "${build_summary}" = "yes" -a ! -f ${d}/${summaryfile}.gz ; then
 205                 echo "${tab}Summary file ${d}/${summaryfile}.gz is missing and will be created."
 206                 need_update=yes
 207         fi
 208         if test "${build_summary}" = "yes" -a ${d}/${summaryfile}.gz -ot ${d}/${cachefile} ; then
 209                 echo "${tab}Summary file ${d}/${summaryfile}.gz is out of date and will be regnerated."
 210                 need_update=yes
 211         fi
 212
 213         # FIX_ME
 214         # We should use stale_entries in a way where we only update the
 215         # cache file entries corresponding to these if we're rebuilding
 216         # due to stale entries.  That should save a good bit of time.
 217         #
 218         if test "X${need_update}" = "Xyes" ; then
 219                 echo "pkgcache_version ${cacheversion}" > ${tmpd}/${cachefile}
 220                 rm -f ${tmpd}/${summaryfile}
 221                 touch ${tmpd}/${summaryfile}
 222                 for f in ${d}/*${PKG_SUFX} ; do
 223                         fn=`grep "^${d} " ${all_dirs} | ${AWK} '{print $2}'`"/"`basename ${f}`
 224                         $if_debug echo "     Adding ${fn} (${f}) to the cache"
 225                         echo " " >> ${tmpd}/${cachefile}
 226                         # stat(1) needs to be added to the bootstrap kit
 227                         # first if we want to use it here
 228                         #eval $(${STAT} -s ${f} 2>/dev/null)
 229                         echo "pkgcache_begin ${fn}" >> ${tmpd}/${cachefile}
 230                         #echo "pkgcache_mtime=${st_mtime}" >> ${tmpd}/${cachefile}
 231
 232                         $if_debug echo "${PKG_INFO} -q -B ${f}"
 233                         ${PKG_INFO} -q -B ${f} >> ${tmpd}/${cachefile}
 234
 235                         if test "${build_summary}" = "yes" ; then
 236                                 $if_debug echo "${PKG_INFO} -X ${f}"
 237                                 ${PKG_INFO} -X ${f} >> ${tmpd}/${summaryfile}
 238                         fi
 239                         echo "pkgcache_end ${fn}" >> ${tmpd}/${cachefile}
 240                 done
 241                 mv -f ${tmpd}/${cachefile} ${d}/${cachefile}
 242                 if test $? -ne 0 ; then
 243                         echo "********** WARNING **********"
 244                         echo "move of ${tmpd}/${cachefile} to ${d}/${cachefile} failed!"
 245                         echo "Perhaps you do not have write permissions to ${d}?"
 246                         echo "This directory will be dropped from the master cache file."
 247                         echo "********** WARNING **********"
 248                         return
 249                 fi
 250
 251                 if test "${build_summary}" = "yes" ; then
 252                         cat ${tmpd}/${summaryfile} | ${GZIP_CMD} > ${d}/${summaryfile}.gz
 253                         if test $? -ne 0 ; then
 254                                 echo "********** WARNING **********"
 255                                 echo "${GZIP_CMD} of ${tmpd}/${summaryfile} to ${d}/${summaryfile}.gz failed!"
 256                                 echo "Perhaps you do not have write permissions to ${d}?"
 257                                 echo "********** WARNING **********"
 258                                 return
 259                         fi
 260                 fi
 261
 262         fi
 263
 264         # if we got here, then this directory should have a good cache file in
 265         # it and we should be able to add it to the master cache file
 266         all_cache_files="${all_cache_files} ${d}/${cachefile}"
 267
 268 }
 269
 270 process_cache_files(){
 271         echo "${prompt}Checking master cache file ${PACKAGES}/${cachefile}"
 272         echo "pkgcache_version ${cacheversion}" > ${tmpd}/${cachefile}
 273         if test -n "${all_cache_files}" ; then
 274                 for c in ${all_cache_files} ; do
 275                         echo "pkgcache_cachefile ${c}" >> ${tmpd}/${cachefile}
 276                 done
 277         fi
 278         if test ! -f ${PACKAGES}/${cachefile} ; then
 279                 echo "${tab}Creating master cache file ${PACKAGES}/${cachefile}"
 280                 mv -f ${tmpd}/${cachefile} ${PACKAGES}/${cachefile}
 281                 if test $? -ne 0 ; then
 282                         echo "********** ERROR **********"
 283                         echo "move of ${tmpd}/${cachefile} to ${PACKAGES}/${cachefile} failed!"
 284                         echo "Perhaps you do not have write permissions to ${PACKAGES}?"
 285                         echo "********** ERROR **********"
 286                         clean_and_exit1
 287                 fi
 288         elif ${CMP} -s ${tmpd}/${cachefile} ${PACKAGES}/${cachefile} ; then
 289                 echo "${tab}Master cache file ${PACKAGES}/${cachefile} is up to date"
 290         else
 291                 echo "${tab}Updating master cache file ${PACKAGES}/${cachefile}"
 292                 mv -f ${tmpd}/${cachefile} ${PACKAGES}/${cachefile}
 293                 if test $? -ne 0 ; then
 294                         echo "********** ERROR **********"
 295                         echo "move of ${tmpd}/${cachefile} to ${PACKAGES}/${cachefile} failed!"
 296                         echo "Perhaps you do not have write permissions to ${PACKAGES}?"
 297                         echo "********** ERROR **********"
 298                         clean_and_exit1
 299                 fi
 300         fi
 301 }
 302
 303 ######################################################################
 304 #
 305 #  Handle command line options
 306 #
 307 ######################################################################
 308
 309 if_debug=:              # either ":" or ""
 310 verbose=no
 311 force=no
 312 build_summary=no
 313
 314 while
 315         test -n "$1"
 316 do
 317         case "$1" in
 318
 319                 # Turn on debugging
 320                 -d|--debug)
 321                         if_debug=""
 322                         verbose=yes
 323                         shift
 324                         ;;
 325
 326                 # Force a rebuilde of the cache
 327                 -F|--force)
 328                         force=yes
 329                         shift
 330                         ;;
 331                 # Help
 332                 -h|--help)
 333                         usage
 334                         exit 0
 335                         ;;
 336
 337                 # Use the specified packages directory
 338                 -p|--packages)
 339                         PACKAGES=$2
 340                         shift 2
 341                         ;;
 342
 343                 # Build the pkg_summary.gz files
 344                 -s|--summary)
 345                         build_summary=yes
 346                         shift
 347                         ;;
 348
 349                 # Version
 350                 -V|--version)
 351                         ${AWK} '/^#[ \t]*\$NetBSD/ {gsub(/,v/,"",$3);printf("%s:  Version %s, %s\n",$3,$4,$5); exit 0;}' $prog
 352                         exit 0
 353                         ;;
 354
 355                 # Turn on verbose output, but not as noisy as debug
 356                 -v|--verbose)
 357                         verbose=yes
 358                         shift
 359                         ;;
 360
 361                 -*) echo "$prog:  ERROR:  $1 is not a valid option"
 362                         usage
 363                         clean_and_exit1
 364                         ;;
 365
 366                 *)
 367                         break
 368                         ;;
 369
 370         esac
 371 done
 372
 373 if test $# -ne 0 ; then
 374         echo "$0:  $* is invalid"
 375         usage
 376         clean_and_exit1
 377 fi
 378
 379 if test ! -d ${PACKAGES} ; then
 380         echo "Packages directory ${PACKAGES} seems to be missing"
 381         clean_and_exit2
 382 fi
 383
 384 # put a trailing / after ${PACKAGES} in case ${PACKAGES} is
 385 # a link.
 386
 387 # pass 1, we find all directories under PACKAGES.  Note that this
 388 # may contain some directories more than once depending on what sort
 389 # of soft links may be in place
 390 rm -f ${all_dirs}.tmp
 391 for d in `${FIND} ${PACKAGES}/ -type d -follow -print` ; do
 392         cname=`(cd ${d} && pwd -P)`
 393         rname=`echo ${d} | ${SED} "s;^${PACKAGES}/*;;g"`
 394         echo "${cname} ${rname}" >> ${all_dirs}.tmp
 395 done
 396 ${SORT} -u -k1,1 ${all_dirs}.tmp > ${all_dirs}
 397 $if_debug echo "Full directory list:"
 398 $if_debug cat ${all_dirs}.tmp
 399 $if_debug echo "Unique directory list:"
 400 $if_debug cat ${all_dirs}
 401
 402 for d in `${AWK} '{print $1}' ${all_dirs}` ; do
 403         $if_debug echo "${prompt}Processing directory ${d}"
 404         is_pkg_dir=no
 405         for f in ${d}/*${PKG_SUFX} ; do
 406                 if test -f "${f}" -a ! -h "${f}" ; then
 407                         ${PKG_INFO} ${f} >/dev/null 2>&1
 408                         if test $? -eq 0 ; then
 409                                 is_pkg_dir=yes
 410                                 break
 411                         fi
 412                 fi
 413         done
 414         if test "X${is_pkg_dir}" = "Xyes" ; then
 415                 echo "${prompt}Checking cache in ${d}"
 416                 process_binpkg_dir
 417         else
 418                 $if_debug echo "${prompt}no binary packages in ${d}"
 419         fi
 420
 421 done
 422
 423 process_cache_files
 424
 425 clean_and_exit0