mk/bulk/upload

   1 #!/bin/sh
   2 # $NetBSD: upload,v 1.43 2008/03/15 16:27:43 joerg Exp $
   3
   4 #
   5 # Upload non-restricted binary pkgs to ftp server
   6 #
   7
   8 AWK=${AWK:-/usr/bin/awk}
   9 PKG_ADMIN="pkg_admin"
  10 PKG_INFO="pkg_info"
  11
  12 set -eu
  13
  14 #
  15 # Find out where we are
  16 #
  17 scriptdir=`dirname "$0"`
  18 scriptdir=`cd "${scriptdir}" && pwd`
  19
  20 usage()
  21 {
  22 cat << EOF
  23
  24 $prog:  Uploads binary packages.
  25
  26 Usage:  $prog [-c|--config <file>] [-n|--no-upload] [-d|--debug] [-v|--verbose]
  27         $prog -h|--help
  28         $prog -V|--version
  29
  30     -c | --config <file>
  31                         Load the following configuration file instead of
  32                         the default one.
  33
  34     -d|--debug          Preserves some intermediate files to help
  35                         debug any problems.
  36
  37     -h|--help           Displays this help message and exits.
  38
  39     -n|--no-upload      Does all of the work except for the actual upload.
  40                         This option may be used to generate the upload script
  41                         along with the list of packages to be excluded.
  42
  43     -i|--no-install     Do not install the required packages; assume instead
  44                         that they are already available.
  45
  46     -v|--verbose        Enables verbose output.
  47
  48     -V|--version        Displays the version of this script and exits.
  49
  50 Example:
  51
  52     cd /usr/pkgsrc && $prog
  53
  54 Environment Variables:
  55
  56     AWK  --  If specified, sets the awk program to be used
  57
  58 Files:
  59
  60     pkgsrc/mk/bulk/build.conf
  61
  62 EOF
  63 }
  64
  65 # print out error message and exit 1
  66 die () {
  67         echo "$0: error:" 1>&2
  68         for i in "$@"; do
  69                 echo "    $i" 1>&2
  70         done
  71         exit 1
  72 }
  73
  74 prog=$0
  75 debug=no
  76 do_upload=yes
  77 do_install=yes
  78 verbose=no
  79 while test $# -gt 0; do
  80         case "$1" in
  81         -c|--config)
  82                 shift
  83                 BULK_BUILD_CONF=$1; shift
  84                 ;;
  85
  86         -d|--debug)
  87                 debug=yes
  88                 shift
  89                 ;;
  90
  91         -h|--help)
  92                 usage
  93                 exit 0
  94                 ;;
  95
  96         -i|--no-install)
  97                 do_install=no
  98                 shift
  99                 ;;
 100
 101         -n|--no-upload)
 102                 do_upload=no
 103                 shift
 104                 ;;
 105
 106         -v|--verbose)
 107                 verbose=yes
 108                 shift
 109                 ;;
 110
 111         -V|--version)
 112                 ${AWK} '/^#[ \t]*\$NetBSD/ {gsub(/,v/,"",$3);printf("%s:  Version %s, %s\n",$3,$4,$5); exit 0;}' $prog
 113                 exit 0
 114                 ;;
 115
 116         -*)
 117                 echo "$prog:  Unknown option:  $1" 1>&2
 118                 usage
 119                 exit 1
 120                 ;;
 121
 122         *)
 123                 break
 124                 ;;
 125         esac
 126 done
 127 if test "$#" -gt 0; then
 128         echo "$prog:  Unknown argument: $1" 1>&2
 129         usage
 130         exit 1
 131 fi
 132
 133 install_required()
 134 {
 135
 136         [ "$do_install" = "yes" ] || return 0
 137         pkg=$1
 138         if [ "${verbose}" = "yes" ]; then
 139                 echo "upload> Installing ${pkg}"
 140         fi
 141         (cd "$USR_PKGSRC" && cd "$pkg" && ${BMAKE} bulk-install) \
 142         || {
 143                 echo "upload> ERROR: Unable to install required package $pkg!" 1>&2
 144                 echo "        Bailing out -- you're on your own." 1>&2
 145                 exit 1
 146         }
 147 }
 148
 149 MD5="digest md5";
 150 SHA1="digest sha1";
 151
 152 _INFO_VER=`${PKG_INFO} -V`;
 153 REQUIRED_PACKAGES="pkgtools/lintpkgsrc net/rsync"
 154
 155 opsys=`uname -s`
 156 case "$opsys" in
 157 NetBSD) BMAKE=make;
 158         BSDSUM="sum -o 1";
 159         CKSUM="cksum";
 160         SYSVSUM="sum -o 2";
 161         ;;
 162 IRIX*)  BMAKE=bmake;
 163         BSDSUM="sum -r";
 164         CKSUM="cksum";
 165         SYSVSUM="sum";
 166         ;;
 167 *)      BMAKE=bmake;
 168         BSDSUM="";
 169         CKSUM="";
 170         SYSVSUM="";
 171         ;;
 172 esac
 173
 174 export BMAKE
 175
 176 # Bring in variables for bulk-install
 177 BATCH=1
 178 DEPENDS_TARGET=bulk-install
 179 export BATCH DEPENDS_TARGET
 180
 181 #
 182 # Find the configuration file.
 183 #
 184 BULK_BUILD_CONF="${BULK_BUILD_CONF-${scriptdir}/build.conf}"
 185 case $BULK_BUILD_CONF in
 186         /*) ;;
 187          *)     BULK_BUILD_CONF="${PWD}/${BULK_BUILD_CONF}"
 188 esac
 189
 190 #
 191 # Get the variables MAKECONF, RSYNC_DST, RSYNC_OPTS from the bulk build
 192 # configuration file.
 193 #
 194 { test -f "${BULK_BUILD_CONF}"          \
 195   && . "${BULK_BUILD_CONF}"             \
 196   && . "${scriptdir}/post-build-conf"   \
 197   && check_config_vars                  \
 198   && export_config_vars
 199 } || die "Cannot load config file ${BULK_BUILD_CONF}, aborting."
 200
 201 cd $USR_PKGSRC
 202
 203 if [ -z "$RSYNC_DST" ]; then
 204         echo "upload> ERROR: You must set the variable RSYNC_DST, see build.conf-example." 1>&2
 205         exit 1
 206 fi
 207
 208 #
 209 # Some temp files
 210 #
 211
 212 umask 022
 213 TMPDIR="${TMPDIR:-/tmp}"
 214 TMP="${TMPDIR}"/pkg_upload.$$
 215 (umask 077 && mkdir "${TMP}") \
 216 || {
 217         echo "upload> ERROR: cannot create temporary directory \"${TMP}\"." 1>&2
 218         exit 1
 219 }
 220
 221 vulnerable_packages="$TMP/vulnerable_packages"
 222 restricted_packages="$TMP/restricted_packages"
 223 old_packages="$TMP/old_packages"
 224 good_packages="$TMP/regular_packages"
 225 all_good_packages="$TMP/all_regular_packages"
 226
 227 upload_general="$TMP"/upload_general
 228 upload_vulnerable="$TMP"/upload_vulnerable
 229
 230 # May be different than $USR_PKGSRC:
 231 echo "upload> Running ${BMAKE} to get the pkgsrc variables"
 232 pkgsrcdir=`cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=_PKGSRCDIR`
 233 packages=`cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=PACKAGES`
 234 distdir=`cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=DISTDIR`
 235 gzip_cmd=`cd pkgtools/lintpkgsrc; ${BMAKE} show-var VARNAME=GZIP_CMD USE_TOOLS=gzip`
 236 pkg_info=`cd pkgtools/lintpkgsrc && ${BMAKE} show-var VARNAME=PKG_INFO`
 237 shell=`cd pkgtools/lintpkgsrc && ${BMAKE} show-var VARNAME=TOOLS_PLATFORM.sh`
 238
 239 # Pull in some pkgs needed
 240 for pkg in ${REQUIRED_PACKAGES}; do
 241         install_required $pkg
 242 done
 243
 244 echo "upload> Making sure vulnerability-list is up-to-date:"
 245 if [ -z "$UPDATE_VULNERABILITY_LIST" -o "$UPDATE_VULNERABILITY_LIST" = "yes" ]
 246 then
 247         _PKGVULNDIR=`audit-packages ${AUDIT_PACKAGES_FLAGS} -Q PKGVULNDIR`
 248         download-vulnerability-list ${DOWNLOAD_VULNERABILITY_LIST_FLAGS}
 249         if [ "x${_PKGVULNDIR}" != "x${distdir}" ]; then
 250                 cp ${_PKGVULNDIR}/pkg-vulnerabilities ${distdir}
 251         fi
 252         echo "        done."
 253 else
 254         echo "        (skipped)"
 255 fi
 256
 257 case $LINTPKGSRC_CACHE in
 258 yes|YES)
 259         lintpkgsrc_cache="-I `cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=LINTPKGSRC_DB`"
 260         ;;
 261 *)
 262         lintpkgsrc_cache=''
 263         ;;
 264 esac
 265
 266 echo "upload> Checking for out of date packages:"
 267 # -p  =  report old versions of packages
 268 lintpkgsrc $lintpkgsrc_cache -K $packages -P $pkgsrcdir -p > "${old_packages}.tmp"
 269 sed 's@'$packages'/@@' < "${old_packages}.tmp" > "$old_packages"
 270
 271 RSFLAGS="-vap --progress $RSYNC_OPTS"
 272
 273 failed=no
 274 cd $packages
 275
 276 echo "upload> Checking for restricted and vulnerable packages"
 277 (cd All && env PKG_INFO="${pkg_info}" OUTDIR="${TMP}" PKGVULNDIR="${distdir}" \
 278                AUDIT_PACKAGES_FLAGS="${AUDIT_PACKAGES_FLAGS}" \
 279                DOWNLOAD_VULNERABILITY_LIST_FLAGS="${DOWNLOAD_VULNERABILITY_LIST_FLAGS}" \
 280                ${shell} "${pkgsrcdir}/mk/bulk/sort-packages")
 281
 282 # Add the name of the package file, including all its symlinks to the
 283 # list of files to be uploaded.
 284 while read package; do
 285         ls -1 */"$package"
 286 done < "$good_packages" > "$all_good_packages"
 287
 288 if [ "${MKSUMS}" = "yes" -o "${MKSUMS}" = "YES" ]; then
 289
 290         echo "upload> Calculating checksum files..."
 291
 292         SUMFILES="BSDSUM CKSUM MD5 SHA1 SYSVSUM"
 293
 294         rm -f ${SUMFILES}
 295
 296         if [ "${SIGN_AS-}" != "" ]; then
 297                 install_required "security/gnupg"
 298                 for i in ${SUMFILES}; do
 299                         echo > $i
 300                         echo "This file is signed with ${SIGN_AS}'s PGP key." >> $i
 301                         echo >> $i
 302                 done
 303         fi
 304
 305         install_required "pkgtools/digest"
 306
 307         [ -z "${BSDSUM}" ] && BSDSUM="echo"
 308         [ -z "${CKSUM}" ] && CKSUM="echo"
 309         [ -z "${SYSVSUM}" ] && SYSVSUM="echo"
 310
 311         for pkg in `cat "${good_packages}" "${vulnerable_packages}"`; do
 312                 pkg="All/$pkg"
 313                 ${BSDSUM}       "$pkg" >> BSDSUM
 314                 ${CKSUM}        "$pkg" >> CKSUM
 315                 ${MD5}          "$pkg" >> MD5
 316                 ${SHA1}         "$pkg" >> SHA1
 317                 ${SYSVSUM}      "$pkg" >> SYSVSUM
 318         done
 319
 320         [ "${BSDSUM}" = "echo" ] && rm BSDSUM
 321         [ "${CKSUM}" = "echo" ] && rm CKSUM
 322         [ "${SYSVSUM}" = "echo" ] && rm SYSVSUM
 323
 324         if [ "${SIGN_AS-}" != "" ]; then
 325                 for i in ${SUMFILES}; do
 326                         if [ -s $i ]; then
 327                                 echo "upload> Signing $i"
 328                                 gpg --clearsign $i && rm $i
 329                         fi
 330                 done
 331         else
 332                 echo "upload> Checksum files not PGP-signed. Please do so manually!"
 333                 echo "        (Run 'gpg --clearsign' on all of them)"
 334         fi
 335 fi
 336
 337 if [ "${MKSUMMARY-}" = "yes" -o "${MKSUMMARY-}" = "YES" ]; then
 338         echo "upload> Creating summary file..."
 339         (cd "${packages}/All" \
 340                 && ls -t | grep '\.t[gb]z$' | while read n; do pkg_info -X "$n"; done) \
 341                 | ${gzip_cmd} > "${packages}"/All/pkg_summary.gz
 342 fi
 343
 344 cat <<EOF > "$upload_general"
 345 #! /bin/sh
 346 set -e
 347 cd "$packages"
 348 rsync $RSFLAGS --files-from="${all_good_packages}" --exclude-from="${old_packages}" . "$RSYNC_DST/"
 349 EOF
 350 chmod +x "$upload_general"
 351
 352 if [ "$do_upload" = "yes" ]; then
 353         echo "upload> Uploading non-vulnerable packages"
 354         ${shell} "$upload_general" \
 355         || {
 356                 echo "upload> ERROR: rsync failed.  To retry later, you can run $upload_general" 1>&2
 357                 failed=yes
 358         }
 359 else
 360         echo "upload> Skipping upload of non-vulnerable packages."
 361         echo "        Run \"$upload_general\" to upload them later."
 362 fi
 363
 364 cat <<EOF > "$upload_vulnerable"
 365 #! /bin/sh
 366 set -e
 367 cd "$packages/All"
 368 rsync $RSFLAGS --files-from="${vulnerable_packages}" --exclude-from="${old_packages}" . "$RSYNC_DST/vulnerable/"
 369 EOF
 370 chmod +x "$upload_vulnerable"
 371
 372 if [ "$do_upload" = "yes" ]; then
 373         echo "upload> Uploading vulnerable packages"
 374         ${shell} "$upload_vulnerable" \
 375         || {
 376                 echo "upload> ERROR: rsync failed.  To retry later, you can run $upload_vulnerable" 1>&2
 377                 failed=yes
 378         }
 379 else
 380         echo "upload> Skipping upload of vulnerable packages."
 381         echo "        Run \"$upload_vulnerable\" to upload them later."
 382 fi
 383
 384 # clean up temp files
 385 if [ "$failed,$debug,$do_upload" = "no,no,yes" ]; then
 386         rm -fr "$TMP"
 387 else
 388         echo "upload> Preserving temporary directory ${TMP}"
 389 fi
 390
 391 if [ "$failed" = "yes" ]; then
 392         exit 1
 393 fi