mk/scripts/mkreadme

   1 #!/bin/sh
   2 # $NetBSD: mkreadme,v 1.22 2008/03/15 16:27:43 joerg Exp $
   3 #
   4 # Script for README.html generation
   5 #
   6 # Copyright (c) 2002, 2005 The NetBSD Foundation, Inc.
   7 # All rights reserved.
   8 #
   9 # This code is derived from software contributed to The NetBSD Foundation
  10 # by Dan McMahill.
  11 #
  12 # Redistribution and use in source and binary forms, with or without
  13 # modification, are permitted provided that the following conditions
  14 # are met:
  15 # 1. Redistributions of source code must retain the above copyright
  16 #    notice, this list of conditions and the following disclaimer.
  17 # 2. Redistributions in binary form must reproduce the above copyright
  18 #    notice, this list of conditions and the following disclaimer in the
  19 #    documentation and/or other materials provided with the distribution.
  20 # 3. All advertising materials mentioning features or use of this software
  21 #    must display the following acknowledgement:
  22 #        This product includes software developed by the NetBSD
  23 #        Foundation, Inc. and its contributors.
  24 # 4. Neither the name of The NetBSD Foundation nor the names of its
  25 #    contributors may be used to endorse or promote products derived
  26 #    from this software without specific prior written permission.
  27 #
  28 # THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  29 # ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  30 # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  31 # PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  32 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  33 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  34 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  35 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  36 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  37 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  38 # POSSIBILITY OF SUCH DAMAGE.
  39 #
  40
  41
  42 TMPDIR=${TMPDIR:-/tmp/mkreadme}
  43 PKGSRCDIR=${PKGSRCDIR:-/usr/pkgsrc}
  44 AWK=${AWK:-/usr/bin/awk}
  45
  46 opsys=`uname -s`
  47 case "$opsys" in
  48         *BSD)
  49                 makeprog=make
  50                 ;;
  51
  52         *)
  53                 makeprog=bmake
  54                 ;;
  55 esac
  56
  57 BMAKE=${BMAKE:-${makeprog}}
  58
  59 usage(){
  60     echo "$prog - Generates README.html files for a pkgsrc tree"
  61     echo "Usage:      $prog [-c|--cdrom] [-C|--prune] [-d|--debug] [-f|--ftp] "
  62     echo "                  [-p|--pkgsrc directory] "
  63     echo "                  [-P|--packages directory] [-r|--restart] "
  64     echo "                  [-s|--summary]"
  65     echo "                  [-S|--save-database]"
  66     echo " "
  67     echo "            $prog -h|--help"
  68     echo " "
  69     echo "            $prog -v|--version"
  70     echo " "
  71     echo "The options supported by $prog are: "
  72     echo " "
  73     echo "  -C|--prune          Prune unused README.html files which may exist in"
  74     echo "                      pkg directories which have been removed from CVS."
  75     echo " "
  76     echo "  -c|--cdrom          Generates CD-ROM README.html files"
  77     echo " "
  78     echo "  -d|--debug          Enables (extremely verbose) debug output"
  79     echo " "
  80     echo "  -f|--ftp            Generates FTP README.html files"
  81     echo " "
  82     echo "  -h|--help           Displays this help message"
  83     echo " "
  84     echo "  -p|--pkgsrc dir     Specifies the pkgsrc directory.  Defaults to"
  85     echo "                      the value of the PKGSRCDIR environment variable"
  86     echo "                      if set or /usr/pkgsrc otherwise."
  87     echo " "
  88     echo "  -P|--packages dir   Specifies the packages directory."
  89     echo " "
  90     echo "  -r|--restart        Restart.  This option assumes that the database file"
  91     echo "                      from a previous run still exists and that the script"
  92     echo "                      should use that instead of recreating the database."
  93     echo " "
  94     echo "  -s|--summary        Generate pkg_summary.gz files for the binary packages"
  95     echo "                      directories while processing them."
  96     echo " "
  97     echo "  -S|--save-database  Does not delete the database file after the run."
  98     echo "                      This is useful for debugging or re-running this script"
  99     echo "                      with the -r option."
 100     echo " "
 101     echo "  -v|--version        Displays the version of this script and exits."
 102     echo " "
 103     echo "Example:    $prog -p /pub/NetBSD/packages/pkgsrc -P /pub/NetBSD/packages -f"
 104     echo " "
 105 }
 106
 107 clean_and_exit(){
 108     if [ "x$DEBUG" = "xno" -a "x$restart" = "xno" -a "x$save" = "xno" ]; then
 109         rm -fr $TMPDIR
 110     else
 111         echo "Debugging output left in $TMP"
 112     fi
 113     exit 1
 114 }
 115
 116 prog=$0
 117
 118
 119 ######################################################################
 120 #
 121 #  Handle command line options
 122 #
 123 ######################################################################
 124
 125 cmdline=$*
 126
 127 ftp_readme=no
 128 restart=no
 129 prune=no
 130 DEBUG=no
 131 save=no
 132 pv=default
 133
 134 summary=""
 135
 136 while
 137     test -n "$1"
 138 do
 139     case "$1"
 140     in
 141
 142     # We're generating README.html's for a CD-ROM
 143     -c|--cdrom)
 144         ftp_readme=no
 145         shift
 146         ;;
 147
 148     # Prune old README.html files from pkgs which no longer exist
 149     -C|--prune)
 150         prune=yes
 151         shift
 152         ;;
 153
 154     # Turn on debugging
 155     -d|--debug)
 156         DEBUG=yes
 157         shift
 158         ;;
 159
 160     # We're generating README.html's for a CD-ROM
 161     -f|--ftp)
 162         ftp_readme=yes
 163         shift
 164         ;;
 165
 166
 167     # Help
 168     -h|--help)
 169         usage
 170         exit 0
 171         ;;
 172
 173     # Specify pkgsrc directory
 174     -p|--pkgsrc)
 175         PKGSRCDIR=$2
 176         shift 2
 177         ;;
 178
 179     # Specify PACKAGES directory
 180     -P|--packages)
 181         PKGDIR=$2
 182         shift 2
 183         ;;
 184
 185     # Restart (ie, don't re-generate the database file)
 186     -r|--restart)
 187         restart=yes
 188         shift
 189         ;;
 190
 191     # Generate the pkg_summary.gz files
 192     -s|--summary)
 193         summary="--summary"
 194         shift
 195         ;;
 196
 197     # Save the database files
 198     -S|--save-database)
 199         save=yes
 200         shift
 201         ;;
 202
 203     # Version
 204     -v|--version)
 205         ${AWK} '/^#[ \t]*\$NetBSD/ {gsub(/,v/,"",$3);printf("%s:  Version %s, %s\n",$3,$4,$5); exit 0;}' $prog
 206         exit 0
 207         ;;
 208
 209     -*) echo "$prog:  ERROR:  $1 is not a valid option"
 210         usage
 211         clean_and_exit
 212         ;;
 213
 214     *)
 215         break
 216         ;;
 217
 218     esac
 219 done
 220
 221 if [ "x$DEBUG" = "xyes" ]; then
 222     set -v
 223 fi
 224
 225 if [ ! -d ${PKGSRCDIR} ]; then
 226     echo "ERROR:  package source directory ${PKGSRCDIR} does not exist"
 227     echo ""
 228     clean_and_exit
 229 fi
 230
 231 if [ ! -d $TMPDIR ]; then
 232         mkdir -p $TMPDIR
 233 fi
 234
 235 DEPENDSTREEFILE=$TMPDIR/dependstree
 236 export DEPENDSTREEFILE
 237 DEPENDSFILE=$TMPDIR/depends
 238 export DEPENDSFILE
 239 SUPPORTSFILE=$TMPDIR/supports
 240 export SUPPORTSFILE
 241 INDEXFILE=$TMPDIR/index
 242 export SUPPORTSFILE
 243 ORDERFILE=$TMPDIR/order
 244 export ORDERFILE
 245 DATABASEFILE=$TMPDIR/database
 246 export DATABASEFILE
 247 BINPKGFILE=$TMPDIR/binpkglist
 248
 249 echo "Starting README.html generation: `date`"
 250
 251 ######################################################################
 252 #
 253 #  Extract key pkgsrc configuration variables
 254 #
 255 ######################################################################
 256
 257 echo " "
 258 echo "Extracting tool variables:"
 259 echo " "
 260 if [ -d ${PKGSRCDIR}/pkgtools/prereq-readme ]; then
 261     cd ${PKGSRCDIR}/pkgtools/prereq-readme
 262     eval "`${BMAKE} show-tools`"
 263     for v in AWK CMP ECHO EXPR FGREP FIND GREP GZIP_CMD SED SETENV SORT
 264     do
 265         eval "echo '---->'  ${v}=\"\${${v}}\""
 266     done
 267 else
 268     echo "Error:   ${PKGSRCDIR}/pkgtools/prereq-readme does not seem to exist"
 269     exit 1
 270 fi
 271
 272 echo " "
 273 echo "Extracting configuration variables:"
 274 echo " "
 275 if [ -d ${PKGSRCDIR}/pkgtools/prereq-readme ]; then
 276     cd ${PKGSRCDIR}/pkgtools/prereq-readme
 277     for v in CDROM_PKG_URL_HOST CDROM_PKG_URL_DIR DISTDIR \
 278         FTP_PKG_URL_HOST FTP_PKG_URL_DIR PACKAGES PKG_INFO PKG_SUFX PKG_ADMIN \
 279         AUDIT_PACKAGES AUDIT_PACKAGES_FLAGS PKGTOOLS_VERSION
 280     do
 281         val=`${BMAKE} show-var VARNAME=${v}`
 282         if [ $? != 0 ]; then
 283             echo "Error:  make show-var VARNAME=${v} in `pwd` "
 284             echo "Failed.  This is a fatal error"
 285             clean_and_exit
 286         fi
 287         echo "---->  ${v}=\"${val}\""
 288         eval "${v}=\"${val}\""
 289     done
 290 else
 291     echo "Error:   ${PKGSRCDIR}/pkgtools/prereq-readme does not seem to exist"
 292     exit 1
 293 fi
 294
 295 if [ `${PKG_ADMIN} -V` -lt 20080415 ]; then
 296         SCAN_VULNERABILITIES=0
 297         echo "---->  NOT checking for vulnerabilities, pkg_install too old"
 298 else
 299         _PVDIR=`${PKG_ADMIN} config-var PKGVULNDIR`;
 300         if [ -e "${_PVDIR}"/pkg-vulnerabilities ]; then
 301                 SCAN_VULNERABILITIES=2
 302                 echo "---->  Checking for vulnerabilities"
 303         else
 304                 SCAN_VULNERABILITIES=1
 305                 echo "---->  NOT checking for vulnerabilities"
 306         fi
 307 fi
 308
 309 ######################################################################
 310 #
 311 #  Decide on FTP vs CDROM README.html files
 312 #
 313 ######################################################################
 314
 315 if [ "$ftp_readme" = "yes" ]; then
 316     PKG_URL=${FTP_PKG_URL_HOST}${FTP_PKG_URL_DIR}
 317     echo "Will generate FTP readme files with PKG_URL=$PKG_URL"
 318 else
 319     PKG_URL=${CDROM_PKG_URL_HOST}${CDROM_PKG_URL_DIR}
 320     echo "Will generate CD-ROM readme files with PKG_URL=$PKG_URL"
 321 fi
 322
 323 ######################################################################
 324 #
 325 #  Check for command line switch for packages directory
 326 #
 327 ######################################################################
 328
 329 # we've been given the directory as a command line switch
 330 if [ ! -z "$PKGDIR" ]; then
 331     PACKAGES=$PKGDIR
 332     echo "PACKAGES specified on command line to be $PKGDIR"
 333 fi
 334
 335
 336 ######################################################################
 337 #
 338 #  Extract Database for All Packages (longest step)
 339 #
 340 ######################################################################
 341
 342 if [ "x$restart" = "xno" ] ; then
 343     echo " "
 344     echo "Extracting data.  This could take a while"
 345     echo "Started at: `date` "
 346     echo " "
 347     npkg=1
 348
 349     # make sure we don't have an old database lying around
 350     rm -fr $DATABASEFILE
 351
 352     cd ${PKGSRCDIR}
 353     cats=`${BMAKE} show-subdir-var VARNAME=SUBDIR`
 354     for c in ${cats} ; do
 355         if [ ! -d ${PKGSRCDIR}/${c} ]; then
 356                 echo "WARNING:  The category directory ${c} does not seem to" > /dev/stderr
 357                 echo "          exist under ${PKGSRCDIR}" > /dev/stderr
 358         else
 359                 echo " "
 360                 echo "Extracting data for category ${c}"
 361                 cd ${PKGSRCDIR}/${c}
 362                 list=`${BMAKE} show-subdir-var VARNAME=SUBDIR`
 363                 for pkgdir in $list ; do
 364                         cd ${PKGSRCDIR}/${c}
 365                         if [ ! -d $pkgdir ]; then
 366                                 echo " "
 367                                 echo "WARNING:  the package directory $pkgdir is listed in" > /dev/stderr
 368                                 echo "          ${PKGSRCDIR}/${c}/Makefile" > /dev/stderr
 369                                 echo "          but the directory does not exist.  Please fix this!" > /dev/stderr
 370                         else
 371                                 cd ${PKGSRCDIR}/${c}/${pkgdir}
 372                                 l=`${BMAKE} print-summary-data`
 373                                 if [ $? != 0 ]; then
 374                                         echo "WARNING (printdepends):  the package in ${c}/${pkgdir} had problem with" \
 375                                                 > /dev/stderr
 376                                         echo "    ${BMAKE} print-summary-data" > /dev/stderr
 377                                         echo "    database information for this package" > /dev/stderr
 378                                         echo "    will be dropped." > /dev/stderr
 379                                         ${BMAKE} print-summary-data  2>&1 > /dev/stderr
 380                                 else
 381                                         echo "$l" >> $DATABASEFILE
 382                                 fi
 383                         fi
 384                         ${ECHO} -n "."
 385                         if [ `${EXPR} $npkg % 100 = 0` -eq 1 ]; then
 386                                 echo " "
 387                                 echo "$npkg"
 388                         fi
 389                         npkg=`${EXPR} $npkg + 1`
 390                         cd ${PKGSRCDIR}
 391                 done
 392         fi
 393     done
 394     echo " "
 395     echo "Finished extracting data for ${npkg} packages at: `date` "
 396 else
 397     echo " "
 398     echo "Using existing database (are you sure you wanted the -r/--restart flag?)"
 399     echo " "
 400     if [ ! -f $DATABASEFILE ]; then
 401         echo " "
 402         echo "ERROR:  You have use the -r/--restart flag but the database "
 403         echo "        file $DATABASEFILE does not exist"
 404         echo " "
 405         exit 1
 406     fi
 407 fi
 408
 409 ######################################################################
 410 #
 411 #  Generate the package and category README.html files
 412 #
 413 ######################################################################
 414
 415 echo " "
 416 echo "Generating package README.html files"
 417 echo " "
 418 if [ "x$DEBUG" = "xyes" ]; then
 419     debug=1;
 420 else
 421     debug=0;
 422 fi
 423
 424 ${AWK} -f ${PKGSRCDIR}/mk/scripts/genreadme.awk  \
 425     builddependsfile=${TMPDIR}/pkgsrc.builddepends.debug \
 426     debug=$debug \
 427     dependsfile=${TMPDIR}/pkgsrc.depends.debug \
 428     summary=${summary} \
 429     AWK=$AWK \
 430     CMP=$CMP \
 431     DISTDIR=$DISTDIR \
 432     FIND=$FIND \
 433     GREP=$GREP \
 434     GZIP_CMD="$GZIP_CMD" \
 435     PACKAGES=$PACKAGES \
 436     PKG_ADMIN="$PKG_ADMIN" \
 437     PKG_INFO="$PKG_INFO" \
 438     PKG_SUFX=$PKG_SUFX \
 439     PKG_URL=$PKG_URL \
 440     PKGSRCDIR=$PKGSRCDIR \
 441     PKGTOOLS_VERSION=$PKGTOOLS_VERSION \
 442     SCAN_VULNERABILITIES=${SCAN_VULNERABILITIES} \
 443     SED=$SED \
 444     SETENV=$SETENV \
 445     SORT=$SORT \
 446     TMPDIR=$TMPDIR \
 447         ${DATABASEFILE}
 448
 449 if [ $? != 0 ]; then
 450     echo "Error:  genreadme.awk failed to create README.html files"
 451     clean_and_exit
 452 fi
 453
 454 ######################################################################
 455 #
 456 #  Generate the README-IPv6.html file
 457 #
 458 ######################################################################
 459
 460 echo " "
 461 echo "Generating the README-IPv6.html file"
 462 echo " "
 463 cd ${PKGSRCDIR}
 464 ipv6=${TMPDIR}/ipv6pkgs
 465 ipv6_entries=${TMPDIR}/ipv6_entries
 466 ${GREP} -l -e '^BUILD_DEFS.*=.*IPV6_READY' -e '^PKG_SUPPORTED_OPTIONS.*=.*inet6' -e '^USE_FEATURES.*=.*inet6' */*/Makefile */*/options.mk | ${SED} -e 's;Makefile;;g' -e 's;options.mk;;g' > $ipv6
 467 ${FGREP} -f $ipv6 README-all.html | sort -t/ +1 > $ipv6_entries
 468 ${SED} \
 469     -e "/%%TRS%%/r${ipv6_entries}" \
 470     -e '/%%TRS%%/d' \
 471     templates/README.ipv6 > ${TMPDIR}/README-IPv6.html
 472 if [ $? != 0 ]; then
 473     echo "Error:  README-IPv6.html generation failed (on sed script)"
 474     clean_and_exit
 475 fi
 476
 477 if [ ! -f README-IPv6.html ]; then
 478         mv -f ${TMPDIR}/README-IPv6.html README-IPv6.html
 479 elif cmp -s ${TMPDIR}/README-IPv6.html README-IPv6.html ; then
 480         echo "README-IPv6.html is unchanged (no changes were needed)"
 481 else
 482         mv -f ${TMPDIR}/README-IPv6.html README-IPv6.html
 483 fi
 484
 485 ######################################################################
 486 #
 487 #  Prune README.html files which are no longer needed
 488 #
 489 ######################################################################
 490 if [ "x$prune" = "xyes" ]; then
 491     echo " "
 492     echo "Pruning unused README.html files"
 493     echo " "
 494     cd ${PKGSRCDIR}
 495     for d in `ls -d */*` ; do
 496         if [ -d $d -a ! -f ${d}/Makefile -a -f ${d}/README.html ]; then
 497             echo "Pruning ${d}/README.html which is no longer used"
 498             rm -f ${d}/README.html
 499         fi
 500     done
 501 fi
 502
 503 ######################################################################
 504 #
 505 # All done.  Clean (if needed) and exit
 506 #
 507 ######################################################################
 508 echo " "
 509 echo "README.html generation finished:  `date`"
 510 echo " "
 511 if [ "x$DEBUG" = "xno" -a "x$restart" = "xno" -a "x$save" = "xno" ]; then
 512     rm -fr $TMPDIR
 513 else
 514     echo "Debugging output left in $TMPDIR"
 515 fi
 516