Merge remote branch 'crater/master' into vkernel64
authorMatthew Dillon <dillon@apollo.backplane.com>
Wed, 24 Mar 2010 18:25:37 +0000 (11:25 -0700)
committerMatthew Dillon <dillon@apollo.backplane.com>
Wed, 24 Mar 2010 18:25:37 +0000 (11:25 -0700)
sys/kern/vfs_cache.c
sys/vfs/hammer/hammer_rebalance.c
sys/vfs/nfs/nfs_vnops.c
sys/vm/vm_extern.h
sys/vm/vm_page.c
sys/vm/vm_pageout.c
tools/snapshots/README [new file with mode: 0644]
tools/snapshots/config [new file with mode: 0644]
tools/snapshots/dobuild [new file with mode: 0755]
tools/snapshots/doclean [new file with mode: 0755]
tools/snapshots/docron [new file with mode: 0755]

index ca250a6..08b58df 100644 (file)
@@ -173,7 +173,7 @@ SYSCTL_INT(_debug, OID_AUTO, numcache, CTLFLAG_RD, &numcache, 0, "");
 SYSCTL_INT(_debug, OID_AUTO, vnsize, CTLFLAG_RD, 0, sizeof(struct vnode), "");
 SYSCTL_INT(_debug, OID_AUTO, ncsize, CTLFLAG_RD, 0, sizeof(struct namecache), "");
 
-int cache_mpsafe;
+int cache_mpsafe = 1;
 SYSCTL_INT(_vfs, OID_AUTO, cache_mpsafe, CTLFLAG_RW, &cache_mpsafe, 0, "");
 
 static int cache_resolve_mp(struct mount *mp);
index 5390aa2..e232617 100644 (file)
@@ -119,7 +119,11 @@ retry:
                 * Rebalancing can be hard on the memory allocator, make
                 * sure there is enough free memory before doing it.
                 */
-               vm_wait_nominal();
+               if (vm_test_nominal()) {
+                       hammer_unlock_cursor(&cursor);
+                       vm_wait_nominal();
+                       hammer_lock_cursor(&cursor);
+               }
 
                /*
                 * We only care about internal nodes visited for the last
index 3b5685d..0275b62 100644 (file)
@@ -2701,10 +2701,13 @@ nfs_readdirplusrpc_uio(struct vnode *vp, struct uio *uiop)
                                    cache_put(&nch);
                                } else {
 rdfail:
+                                   ;
+#if 0
                                    kprintf("Warning: NFS/rddirplus, "
                                            "UNABLE TO ENTER %*.*s\n",
                                        nlc.nlc_namelen, nlc.nlc_namelen,
                                        nlc.nlc_nameptr);
+#endif
                                }
                            }
                        } else {
index 49fa4b9..eb5e914 100644 (file)
@@ -96,6 +96,7 @@ int vm_fault_wire (vm_map_t, vm_map_entry_t, boolean_t);
 void vm_fork (struct proc *, struct proc *, int);
 void vm_fault_ratecheck(void);
 void vm_waitproc (struct proc *);
+int vm_test_nominal (void);
 void vm_wait_nominal (void);
 void vm_init_limits(struct proc *);
 
index b360147..c25de4e 100644 (file)
@@ -845,6 +845,17 @@ vm_wait_nominal(void)
                vm_wait(0);
 }
 
+/*
+ * Test if vm_wait_nominal() would block.
+ */
+int
+vm_test_nominal(void)
+{
+       if (vm_page_count_min(0))
+               return(1);
+       return(0);
+}
+
 /*
  * Block until free pages are available for allocation, called in various
  * places before memory allocations.
index eb13c92..691824b 100644 (file)
@@ -1483,7 +1483,7 @@ vm_pageout(void)
         *       inactive queue becomes too small.  If the inactive queue
         *       is large enough to satisfy page movement to free+cache
         *       then it is repopulated more slowly from the active queue.
-        *       This allows a generate inactive_target default to be set.
+        *       This allows a general inactive_target default to be set.
         *
         *       There is an issue here for processes which sit mostly idle
         *       'overnight', such as sshd, tcsh, and X.  Any movement from
@@ -1491,8 +1491,11 @@ vm_pageout(void)
         *       recycle eventually causing a lot of paging in the morning.
         *       To reduce the incidence of this pages cycled out of the
         *       buffer cache are moved directly to the inactive queue if
-        *       they were only used once or twice.  The vfs.vm_cycle_point
-        *       sysctl can be used to adjust this.
+        *       they were only used once or twice.
+        *
+        *       The vfs.vm_cycle_point sysctl can be used to adjust this.
+        *       Increasing the value (up to 64) increases the number of
+        *       buffer recyclements which go directly to the inactive queue.
         */
        if (vmstats.v_free_count > 2048) {
                vmstats.v_cache_min = vmstats.v_free_target;
@@ -1501,7 +1504,7 @@ vm_pageout(void)
                vmstats.v_cache_min = 0;
                vmstats.v_cache_max = 0;
        }
-       vmstats.v_inactive_target = vmstats.v_free_count / 2;
+       vmstats.v_inactive_target = vmstats.v_free_count / 4;
 
        /* XXX does not really belong here */
        if (vm_page_max_wired == 0)
diff --git a/tools/snapshots/README b/tools/snapshots/README
new file mode 100644 (file)
index 0000000..93d8c29
--- /dev/null
@@ -0,0 +1,20 @@
+
+    Generate daily HEAD snapshots.  Basically setup the config file and
+    these files in ~root/snapshots and add a root cron job:
+
+    23 2 * * *      cd /root/snapshots; lockf -ks -t 0 .buildlock ./docron
+
+    The snapshots can be made available via rsyncd using a rsyncd.conf
+    file similar to this:
+
+uid = nobody
+gid = nobody
+use chroot = yes
+
+[snapshots]
+        path = /archive/snapshots
+
+    Another machine can mirror the snapshots using something like this:
+
+    lockf -ks -t 0 /root/adm/.rsynclock2 /usr/pkg/bin/rsync -aHS       \
+       snapshotbox.domain::snapshots/ /archive/snapshots/
diff --git a/tools/snapshots/config b/tools/snapshots/config
new file mode 100644 (file)
index 0000000..e136823
--- /dev/null
@@ -0,0 +1,10 @@
+# Config file for builder
+#
+
+set build = /build/snapshots
+set dflygit = /repository/git/dragonfly.git
+set pkgsrcgit = /repository/git/pkgsrc.git
+set dfly_branch = master
+set pkgsrc_branch = vendor
+set snapshots = /archive/snapshots
+set relsname = DEV
diff --git a/tools/snapshots/dobuild b/tools/snapshots/dobuild
new file mode 100755 (executable)
index 0000000..ccef987
--- /dev/null
@@ -0,0 +1,117 @@
+#!/bin/csh
+#
+# Create a working chroot for snapshot building
+#
+
+source config
+
+set ncpu = `sysctl -n hw.ncpu`
+set pushdsilent
+set xdate = "date"
+
+echo "`$xdate` - SNAPSHOTS DOBUILD BEGIN"
+
+# Options
+#
+set nrelease_args = "release"
+
+foreach arg ( $argv )
+    switch ( $arg )
+    case "realquick":
+       set quick = 1
+       set realquick = 1
+       set nrelease_args = "realquick"
+       breaksw
+    case "quick":
+       set quick = 1
+       set nrelease_args = "quick"
+       breaksw
+    default:
+       echo "`$xdate` - Unknown option $arg"
+       echo "`$xdate` - SNAPSHOTS DOBUILD END"
+       exit 1
+    endsw
+end
+
+echo "`$xdate` - Setting up chroot environment"
+if ( ! -e $build/root/dev/null ) then
+    mount_devfs dummy $build/root/dev
+endif
+
+umount $build/root/usr/pkgsrc/distfiles >& /dev/null
+mount_null $build/distfiles $build/root/usr/pkgsrc/distfiles
+
+setenv PKGSRC_PATH /usr/pkgsrc
+
+echo "`$xdate` - Build is running nrelease, tail -f $build/nrelease-build.out"
+chroot $build/root csh -c "cd /usr/src/nrelease; make -DNOPROFILE -DWITHOUT_SRCS img $nrelease_args" >& $build/nrelease-build.out
+
+if ( $? > 0 ) then
+    echo "`$xdate` - NRelease build had a problem, see $build/nrelease-build.out"
+    echo "`$xdate` - SNAPSHOTS DOBUILD END"
+    exit 1
+endif
+
+# Figure out the filename
+#
+set arch = `uname -p`
+if ( "$arch" == "i386" ) then
+set kobjs = $build/root/usr/obj/usr/src/sys/GENERIC
+else
+set kobjs = $build/root/usr/obj/usr/src/sys/X86_64_GENERIC
+endif
+
+if ( ! -f $kobjs/vers.txt ) then
+    echo "`$xdate` - nrelease build did not generate $kobjs/vers.txt"
+    echo "`$xdate` - SNAPSHOTS DOBUILD END"
+    exit 1
+endif
+
+# Store the snapshots.  Make sure the files are inaccessible until
+# completely written out.
+#
+set basename = `cat $kobjs/vers.txt`
+mkdir -p $snapshots/$arch/images
+mkdir -p $snapshots/$arch/kernels
+set filepath = $snapshots/$arch
+umask 077
+bzip2 < $build/root/usr/release/dfly.iso > $filepath/images/$basename.iso.bz2
+bzip2 < $build/root/usr/release/dfly.img > $filepath/images/$basename.img.bz2
+(cd $build/root/usr/release/root; tar cf - boot | bzip2 > $filepath/kernels/$basename.boot.tar.bz2)
+chmod 644 $filepath/images/$basename.iso.bz2
+chmod 644 $filepath/images/$basename.img.bz2
+chmod 644 $filepath/kernels/$basename.boot.tar.bz2
+sync
+pushd $filepath/images
+md5 $basename.iso.bz2 >> CHECKSUM.MD5
+md5 $basename.img.bz2 >> CHECKSUM.MD5
+popd
+
+pushd $filepath/kernels
+md5 $basename.boot.tar.bz2 >> CHECKSUM.MD5
+popd
+
+# Only adjust the 'latest' softlink shortcuts for
+# development snapshots.  Release shortcuts have
+# to be done manually.
+#
+if ( "$relsname" == "DEV" ) then
+    pushd $filepath
+    rm -f DragonFly-${arch}-LATEST-ISO.iso.bz2.new
+    rm -f DragonFly-${arch}-LATEST-IMG.img.bz2.new
+    rm -f DragonFly-${arch}-LATEST-BOOT.tar.bz2.new
+    ln -s images/$basename.iso.bz2 DragonFly-${arch}-LATEST-ISO.iso.bz2.new
+    ln -s images/$basename.img.bz2 DragonFly-${arch}-LATEST-IMG.img.bz2.new
+    ln -s kernels/$basename.boot.tar.bz2 DragonFly-${arch}-LATEST-BOOT.tar.bz2.new
+    mv -f DragonFly-${arch}-LATEST-ISO.iso.bz2.new \
+           DragonFly-${arch}-LATEST-ISO.iso.bz2
+    mv -f DragonFly-${arch}-LATEST-IMG.img.bz2.new \
+           DragonFly-${arch}-LATEST-IMG.img.bz2
+    mv -f DragonFly-${arch}-LATEST-BOOT.tar.bz2.new \
+           DragonFly-${arch}-LATEST-BOOT.tar.bz2
+    popd
+endif
+
+sync
+
+echo "`$xdate` - SNAPSHOTS DOBUILD END"
diff --git a/tools/snapshots/doclean b/tools/snapshots/doclean
new file mode 100755 (executable)
index 0000000..680b202
--- /dev/null
@@ -0,0 +1,219 @@
+#!/bin/csh
+#
+# Create a working chroot for snapshot building
+#
+
+source config
+
+set ncpu = `sysctl -n hw.ncpu`
+set pushdsilent
+set xdate = "date"
+
+echo "`$xdate` - SNAPSHOTS DOCLEAN BEGIN"
+
+# Options
+#
+foreach arg ( $argv )
+    switch ( $arg )
+    case "realquick":
+       set quick = 1
+       set realquick = 1
+       breaksw
+    case "quick":
+       set quick = 1
+       breaksw
+    default:
+       echo "Unknown option $arg"
+       echo "`$xdate` - SNAPSHOTS DOCLEAN END"
+       exit 1
+    endsw
+end
+
+# Remove environment
+#
+if ( "$build" == "" ) then
+    echo "build variable not set"
+    echo "`$xdate` - SNAPSHOTS DOCLEAN END"
+    exit 1
+endif
+
+if ( -e $build/root/dev/null ) then
+    echo "`$xdate` - Unmount devfs from $build/root/dev"
+    umount $build/root/dev
+    if ( $? > 0 ) then
+       echo "`$xdate` - Unable to umount $build/root/dev"
+       echo "`$xdate` - SNAPSHOTS DOCLEAN END"
+       exit 1
+    endif
+endif
+
+# remount the nullfs mount read-only just in case the umount fails.
+#
+umount $build/root/usr/pkgsrc/distfiles >& /dev/null
+mount -u -o ro $build/root/usr/pkgsrc/distfiles >& /dev/null
+
+if ( $?quick == 0 ) then
+    echo "`$xdate` - Destroying root environment at $build/root"
+    sleep 5
+    rm -rf $build/root >& /dev/null
+    if ( -d $build/root ) then
+       chflags -R noschg $build/root
+    endif
+    rm -rf $build/root
+endif
+mkdir -p $build/distfiles
+mkdir -p $build/root
+mkdir -p $build/root/usr
+
+echo "`$xdate` - Log files are in $build/*.out"
+
+# Ignore quick/realquick options if there is no prior
+# installation
+#
+if ( ! -e $build/root/etc/networks ) then
+    unset realquick
+endif
+if ( ! -d $build/root/usr/src ) then
+    unset quick
+    unset realquick
+endif
+if ( ! -d $build/root/usr/pkgsrc ) then
+    unset quick
+    unset realquick
+endif
+
+# Clone the repos, create trakcing branches as necessary, checkout,
+# and update (using fetch)
+#
+# DragonFly sources
+#
+pushd $build/root/usr
+if ( $?quick == 0 ) then
+    echo "`$xdate` - Cloning $dflygit"
+    rm -rf src
+    git clone -n $dflygit src
+endif
+popd
+pushd $build/root/usr/src
+if ( "`git branch | fgrep $dfly_branch`" == "" ) then
+    echo "`$xdate` - Branching-src $dfly_branch"
+    git branch $dfly_branch origin/$dfly_branch
+endif
+echo "`$xdate` - Updating src"
+git checkout $dfly_branch
+git pull
+popd
+
+# Package sources
+#
+pushd $build/root/usr
+if ( $?quick == 0 ) then
+    echo "`$xdate` - Cloning $pkgsrcgit"
+    rm -rf pkgsrc
+    git clone -n $pkgsrcgit pkgsrc
+endif
+popd
+pushd $build/root/usr/pkgsrc
+if ( "`git branch | fgrep $pkgsrc_branch`" == "" ) then
+    echo "`$xdate` - Branching-pkgsrc $pkgsrc_branch"
+    git branch $pkgsrc_branch origin/$pkgsrc_branch
+endif
+echo "`$xdate` - Updating pkgsrc"
+git checkout $pkgsrc_branch
+git pull
+popd
+
+# Install/upgrade environment
+#
+pushd $build/root/usr/src
+
+if ( $?realquick ) then
+    echo "`$xdate` - Not running build/quickworld - realquick mode"
+else
+if ( $?quick ) then
+    echo "`$xdate` - Starting quickworld, tail -f $build/build.out"
+    make -j $ncpu quickworld >& $build/build.out
+    if ( $? ) then
+       echo "`$xdate` - primary environment quickworld failed"
+       echo "`$xdate` - SNAPSHOTS DOCLEAN END"
+       exit 1
+    endif
+else
+    echo "`$xdate` - Starting buildworld, tail -f $build/build.out"
+    make -j $ncpu buildworld >& $build/build.out
+    if ( $? ) then
+       echo "`$xdate` - primary environment buildworld failed"
+       echo "`$xdate` - SNAPSHOTS DOCLEAN END"
+       exit 1
+    endif
+endif
+endif
+
+if ( $?realquick == 0 ) then
+    echo "`$xdate` - Installing the world, tail -f $build/install.out"
+    make installworld DESTDIR=$build/root >& $build/install.out
+    pushd etc
+    make distribution DESTDIR=$build/root >& $build/install.out
+    popd
+    echo "`$xdate` - Upgrading the world, tail -f $build/upgrade.out"
+    make upgrade DESTDIR=$build/root >& $build/upgrade.out
+endif
+
+popd
+
+# Setup the chroot environment, including packages nrelease needs to
+# build.
+#
+echo "`$xdate` - Setting up chroot environment in $build/root"
+echo "`$xdate` - Mounting devfs"
+if ( ! -e $build/root/dev/null ) then
+    mount_devfs dummy $build/root/dev
+endif
+echo "`$xdate` - Mounting distfiles"
+mount_null $build/distfiles $build/root/usr/pkgsrc/distfiles
+
+#mkdir -m 1777 $build/root/tmp
+#mkdir -m 1777 $build/root/var/tmp
+
+if ( $?realquick == 0 ) then
+    echo "`$xdate` - Setting up ldd"
+    chroot $build/root /etc/rc.d/ldconfig start
+endif
+cp /etc/resolv.conf $build/root/etc/
+
+if ( ! -e $build/root/usr/pkg/bin/bmake ) then
+    echo "`$xdate` - Bootstrapping pkgsrc, tail -f $build/bootstrap.out"
+    chroot $build/root csh -c "cd /usr/pkgsrc/bootstrap; rm -rf /usr/obj/bootstrap; ./bootstrap --workdir=/usr/obj/bootstrap" >& $build/bootstrap.out
+    if ( $? > 0 ) then
+       echo "`$xdate` - pkgsrc bootstrap had problems"
+       echo "`$xdate` - SNAPSHOTS DOCLEAN END"
+       exit 1
+    endif
+endif
+
+if ( ! -e $build/root/usr/pkg/bin/mkisofs ) then
+    echo "`$xdate` - Setting up cdrecord and friends, tail -f $build/nrelease1.out"
+    setenv PKGSRC_PATH /usr/pkgsrc
+    chroot $build/root csh -c "cd /usr/src/nrelease; make fetch pkgsrc_cdrecord" >& $build/nrelease1.out
+    if ( $? > 0 ) then
+       echo "`$xdate` - nrelease had problems"
+       echo "`$xdate` - SNAPSHOTS DOCLEAN END"
+       exit 1
+    endif
+endif
+
+# The nrelease build needs scmgit as well.  Try to get the binary package
+# first and then build whatever else is needed from source.  These packages
+# are not part of the nrelease snapshot build, they are used by nrelease
+# to do the build.
+#
+if ( ! -e $build/root/usr/pkg/bin/git ) then
+    echo "`$xdate` - Setting up scmgit from pkgsrc, tail -f $build/nrelease2.out"
+    chroot $build/root csh -c "pkg_radd scmgit"
+    if ( ! -e $build/root/usr/pkg/bin/git ) then
+           echo "`$xdate` - Binary package not found, building from source"
+           chroot $build/root csh -c "cd /usr/pkgsrc/devel/scmgit; bmake update"
+    endif
+endif
+
+echo "`$xdate` - SNAPSHOTS DOCLEAN END"
diff --git a/tools/snapshots/docron b/tools/snapshots/docron
new file mode 100755 (executable)
index 0000000..a7ec0d5
--- /dev/null
@@ -0,0 +1,9 @@
+#!/bin/csh
+#
+
+./doclean quick
+if ( $? > 0 ) then
+    echo "doclean script failed, aborting build"
+    exit 1
+endif
+./dobuild