kernel - Enhance getcacheblk() (improve saturated write performance (3)).

* Change getcacheblk() to use getblk() instead of findblk() when the
  block size is known, allowing a cached buffer to be reinstantiated
  from VM backing store in the MPSAFE path.

  Well, mostly MPSAFE, the vm_token is still acquired, but this is still
  much faster than what HAMMER does when the buffer is not in-cache.

* This more than doubles blogbench performance w/HAMMER and further reduces
  concurrent read vs write conflicts.

diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c
index 2c26b85..054aa5a 100644 (file)
--- a/sys/kern/vfs_bio.c
+++ b/sys/kern/vfs_bio.c
@@ -2807,20 +2807,41 @@ unrefblk(struct buf *bp)
  *     If B_RAM is set the buffer might be just fine, but we return
  *     NULL anyway because we want the code to fall through to the
  *     cluster read.  Otherwise read-ahead breaks.
+ *
+ *     If blksize is 0 the buffer cache buffer must already be fully
+ *     cached.
+ *
+ *     If blksize is non-zero getblk() will be used, allowing a buffer
+ *     to be reinstantiated from its VM backing store.  The buffer must
+ *     still be fully cached after reinstantiation to be returned.
  */
 struct buf *
-getcacheblk(struct vnode *vp, off_t loffset)
+getcacheblk(struct vnode *vp, off_t loffset, int blksize)
 {
        struct buf *bp;
 
-       bp = findblk(vp, loffset, 0);
-       if (bp) {
-               if ((bp->b_flags & (B_INVAL | B_CACHE | B_RAM)) == B_CACHE) {
-                       bp->b_flags &= ~B_AGE;
-                       bremfree(bp);
-               } else {
-                       BUF_UNLOCK(bp);
-                       bp = NULL;
+       if (blksize) {
+               bp = getblk(vp, loffset, blksize, 0, 0);
+               if (bp) {
+                       if ((bp->b_flags & (B_INVAL | B_CACHE | B_RAM)) ==
+                           B_CACHE) {
+                               bp->b_flags &= ~B_AGE;
+                       } else {
+                               brelse(bp);
+                               bp = NULL;
+                       }
+               }
+       } else {
+               bp = findblk(vp, loffset, 0);
+               if (bp) {
+                       if ((bp->b_flags & (B_INVAL | B_CACHE | B_RAM)) ==
+                           B_CACHE) {
+                               bp->b_flags &= ~B_AGE;
+                               bremfree(bp);
+                       } else {
+                               BUF_UNLOCK(bp);
+                               bp = NULL;
+                       }
                }
        }
        return (bp);

diff --git a/sys/sys/buf.h b/sys/sys/buf.h
index 5c2a0d6..c09f235 100644 (file)
--- a/sys/sys/buf.h
+++ b/sys/sys/buf.h
@@ -422,7 +422,7 @@ struct buf *getpbuf_kva (int *);
 int    inmem (struct vnode *, off_t);
 struct buf *findblk (struct vnode *, off_t, int);
 struct buf *getblk (struct vnode *, off_t, int, int, int);
-struct buf *getcacheblk (struct vnode *, off_t);
+struct buf *getcacheblk (struct vnode *, off_t, int);
 struct buf *geteblk (int);
 void unrefblk(struct buf *bp);
 void regetblk(struct buf *bp);

diff --git a/sys/vfs/hammer/hammer_vnops.c b/sys/vfs/hammer/hammer_vnops.c
index 8ddb6a2..79bd55b 100644 (file)
--- a/sys/vfs/hammer/hammer_vnops.c
+++ b/sys/vfs/hammer/hammer_vnops.c
@@ -372,7 +372,7 @@ hammer_vop_read(struct vop_read_args *ap)
                /*
                 * MPSAFE
                 */
-               bp = getcacheblk(ap->a_vp, base_offset);
+               bp = getcacheblk(ap->a_vp, base_offset, blksize);
                if (bp) {
                        error = 0;
                        goto skip;

diff --git a/sys/vfs/tmpfs/tmpfs_vnops.c b/sys/vfs/tmpfs/tmpfs_vnops.c
index 1563245..c73dda9 100644 (file)
--- a/sys/vfs/tmpfs/tmpfs_vnops.c
+++ b/sys/vfs/tmpfs/tmpfs_vnops.c
@@ -447,7 +447,7 @@ tmpfs_read (struct vop_read_args *ap)
                 */
                offset = (size_t)uio->uio_offset & BMASK;
                base_offset = (off_t)uio->uio_offset - offset;
-               bp = getcacheblk(vp, base_offset);
+               bp = getcacheblk(vp, base_offset, BSIZE);
                if (bp == NULL)
                {
                        error = bread(vp, base_offset, BSIZE, &bp);