MPSAFE - Add read() and write() path MPSAFE support, and more.

* Add sysctl vfs.read_mpsafe, vfs.write_mpsafe, and vfs.getattr_mpsafe.
  Only vfs.read_mpsafe is currently implemented, and only for HAMMER
  filesystems.

* read() and write() will now call VOP_READ()/VOP_WRITE() without the
  MP lock if the vnode->v_flag says it is ok to do so.

* read() and write() now serialize reads and writes which depend on the
  file offset stored in the fp, when the concurrent operations share the
  same fp.

* Add vnode flags VMP_READ, VMP_WRITE, VMP_GETATTR.

* Add vnode flag VNOTSEEKABLE indicating that fp->f_offset will never be
  used and no serialization is needed (which e.g. can mess up read()'s
  blocking devices such as ttys).

* Correct bug in FIONREAD related to the data field being too small.

diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c
index e78950c..cceaa54 100644 (file)
--- a/sys/kern/vfs_bio.c
+++ b/sys/kern/vfs_bio.c
@@ -2567,6 +2567,8 @@ vfs_setdirty(struct buf *bp)
  *                       to acquire the lock we return NULL, even if the
  *                       buffer exists.
  *
+ *     (0)             - Lock the buffer blocking.
+ *
  * MPSAFE
  */
 struct buf *
@@ -2586,7 +2588,7 @@ findblk(struct vnode *vp, off_t loffset, int flags)
                lwkt_reltoken(&vlock);
                if (bp == NULL || (flags & FINDBLK_TEST))
                        break;
-               if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT)) {
+               if (BUF_LOCK(bp, lkflags)) {
                        bp = NULL;
                        break;
                }
@@ -2598,6 +2600,35 @@ findblk(struct vnode *vp, off_t loffset, int flags)
 }
 
 /*
+ * getcacheblk:
+ *
+ *     Similar to getblk() except only returns the buffer if it is
+ *     B_CACHE and requires no other manipulation.  Otherwise NULL
+ *     is returned.
+ *
+ *     If B_RAM is set the buffer might be just fine, but we return
+ *     NULL anyway because we want the code to fall through to the
+ *     cluster read.  Otherwise read-ahead breaks.
+ */
+struct buf *
+getcacheblk(struct vnode *vp, off_t loffset)
+{
+       struct buf *bp;
+
+       bp = findblk(vp, loffset, 0);
+       if (bp) {
+               if ((bp->b_flags & (B_INVAL | B_CACHE | B_RAM)) == B_CACHE) {
+                       bp->b_flags &= ~B_AGE;
+                       bremfree(bp);
+               } else {
+                       BUF_UNLOCK(bp);
+                       bp = NULL;
+               }
+       }
+       return (bp);
+}
+
+/*
  * getblk:
  *
  *     Get a block given a specified block and offset into a file/device.

diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c
index 89c30f8..357e2b2 100644 (file)
--- a/sys/kern/vfs_vnops.c
+++ b/sys/kern/vfs_vnops.c
@@ -54,8 +54,11 @@
 #include <sys/filio.h>
 #include <sys/ttycom.h>
 #include <sys/conf.h>
+#include <sys/sysctl.h>
 #include <sys/syslog.h>
 
+#include <sys/thread2.h>
+
 static int vn_closefile (struct file *fp);
 static int vn_ioctl (struct file *fp, u_long com, caddr_t data,
                struct ucred *cred);
@@ -71,6 +74,19 @@ static int vn_write (struct file *fp, struct uio *uio,
 static int svn_write (struct file *fp, struct uio *uio, 
                struct ucred *cred, int flags);
 
+#ifdef SMP
+static int read_mpsafe = 0;
+SYSCTL_INT(_vfs, OID_AUTO, read_mpsafe, CTLFLAG_RW, &read_mpsafe, 0, "");
+static int write_mpsafe = 0;
+SYSCTL_INT(_vfs, OID_AUTO, write_mpsafe, CTLFLAG_RW, &write_mpsafe, 0, "");
+static int getattr_mpsafe = 0;
+SYSCTL_INT(_vfs, OID_AUTO, getattr_mpsafe, CTLFLAG_RW, &getattr_mpsafe, 0, "");
+#else
+#define read_mpsafe    0
+#define write_mpsafe   0
+#define getattr_mpsafe 0
+#endif
+
 struct fileops vnode_fileops = {
        .fo_read = vn_read,
        .fo_write = vn_write,
@@ -421,6 +437,8 @@ sequential_heuristic(struct uio *uio, struct file *fp)
 {
        /*
         * Sequential heuristic - detect sequential operation
+        *
+        * NOTE: SMP: We allow f_seqcount updates to race.
         */
        if ((uio->uio_offset == 0 && fp->f_seqcount > 0) ||
            uio->uio_offset == fp->f_nextoff) {
@@ -440,6 +458,8 @@ sequential_heuristic(struct uio *uio, struct file *fp)
 
        /*
         * Not sequential, quick draw-down of seqcount
+        *
+        * NOTE: SMP: We allow f_seqcount updates to race.
         */
        if (fp->f_seqcount > 1)
                fp->f_seqcount = 1;
@@ -449,6 +469,85 @@ sequential_heuristic(struct uio *uio, struct file *fp)
 }
 
 /*
+ * get - lock and return the f_offset field.
+ * set - set and unlock the f_offset field.
+ *
+ * These routines serve the dual purpose of serializing access to the
+ * f_offset field (at least on i386) and guaranteeing operational integrity
+ * when multiple read()ers and write()ers are present on the same fp.
+ */
+static __inline off_t
+vn_get_fpf_offset(struct file *fp)
+{
+       u_int   flags;
+       u_int   nflags;
+
+       /*
+        * Shortcut critical path.
+        */
+       flags = fp->f_flag & ~FOFFSETLOCK;
+       if (atomic_cmpset_int(&fp->f_flag, flags, flags | FOFFSETLOCK))
+               return(fp->f_offset);
+
+       /*
+        * The hard way
+        */
+       for (;;) {
+               flags = fp->f_flag;
+               if (flags & FOFFSETLOCK) {
+                       nflags = flags | FOFFSETWAKE;
+                       crit_enter();
+                       tsleep_interlock(&fp->f_flag);
+                       if (atomic_cmpset_int(&fp->f_flag, flags, nflags))
+                               tsleep(&fp->f_flag, 0, "fpoff", 0);
+                       crit_exit();
+               } else {
+                       nflags = flags | FOFFSETLOCK;
+                       if (atomic_cmpset_int(&fp->f_flag, flags, nflags))
+                               break;
+               }
+       }
+       return(fp->f_offset);
+}
+
+static __inline void
+vn_set_fpf_offset(struct file *fp, off_t offset)
+{
+       u_int   flags;
+       u_int   nflags;
+
+       /*
+        * We hold the lock so we can set the offset without interference.
+        */
+       fp->f_offset = offset;
+
+       /*
+        * Normal release is already a reasonably critical path.
+        */
+       for (;;) {
+               flags = fp->f_flag;
+               nflags = flags & ~(FOFFSETLOCK | FOFFSETWAKE);
+               if (atomic_cmpset_int(&fp->f_flag, flags, nflags)) {
+                       if (flags & FOFFSETWAKE)
+                               wakeup(&fp->f_flag);
+                       break;
+               }
+       }
+}
+
+static __inline off_t
+vn_poll_fpf_offset(struct file *fp)
+{
+#if defined(__amd64__) || !defined(SMP)
+       return(fp->f_offset);
+#else
+       off_t off = vn_get_fpf_offset(fp);
+       vn_set_fpf_offset(fp, off);
+       return(off);
+#endif
+}
+
+/*
  * Package up an I/O request on a vnode into a uio and do it.
  */
 int
@@ -543,6 +642,11 @@ vn_rdwr_inchunks(enum uio_rw rw, struct vnode *vp, caddr_t base, int len,
 
 /*
  * MPALMOSTSAFE - acquires mplock
+ *
+ * File pointers can no longer get ripped up by revoke so
+ * we don't need to lock access to the vp.
+ *
+ * f_offset updates are not guaranteed against multiple readers
  */
 static int
 vn_read(struct file *fp, struct uio *uio, struct ucred *cred, int flags)
@@ -551,7 +655,6 @@ vn_read(struct file *fp, struct uio *uio, struct ucred *cred, int flags)
        struct vnode *vp;
        int error, ioflag;
 
-       get_mplock();
        KASSERT(uio->uio_td == curthread,
                ("uio_td %p is not td %p", uio->uio_td, curthread));
        vp = (struct vnode *)fp->f_data;
@@ -571,19 +674,24 @@ vn_read(struct file *fp, struct uio *uio, struct ucred *cred, int flags)
        } else if (fp->f_flag & O_DIRECT) {
                ioflag |= IO_DIRECT;
        }
+       if ((flags & O_FOFFSET) == 0 && (vp->v_flag & VNOTSEEKABLE) == 0)
+               uio->uio_offset = vn_get_fpf_offset(fp);
        vn_lock(vp, LK_SHARED | LK_RETRY);
-       if ((flags & O_FOFFSET) == 0)
-               uio->uio_offset = fp->f_offset;
        ioflag |= sequential_heuristic(uio, fp);
 
        ccms_lock_get_uio(&vp->v_ccms, &ccms_lock, uio);
-       error = VOP_READ(vp, uio, ioflag, cred);
+       if (read_mpsafe && (vp->v_flag & VMP_READ)) {
+               error = VOP_READ(vp, uio, ioflag, cred);
+       } else {
+               get_mplock();
+               error = VOP_READ(vp, uio, ioflag, cred);
+               rel_mplock();
+       }
        ccms_lock_put(&vp->v_ccms, &ccms_lock);
-       if ((flags & O_FOFFSET) == 0)
-               fp->f_offset = uio->uio_offset;
        fp->f_nextoff = uio->uio_offset;
        vn_unlock(vp);
-       rel_mplock();
+       if ((flags & O_FOFFSET) == 0 && (vp->v_flag & VNOTSEEKABLE) == 0)
+               vn_set_fpf_offset(fp, uio->uio_offset);
        return (error);
 }
 
@@ -623,8 +731,8 @@ svn_read(struct file *fp, struct uio *uio, struct ucred *cred, int flags)
                error = 0;
                goto done;
        }
-       if ((flags & O_FOFFSET) == 0)
-               uio->uio_offset = fp->f_offset;
+       if ((flags & O_FOFFSET) == 0 && (vp->v_flag & VNOTSEEKABLE) == 0)
+               uio->uio_offset = vn_get_fpf_offset(fp);
 
        ioflag = 0;
        if (flags & O_FBLOCKING) {
@@ -646,9 +754,9 @@ svn_read(struct file *fp, struct uio *uio, struct ucred *cred, int flags)
        error = dev_dread(dev, uio, ioflag);
 
        release_dev(dev);
-       if ((flags & O_FOFFSET) == 0)
-               fp->f_offset = uio->uio_offset;
        fp->f_nextoff = uio->uio_offset;
+       if ((flags & O_FOFFSET) == 0 && (vp->v_flag & VNOTSEEKABLE) == 0)
+               vn_set_fpf_offset(fp, uio->uio_offset);
 done:
        rel_mplock();
        return (error);
@@ -664,16 +772,9 @@ vn_write(struct file *fp, struct uio *uio, struct ucred *cred, int flags)
        struct vnode *vp;
        int error, ioflag;
 
-       get_mplock();
        KASSERT(uio->uio_td == curthread,
                ("uio_td %p is not p %p", uio->uio_td, curthread));
        vp = (struct vnode *)fp->f_data;
-#if 0
-       /* VOP_WRITE should handle this now */
-       if (vp->v_type == VREG || vp->v_type == VDATABASE)
-               bwillwrite();
-#endif
-       vp = (struct vnode *)fp->f_data;        /* XXX needed? */
 
        ioflag = IO_UNIT;
        if (vp->v_type == VREG &&
@@ -705,18 +806,23 @@ vn_write(struct file *fp, struct uio *uio, struct ucred *cred, int flags)
 
        if (vp->v_mount && (vp->v_mount->mnt_flag & MNT_SYNCHRONOUS))
                ioflag |= IO_SYNC;
-       vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
        if ((flags & O_FOFFSET) == 0)
-               uio->uio_offset = fp->f_offset;
+               uio->uio_offset = vn_get_fpf_offset(fp);
+       vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
        ioflag |= sequential_heuristic(uio, fp);
        ccms_lock_get_uio(&vp->v_ccms, &ccms_lock, uio);
-       error = VOP_WRITE(vp, uio, ioflag, cred);
+       if (write_mpsafe && (vp->v_flag & VMP_WRITE)) {
+               error = VOP_WRITE(vp, uio, ioflag, cred);
+       } else {
+               get_mplock();
+               error = VOP_WRITE(vp, uio, ioflag, cred);
+               rel_mplock();
+       }
        ccms_lock_put(&vp->v_ccms, &ccms_lock);
-       if ((flags & O_FOFFSET) == 0)
-               fp->f_offset = uio->uio_offset;
        fp->f_nextoff = uio->uio_offset;
        vn_unlock(vp);
-       rel_mplock();
+       if ((flags & O_FOFFSET) == 0)
+               vn_set_fpf_offset(fp, uio->uio_offset);
        return (error);
 }
 
@@ -756,7 +862,7 @@ svn_write(struct file *fp, struct uio *uio, struct ucred *cred, int flags)
        reference_dev(dev);
 
        if ((flags & O_FOFFSET) == 0)
-               uio->uio_offset = fp->f_offset;
+               uio->uio_offset = vn_get_fpf_offset(fp);
 
        ioflag = IO_UNIT;
        if (vp->v_type == VREG && 
@@ -793,9 +899,9 @@ svn_write(struct file *fp, struct uio *uio, struct ucred *cred, int flags)
        error = dev_dwrite(dev, uio, ioflag);
 
        release_dev(dev);
-       if ((flags & O_FOFFSET) == 0)
-               fp->f_offset = uio->uio_offset;
        fp->f_nextoff = uio->uio_offset;
+       if ((flags & O_FOFFSET) == 0)
+               vn_set_fpf_offset(fp, uio->uio_offset);
 done:
        rel_mplock();
        return (error);
@@ -963,6 +1069,7 @@ vn_ioctl(struct file *fp, u_long com, caddr_t data, struct ucred *ucred)
        struct vnode *ovp;
        struct vattr vattr;
        int error;
+       off_t size;
 
        get_mplock();
 
@@ -973,7 +1080,12 @@ vn_ioctl(struct file *fp, u_long com, caddr_t data, struct ucred *ucred)
                        error = VOP_GETATTR(vp, &vattr);
                        if (error)
                                break;
-                       *(int *)data = vattr.va_size - fp->f_offset;
+                       size = vattr.va_size;
+                       if ((vp->v_flag & VNOTSEEKABLE) == 0)
+                               size -= vn_poll_fpf_offset(fp);
+                       if (size > 0x7FFFFFFF)
+                               size = 0x7FFFFFFF;
+                       *(int *)data = size;
                        error = 0;
                        break;
                }

diff --git a/sys/sys/buf.h b/sys/sys/buf.h
index d97f7e2..c08afba 100644 (file)
--- a/sys/sys/buf.h
+++ b/sys/sys/buf.h
@@ -408,6 +408,7 @@ struct buf *getpbuf (int *);
 int    inmem (struct vnode *, off_t);
 struct buf *findblk (struct vnode *, off_t, int);
 struct buf *getblk (struct vnode *, off_t, int, int, int);
+struct buf *getcacheblk (struct vnode *, off_t);
 struct buf *geteblk (int);
 void regetblk(struct buf *bp);
 struct bio *push_bio(struct bio *);

diff --git a/sys/sys/device.h b/sys/sys/device.h
index 8b0633b..c5e9748 100644 (file)
--- a/sys/sys/device.h
+++ b/sys/sys/device.h
@@ -234,6 +234,7 @@ struct dev_ops {
 #define D_MEM          0x0008
 
 #define D_TYPEMASK     0xffff
+#define D_SEEKABLE     (D_TAPE | D_DISK | D_MEM)
 
 /*
  * Flags for d_flags.

diff --git a/sys/sys/fcntl.h b/sys/sys/fcntl.h
index df00fe2..7593dbb 100644 (file)
--- a/sys/sys/fcntl.h
+++ b/sys/sys/fcntl.h
@@ -114,6 +114,8 @@
 #if defined(_KERNEL) || defined(_KERNEL_STRUCTURES)
 #define FREVOKED       0x10000000      /* revoked by fdrevoke() */
 #define FAPPENDONLY    0x20000000      /* O_APPEND cannot be changed */
+#define FOFFSETLOCK    0x40000000      /* f_offset locked */
+#define FOFFSETWAKE    0x80000000      /* f_offset wakeup */
 #endif
 
 #define O_FMASK                (O_FBLOCKING|O_FNONBLOCKING|O_FAPPEND|O_FOFFSET|\

diff --git a/sys/sys/vnode.h b/sys/sys/vnode.h
index deb0fe6..7dda21e 100644 (file)
--- a/sys/sys/vnode.h
+++ b/sys/sys/vnode.h
@@ -286,7 +286,7 @@ struct vnode {
 #define        VOWANT          0x00020000      /* a process is waiting for VOLOCK */
 #define        VRECLAIMED      0x00040000      /* This vnode has been destroyed */
 #define        VFREE           0x00080000      /* This vnode is on the freelist */
-/* open for business    0x00100000 */
+#define VNOTSEEKABLE   0x00100000      /* rd/wr ignores file offset */
 #define        VONWORKLST      0x00200000      /* On syncer work-list */
 #define VMOUNT         0x00400000      /* Mount in progress */
 #define        VOBJDIRTY       0x00800000      /* object might be dirty */

diff --git a/sys/vfs/fifofs/fifo_vnops.c b/sys/vfs/fifofs/fifo_vnops.c
index 7b16737..9eeca07 100644 (file)
--- a/sys/vfs/fifofs/fifo_vnops.c
+++ b/sys/vfs/fifofs/fifo_vnops.c
@@ -255,6 +255,7 @@ fifo_open(struct vop_open_args *ap)
                        }
                }
        }
+       vp->v_flag |= VNOTSEEKABLE;
        return (vop_stdopen(ap));
 bad:
        vop_stdopen(ap);        /* bump opencount/writecount as appropriate */

diff --git a/sys/vfs/specfs/spec_vnops.c b/sys/vfs/specfs/spec_vnops.c
index c15e097..7c34cad 100644 (file)
--- a/sys/vfs/specfs/spec_vnops.c
+++ b/sys/vfs/specfs/spec_vnops.c
@@ -248,6 +248,8 @@ spec_open(struct vop_open_args *ap)
        /* XXX: Special casing of ttys for deadfs.  Probably redundant */
        if (dev_dflags(dev) & D_TTY)
                vp->v_flag |= VISTTY;
+       if ((dev_dflags(dev) & D_SEEKABLE) == 0)
+               vp->v_flag |= VNOTSEEKABLE;
 
        /*
         * dev_dopen() is always called for each open.  dev_dclose() is