kernel - Implement KVABIO API in TMPFS
authorMatthew Dillon <dillon@apollo.backplane.com>
Mon, 2 Oct 2017 02:42:59 +0000 (19:42 -0700)
committerMatthew Dillon <dillon@apollo.backplane.com>
Mon, 16 Oct 2017 18:30:21 +0000 (11:30 -0700)
* TMPFS now fully supports the KVABIO API.  This removes nearly all
  IPIs from buffer cache operations related to TMPFS.

* In synth tests on 32-way and 48-way servers, the number of IPIs/cpu/sec
  drops from 5000-12000 down to 200-1000.  Needless to say, this is a
  huge win, particularly on VMs.

Recommend-by: mjg_ (Mateusz Guzik)
sys/vfs/tmpfs/tmpfs_subr.c
sys/vfs/tmpfs/tmpfs_vnops.c

index 0c46d82..924aae1 100644 (file)
@@ -441,6 +441,11 @@ loop:
        case VSOCK:
                break;
        case VREG:
+               /*
+                * VMIO is mandatory.  Tmpfs also supports KVABIO
+                * for its tmpfs_strategy().
+                */
+               vsetflags(vp, VKVABIO);
                vinitvmio(vp, node->tn_size, TMPFS_BLKSIZE, -1);
                break;
        case VLNK:
index 048f989..dfcae60 100644 (file)
@@ -484,9 +484,10 @@ tmpfs_read(struct vop_read_args *ap)
                 */
                offset = (size_t)uio->uio_offset & TMPFS_BLKMASK64;
                base_offset = (off_t)uio->uio_offset - offset;
-               bp = getcacheblk(vp, base_offset, TMPFS_BLKSIZE, 0);
+               bp = getcacheblk(vp, base_offset, TMPFS_BLKSIZE, GETBLK_KVABIO);
                if (bp == NULL) {
-                       error = bread(vp, base_offset, TMPFS_BLKSIZE, &bp);
+                       error = bread_kvabio(vp, base_offset,
+                                            TMPFS_BLKSIZE, &bp);
                        if (error) {
                                brelse(bp);
                                kprintf("tmpfs_read bread error %d\n", error);
@@ -633,7 +634,7 @@ tmpfs_write(struct vop_write_args *ap)
                 *
                 * So just use bread() to do the right thing.
                 */
-               error = bread(vp, base_offset, TMPFS_BLKSIZE, &bp);
+               error = bread_kvabio(vp, base_offset, TMPFS_BLKSIZE, &bp);
                error = uiomovebp(bp, (char *)bp->b_data + offset, len, uio);
                if (error) {
                        kprintf("tmpfs_write uiomove error %d\n", error);
@@ -774,6 +775,10 @@ tmpfs_advlock(struct vop_advlock_args *ap)
  * forces the system to attempt to pageout pages.  It can also be called
  * by [n]vtruncbuf() when a truncation cuts a page in half.  Normal write
  * operations
+ *
+ * We set VKVABIO for VREG files so bp->b_data may not be synchronized to
+ * our cpu.  swap_pager_strategy() is all we really use, and it directly
+ * supports this.
  */
 static int
 tmpfs_strategy(struct vop_strategy_args *ap)