From 0202303b67456174b88e4b40807faec651bd38ef Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Mon, 13 Jul 2009 10:55:24 -0700 Subject: [PATCH] buffer cache - Control all access to the buf red-black trees with vp->v_token Access to the buffer cache's RB trees is now controlled via vp->v_token instead of a critical section. We still hold the BGL but this is not quite as simple as it seems because an interrupt calling biodone() on a B_INVAL buffer may now potentially block, where as it would not have before. The buffer is locked. --- sys/kern/vfs_bio.c | 8 ++- sys/kern/vfs_subr.c | 93 ++++++++++++++++----------------- sys/kern/vfs_sync.c | 41 +++++++++------ sys/sys/vnode.h | 6 +-- sys/vfs/gnu/ext2fs/ext2_vnops.c | 17 +++--- sys/vfs/hammer/hammer_vnops.c | 12 ++--- sys/vfs/nfs/nfs_subs.c | 30 ++++++----- sys/vfs/nfs/nfs_vnops.c | 4 +- sys/vfs/ufs/ffs_rawread.c | 24 ++++----- sys/vfs/ufs/ffs_softdep.c | 12 +++++ sys/vfs/ufs/ufs_vnops.c | 12 ++--- 11 files changed, 144 insertions(+), 115 deletions(-) diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c index c1e59f3c0e..9f587e92a6 100644 --- a/sys/kern/vfs_bio.c +++ b/sys/kern/vfs_bio.c @@ -2468,15 +2468,19 @@ vfs_setdirty(struct buf *bp) * not exist. Do not attempt to lock the buffer or manipulate it in * any way. The caller must validate that the correct buffer has been * obtain after locking it. + * + * */ struct buf * findblk(struct vnode *vp, off_t loffset) { + lwkt_tokref vlock; struct buf *bp; - crit_enter(); + lwkt_gettoken(&vlock, &vp->v_token); +/* ASSERT_LWKT_TOKEN_HELD(&vp->v_token);*/ bp = buf_rb_hash_RB_LOOKUP(&vp->v_rbhash_tree, loffset); - crit_exit(); + lwkt_reltoken(&vlock); return(bp); } diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c index b0a5e916e3..b063ead413 100644 --- a/sys/kern/vfs_subr.c +++ b/sys/kern/vfs_subr.c @@ -284,25 +284,23 @@ int vinvalbuf(struct vnode *vp, int flags, int slpflag, int slptimeo) { struct vinvalbuf_bp_info info; - int error; vm_object_t object; + lwkt_tokref vlock; + int error; + + lwkt_gettoken(&vlock, &vp->v_token); /* * If we are being asked to save, call fsync to ensure that the inode * is updated. */ if (flags & V_SAVE) { - crit_enter(); error = bio_track_wait(&vp->v_track_write, slpflag, slptimeo); - if (error) { - crit_exit(); + if (error) goto done; - } if (!RB_EMPTY(&vp->v_rbdirty_tree)) { - crit_exit(); if ((error = VOP_FSYNC(vp, MNT_WAIT)) != 0) goto done; - crit_enter(); /* * Dirty bufs may be left or generated via races @@ -316,9 +314,7 @@ vinvalbuf(struct vnode *vp, int flags, int slpflag, int slptimeo) panic("vinvalbuf: dirty bufs"); } } - crit_exit(); } - crit_enter(); info.slptimeo = slptimeo; info.lkflags = LK_EXCLUSIVE | LK_SLEEPFAIL; if (slpflag & PCATCH) @@ -330,7 +326,7 @@ vinvalbuf(struct vnode *vp, int flags, int slpflag, int slptimeo) * Flush the buffer cache until nothing is left. */ while (!RB_EMPTY(&vp->v_rbclean_tree) || - !RB_EMPTY(&vp->v_rbdirty_tree)) { + !RB_EMPTY(&vp->v_rbdirty_tree)) { error = RB_SCAN(buf_rb_tree, &vp->v_rbclean_tree, NULL, vinvalbuf_bp, &info); if (error == 0) { @@ -351,8 +347,6 @@ vinvalbuf(struct vnode *vp, int flags, int slpflag, int slptimeo) } } while (bio_track_active(&vp->v_track_write)); - crit_exit(); - /* * Destroy the copy in the VM cache, too. */ @@ -367,6 +361,7 @@ vinvalbuf(struct vnode *vp, int flags, int slpflag, int slptimeo) panic("vinvalbuf: flush failed, buffers still present"); error = 0; done: + lwkt_reltoken(&vlock); return (error); } @@ -450,8 +445,9 @@ int vtruncbuf(struct vnode *vp, off_t length, int blksize) { off_t truncloffset; - int count; const char *filename; + lwkt_tokref vlock; + int count; /* * Round up to the *next* block, then destroy the buffers in question. @@ -463,7 +459,7 @@ vtruncbuf(struct vnode *vp, off_t length, int blksize) else truncloffset = length; - crit_enter(); + lwkt_gettoken(&vlock, &vp->v_token); do { count = RB_SCAN(buf_rb_tree, &vp->v_rbclean_tree, vtruncbuf_bp_trunc_cmp, @@ -489,23 +485,17 @@ vtruncbuf(struct vnode *vp, off_t length, int blksize) /* * Clean out any left over VM backing store. - */ - crit_exit(); - - vnode_pager_setsize(vp, length); - - crit_enter(); - - /* + * * It is possible to have in-progress I/O from buffers that were * not part of the truncation. This should not happen if we * are truncating to 0-length. */ + vnode_pager_setsize(vp, length); + bio_track_wait(&vp->v_track_write, 0, 0); + filename = TAILQ_FIRST(&vp->v_namecache) ? TAILQ_FIRST(&vp->v_namecache)->nc_name : "?"; - bio_track_wait(&vp->v_track_write, 0, 0); - /* * Make sure no buffers were instantiated while we were trying * to clean out the remaining VM pages. This could occur due @@ -524,7 +514,7 @@ vtruncbuf(struct vnode *vp, off_t length, int blksize) } } while(count); - crit_exit(); + lwkt_reltoken(&vlock); return (0); } @@ -640,6 +630,7 @@ vfsync(struct vnode *vp, int waitfor, int passes, int (*waitoutput)(struct vnode *, struct thread *)) { struct vfsync_info info; + lwkt_tokref vlock; int error; bzero(&info, sizeof(info)); @@ -647,7 +638,7 @@ vfsync(struct vnode *vp, int waitfor, int passes, if ((info.checkdef = checkdef) == NULL) info.syncdeps = 1; - crit_enter_id("vfsync"); + lwkt_gettoken(&vlock, &vp->v_token); switch(waitfor) { case MNT_LAZY: @@ -697,7 +688,8 @@ vfsync(struct vnode *vp, int waitfor, int passes, kprintf("Warning: vfsync skipped %d dirty bufs in pass2!\n", info.skippedbufs); } while (error == 0 && passes > 0 && - !RB_EMPTY(&vp->v_rbdirty_tree)) { + !RB_EMPTY(&vp->v_rbdirty_tree) + ) { if (--passes == 0) { info.synchronous = 1; info.syncdeps = 1; @@ -712,7 +704,7 @@ vfsync(struct vnode *vp, int waitfor, int passes, } break; } - crit_exit_id("vfsync"); + lwkt_reltoken(&vlock); return(error); } @@ -805,9 +797,7 @@ vfsync_bp(struct buf *bp, void *data) * Synchronous flushing. An error may be returned. */ bremfree(bp); - crit_exit_id("vfsync"); error = bwrite(bp); - crit_enter_id("vfsync"); } else { /* * Asynchronous flushing. A negative return value simply @@ -820,9 +810,7 @@ vfsync_bp(struct buf *bp, void *data) } else { info->lazycount += bp->b_bufsize; bremfree(bp); - crit_exit_id("vfsync"); bawrite(bp); - crit_enter_id("vfsync"); } if (info->lazylimit && info->lazycount >= info->lazylimit) error = 1; @@ -838,14 +826,20 @@ vfsync_bp(struct buf *bp, void *data) void bgetvp(struct vnode *vp, struct buf *bp) { + lwkt_tokref vlock; + KASSERT(bp->b_vp == NULL, ("bgetvp: not free")); KKASSERT((bp->b_flags & (B_HASHED|B_DELWRI|B_VNCLEAN|B_VNDIRTY)) == 0); + /* + * vp is held for each bp associated with it. + */ vhold(vp); + /* * Insert onto list for new vnode. */ - crit_enter(); + lwkt_gettoken(&vlock, &vp->v_token); bp->b_vp = vp; bp->b_flags |= B_HASHED; if (buf_rb_hash_RB_INSERT(&vp->v_rbhash_tree, bp)) @@ -854,7 +848,7 @@ bgetvp(struct vnode *vp, struct buf *bp) bp->b_flags |= B_VNCLEAN; if (buf_rb_tree_RB_INSERT(&vp->v_rbclean_tree, bp)) panic("reassignbuf: dup lblk/clean vp %p bp %p", vp, bp); - crit_exit(); + lwkt_reltoken(&vlock); } /* @@ -864,6 +858,7 @@ void brelvp(struct buf *bp) { struct vnode *vp; + lwkt_tokref vlock; KASSERT(bp->b_vp != NULL, ("brelvp: NULL")); @@ -871,7 +866,7 @@ brelvp(struct buf *bp) * Delete from old vnode list, if on one. */ vp = bp->b_vp; - crit_enter(); + lwkt_gettoken(&vlock, &vp->v_token); if (bp->b_flags & (B_VNDIRTY | B_VNCLEAN)) { if (bp->b_flags & B_VNDIRTY) buf_rb_tree_RB_REMOVE(&vp->v_rbdirty_tree, bp); @@ -887,8 +882,9 @@ brelvp(struct buf *bp) vp->v_flag &= ~VONWORKLST; LIST_REMOVE(vp, v_synclist); } - crit_exit(); bp->b_vp = NULL; + lwkt_reltoken(&vlock); + vdrop(vp); } @@ -900,6 +896,7 @@ void reassignbuf(struct buf *bp) { struct vnode *vp = bp->b_vp; + lwkt_tokref vlock; int delay; KKASSERT(vp != NULL); @@ -912,7 +909,7 @@ reassignbuf(struct buf *bp) if (bp->b_flags & B_PAGING) panic("cannot reassign paging buffer"); - crit_enter(); + lwkt_gettoken(&vlock, &vp->v_token); if (bp->b_flags & B_DELWRI) { /* * Move to the dirty list, add the vnode to the worklist @@ -968,7 +965,7 @@ reassignbuf(struct buf *bp) LIST_REMOVE(vp, v_synclist); } } - crit_exit(); + lwkt_reltoken(&vlock); } /* @@ -1974,11 +1971,11 @@ vfs_msync_scan2(struct mount *mp, struct vnode *vp, void *data) int vn_pollrecord(struct vnode *vp, int events) { - lwkt_tokref ilock; + lwkt_tokref vlock; KKASSERT(curthread->td_proc != NULL); - lwkt_gettoken(&ilock, &vp->v_token); + lwkt_gettoken(&vlock, &vp->v_token); if (vp->v_pollinfo.vpi_revents & events) { /* * This leaves events we are not interested @@ -1990,12 +1987,12 @@ vn_pollrecord(struct vnode *vp, int events) events &= vp->v_pollinfo.vpi_revents; vp->v_pollinfo.vpi_revents &= ~events; - lwkt_reltoken(&ilock); + lwkt_reltoken(&vlock); return events; } vp->v_pollinfo.vpi_events |= events; selrecord(curthread, &vp->v_pollinfo.vpi_selinfo); - lwkt_reltoken(&ilock); + lwkt_reltoken(&vlock); return 0; } @@ -2008,9 +2005,9 @@ vn_pollrecord(struct vnode *vp, int events) void vn_pollevent(struct vnode *vp, int events) { - lwkt_tokref ilock; + lwkt_tokref vlock; - lwkt_gettoken(&ilock, &vp->v_token); + lwkt_gettoken(&vlock, &vp->v_token); if (vp->v_pollinfo.vpi_events & events) { /* * We clear vpi_events so that we don't @@ -2027,7 +2024,7 @@ vn_pollevent(struct vnode *vp, int events) vp->v_pollinfo.vpi_revents |= events; selwakeup(&vp->v_pollinfo.vpi_selinfo); } - lwkt_reltoken(&ilock); + lwkt_reltoken(&vlock); } /* @@ -2038,14 +2035,14 @@ vn_pollevent(struct vnode *vp, int events) void vn_pollgone(struct vnode *vp) { - lwkt_tokref ilock; + lwkt_tokref vlock; - lwkt_gettoken(&ilock, &vp->v_token); + lwkt_gettoken(&vlock, &vp->v_token); if (vp->v_pollinfo.vpi_events) { vp->v_pollinfo.vpi_events = 0; selwakeup(&vp->v_pollinfo.vpi_selinfo); } - lwkt_reltoken(&ilock); + lwkt_reltoken(&vlock); } /* diff --git a/sys/kern/vfs_sync.c b/sys/kern/vfs_sync.c index cdcfac5351..233a2f1ec5 100644 --- a/sys/kern/vfs_sync.c +++ b/sys/kern/vfs_sync.c @@ -107,6 +107,7 @@ SYSCTL_INT(_debug, OID_AUTO, rush_requests, CTLFLAG_RW, static int syncer_delayno = 0; static long syncer_mask; +static struct lwkt_token syncer_token; LIST_HEAD(synclist, vnode); static struct synclist *syncer_workitem_pending; @@ -119,6 +120,7 @@ vfs_sync_init(void) syncer_workitem_pending = hashinit(syncer_maxdelay, M_DEVBUF, &syncer_mask); syncer_maxdelay = syncer_mask + 1; + lwkt_token_init(&syncer_token); } /* @@ -153,21 +155,21 @@ vfs_sync_init(void) void vn_syncer_add_to_worklist(struct vnode *vp, int delay) { + lwkt_tokref ilock; int slot; - crit_enter(); + lwkt_gettoken(&ilock, &syncer_token); - if (vp->v_flag & VONWORKLST) { + if (vp->v_flag & VONWORKLST) LIST_REMOVE(vp, v_synclist); - } - if (delay > syncer_maxdelay - 2) delay = syncer_maxdelay - 2; slot = (syncer_delayno + delay) & syncer_mask; LIST_INSERT_HEAD(&syncer_workitem_pending[slot], vp, v_synclist); vp->v_flag |= VONWORKLST; - crit_exit(); + + lwkt_reltoken(&ilock); } struct thread *updatethread; @@ -185,10 +187,12 @@ SYSINIT(syncer, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, &up_kp) void sched_sync(void) { + struct thread *td = curthread; struct synclist *slp; struct vnode *vp; + lwkt_tokref ilock; + lwkt_tokref vlock; long starttime; - struct thread *td = curthread; EVENTHANDLER_REGISTER(shutdown_pre_sync, shutdown_kproc, td, SHUTDOWN_PRI_LAST); @@ -197,24 +201,22 @@ sched_sync(void) kproc_suspend_loop(); starttime = time_second; + lwkt_gettoken(&ilock, &syncer_token); /* * Push files whose dirty time has expired. Be careful * of interrupt race on slp queue. */ - crit_enter(); slp = &syncer_workitem_pending[syncer_delayno]; syncer_delayno += 1; if (syncer_delayno == syncer_maxdelay) syncer_delayno = 0; - crit_exit(); while ((vp = LIST_FIRST(slp)) != NULL) { if (vget(vp, LK_EXCLUSIVE | LK_NOWAIT) == 0) { VOP_FSYNC(vp, MNT_LAZY); vput(vp); } - crit_enter(); /* * If the vnode is still at the head of the list @@ -228,14 +230,20 @@ sched_sync(void) * here. */ if (LIST_FIRST(slp) == vp) { - if (RB_EMPTY(&vp->v_rbdirty_tree) && - !vn_isdisk(vp, NULL)) { - panic("sched_sync: fsync failed vp %p tag %d", vp, vp->v_tag); + lwkt_gettoken(&vlock, &vp->v_token); + if (LIST_FIRST(slp) == vp) { + if (RB_EMPTY(&vp->v_rbdirty_tree) && + !vn_isdisk(vp, NULL)) { + panic("sched_sync: fsync " + "failed vp %p tag %d", + vp, vp->v_tag); + } + vn_syncer_add_to_worklist(vp, syncdelay); } - vn_syncer_add_to_worklist(vp, syncdelay); + lwkt_reltoken(&vlock); } - crit_exit(); } + lwkt_reltoken(&ilock); /* * Do sync processing for each mount. @@ -441,14 +449,15 @@ static int sync_reclaim(struct vop_reclaim_args *ap) { struct vnode *vp = ap->a_vp; + lwkt_tokref ilock; - crit_enter(); + lwkt_gettoken(&ilock, &syncer_token); KKASSERT(vp->v_mount->mnt_syncer != vp); if (vp->v_flag & VONWORKLST) { LIST_REMOVE(vp, v_synclist); vp->v_flag &= ~VONWORKLST; } - crit_exit(); + lwkt_reltoken(&ilock); return (0); } diff --git a/sys/sys/vnode.h b/sys/sys/vnode.h index e1534eaf21..955c2537a9 100644 --- a/sys/sys/vnode.h +++ b/sys/sys/vnode.h @@ -180,9 +180,9 @@ vrange_lock_excl(struct vnode *vp, struct vrangelock *vr, * * Certain fields within the vnode structure requires v_token to be held. * - * v_rbclean_tree (upcoming) - * v_rbdirty_tree (upcoming) - * v_rbhash_tree (upcoming) + * v_rbclean_tree + * v_rbdirty_tree + * v_rbhash_tree * v_pollinfo * * NOTE: The vnode operations vector, v_ops, is a double-indirect that diff --git a/sys/vfs/gnu/ext2fs/ext2_vnops.c b/sys/vfs/gnu/ext2fs/ext2_vnops.c index 7363ae4cf8..3baa61c6ae 100644 --- a/sys/vfs/gnu/ext2fs/ext2_vnops.c +++ b/sys/vfs/gnu/ext2fs/ext2_vnops.c @@ -210,6 +210,7 @@ ext2_fsync(struct vop_fsync_args *ap) { struct ext2_fsync_bp_info info; struct vnode *vp = ap->a_vp; + lwkt_tokref vlock; int count; /* @@ -221,7 +222,7 @@ ext2_fsync(struct vop_fsync_args *ap) */ ext2_discard_prealloc(VTOI(vp)); - crit_enter(); + lwkt_gettoken(&vlock, &vp->v_token); info.vp = vp; loop: info.waitfor = ap->a_waitfor; @@ -239,7 +240,7 @@ loop: } #endif } - crit_exit(); + lwkt_reltoken(&vlock); return (EXT2_UPDATE(ap->a_vp, ap->a_waitfor == MNT_WAIT)); } @@ -2079,7 +2080,7 @@ ext2_kqfilter(struct vop_kqfilter_args *ap) { struct vnode *vp = ap->a_vp; struct knote *kn = ap->a_kn; - lwkt_tokref ilock; + lwkt_tokref vlock; switch (kn->kn_filter) { case EVFILT_READ: @@ -2097,9 +2098,9 @@ ext2_kqfilter(struct vop_kqfilter_args *ap) kn->kn_hook = (caddr_t)vp; - lwkt_gettoken(&ilock, &vp->v_token); + lwkt_gettoken(&vlock, &vp->v_token); SLIST_INSERT_HEAD(&vp->v_pollinfo.vpi_selinfo.si_note, kn, kn_selnext); - lwkt_reltoken(&ilock); + lwkt_reltoken(&vlock); return (0); } @@ -2108,12 +2109,12 @@ static void filt_ext2detach(struct knote *kn) { struct vnode *vp = (struct vnode *)kn->kn_hook; - lwkt_tokref ilock; + lwkt_tokref vlock; - lwkt_gettoken(&ilock, &vp->v_token); + lwkt_gettoken(&vlock, &vp->v_token); SLIST_REMOVE(&vp->v_pollinfo.vpi_selinfo.si_note, kn, knote, kn_selnext); - lwkt_reltoken(&ilock); + lwkt_reltoken(&vlock); } /*ARGSUSED*/ diff --git a/sys/vfs/hammer/hammer_vnops.c b/sys/vfs/hammer/hammer_vnops.c index 30d8bef97c..b73f67c0e8 100644 --- a/sys/vfs/hammer/hammer_vnops.c +++ b/sys/vfs/hammer/hammer_vnops.c @@ -3051,7 +3051,7 @@ hammer_vop_kqfilter(struct vop_kqfilter_args *ap) { struct vnode *vp = ap->a_vp; struct knote *kn = ap->a_kn; - lwkt_tokref ilock; + lwkt_tokref vlock; switch (kn->kn_filter) { case EVFILT_READ: @@ -3069,9 +3069,9 @@ hammer_vop_kqfilter(struct vop_kqfilter_args *ap) kn->kn_hook = (caddr_t)vp; - lwkt_gettoken(&ilock, &vp->v_token); + lwkt_gettoken(&vlock, &vp->v_token); SLIST_INSERT_HEAD(&vp->v_pollinfo.vpi_selinfo.si_note, kn, kn_selnext); - lwkt_reltoken(&ilock); + lwkt_reltoken(&vlock); return(0); } @@ -3080,12 +3080,12 @@ static void filt_hammerdetach(struct knote *kn) { struct vnode *vp = (void *)kn->kn_hook; - lwkt_tokref ilock; + lwkt_tokref vlock; - lwkt_gettoken(&ilock, &vp->v_token); + lwkt_gettoken(&vlock, &vp->v_token); SLIST_REMOVE(&vp->v_pollinfo.vpi_selinfo.si_note, kn, knote, kn_selnext); - lwkt_reltoken(&ilock); + lwkt_reltoken(&vlock); } static int diff --git a/sys/vfs/nfs/nfs_subs.c b/sys/vfs/nfs/nfs_subs.c index 64274f85f8..f80d0cf912 100644 --- a/sys/vfs/nfs/nfs_subs.c +++ b/sys/vfs/nfs/nfs_subs.c @@ -2025,22 +2025,28 @@ nfs_setvtype(struct vnode *vp, enum vtype vtyp) */ static int nfs_clearcommit_bp(struct buf *bp, void *data __unused); +static int nfs_clearcommit_callback(struct mount *mp, struct vnode *vp, + void *data __unused); void nfs_clearcommit(struct mount *mp) { - struct vnode *vp, *nvp; - lwkt_tokref ilock; - - lwkt_gettoken(&ilock, &mntvnode_token); - crit_enter(); - for (vp = TAILQ_FIRST(&mp->mnt_nvnodelist); vp; vp = nvp) { - nvp = TAILQ_NEXT(vp, v_nmntvnodes); /* ZZZ */ - RB_SCAN(buf_rb_tree, &vp->v_rbdirty_tree, NULL, - nfs_clearcommit_bp, NULL); - } - crit_exit(); - lwkt_reltoken(&ilock); + vmntvnodescan(mp, VMSC_NOWAIT, nfs_clearcommit_callback, NULL, NULL); +} + +static int +nfs_clearcommit_callback(struct mount *mp, struct vnode *vp, + void *data __unused) +{ + lwkt_tokref vlock; + + vhold(vp); + lwkt_gettoken(&vlock, &vp->v_token); + RB_SCAN(buf_rb_tree, &vp->v_rbdirty_tree, NULL, + nfs_clearcommit_bp, NULL); + lwkt_reltoken(&vlock); + vdrop(vp); + return(0); } static int diff --git a/sys/vfs/nfs/nfs_vnops.c b/sys/vfs/nfs/nfs_vnops.c index 0bc5da8c22..31b08f9f68 100644 --- a/sys/vfs/nfs/nfs_vnops.c +++ b/sys/vfs/nfs/nfs_vnops.c @@ -2974,6 +2974,7 @@ nfs_flush(struct vnode *vp, int waitfor, struct thread *td, int commit) struct nfsnode *np = VTONFS(vp); struct nfsmount *nmp = VFSTONFS(vp->v_mount); struct nfs_flush_info info; + lwkt_tokref vlock; int error; bzero(&info, sizeof(info)); @@ -2982,6 +2983,7 @@ nfs_flush(struct vnode *vp, int waitfor, struct thread *td, int commit) info.waitfor = waitfor; info.slpflag = (nmp->nm_flag & NFSMNT_INT) ? PCATCH : 0; info.loops = 0; + lwkt_gettoken(&vlock, &vp->v_token); do { /* @@ -3057,10 +3059,10 @@ nfs_flush(struct vnode *vp, int waitfor, struct thread *td, int commit) error = np->n_error; np->n_flag &= ~NWRITEERR; } + lwkt_reltoken(&vlock); return (error); } - static int nfs_flush_bp(struct buf *bp, void *data) diff --git a/sys/vfs/ufs/ffs_rawread.c b/sys/vfs/ufs/ffs_rawread.c index 7e374ff41a..2af261240a 100644 --- a/sys/vfs/ufs/ffs_rawread.c +++ b/sys/vfs/ufs/ffs_rawread.c @@ -92,14 +92,15 @@ ffs_rawread_sync(struct vnode *vp) { int error; int upgraded; + lwkt_tokref vlock; - /* Check for dirty mmap, pending writes and dirty buffers */ - crit_enter(); + /* + * Check for dirty mmap, pending writes and dirty buffers + */ + lwkt_gettoken(&vlock, &vp->v_token); if (bio_track_active(&vp->v_track_write) || !RB_EMPTY(&vp->v_rbdirty_tree) || (vp->v_flag & VOBJDIRTY) != 0) { - crit_exit(); - if (vn_islocked(vp) != LK_EXCLUSIVE) { upgraded = 1; /* Upgrade to exclusive lock, this might block */ @@ -115,34 +116,31 @@ ffs_rawread_sync(struct vnode *vp) } /* Wait for pending writes to complete */ - crit_enter(); error = bio_track_wait(&vp->v_track_write, 0, 0); if (error != 0) { - crit_exit(); if (upgraded != 0) vn_lock(vp, LK_DOWNGRADE); - return (error); + goto done; } /* Flush dirty buffers */ if (!RB_EMPTY(&vp->v_rbdirty_tree)) { - crit_exit(); if ((error = VOP_FSYNC(vp, MNT_WAIT)) != 0) { if (upgraded != 0) vn_lock(vp, LK_DOWNGRADE); - return (error); + goto done; } - crit_enter(); if (bio_track_active(&vp->v_track_write) || !RB_EMPTY(&vp->v_rbdirty_tree)) panic("ffs_rawread_sync: dirty bufs"); } - crit_exit(); if (upgraded != 0) vn_lock(vp, LK_DOWNGRADE); } else { - crit_exit(); + error = 0; } - return 0; +done: + lwkt_reltoken(&vlock); + return error; } diff --git a/sys/vfs/ufs/ffs_softdep.c b/sys/vfs/ufs/ffs_softdep.c index b14c876274..e0af38e90c 100644 --- a/sys/vfs/ufs/ffs_softdep.c +++ b/sys/vfs/ufs/ffs_softdep.c @@ -1816,6 +1816,7 @@ softdep_setup_freeblocks(struct inode *ip, off_t length) struct freeblks *freeblks; struct inodedep *inodedep; struct allocdirect *adp; + lwkt_tokref vlock; struct vnode *vp; struct buf *bp; struct fs *fs; @@ -1900,10 +1901,13 @@ softdep_setup_freeblocks(struct inode *ip, off_t length) info.fs = fs; info.ip = ip; + lwkt_gettoken(&vlock, &vp->v_token); do { count = RB_SCAN(buf_rb_tree, &vp->v_rbdirty_tree, NULL, softdep_setup_freeblocks_bp, &info); } while (count != 0); + lwkt_reltoken(&vlock); + if (inodedep_lookup(fs, ip->i_number, 0, &inodedep) != 0) (void)free_inodedep(inodedep); @@ -4204,11 +4208,15 @@ static int softdep_fsync_mountdev_bp(struct buf *bp, void *data); void softdep_fsync_mountdev(struct vnode *vp) { + lwkt_tokref vlock; + if (!vn_isdisk(vp, NULL)) panic("softdep_fsync_mountdev: vnode not a disk"); ACQUIRE_LOCK(&lk); + lwkt_gettoken(&vlock, &vp->v_token); RB_SCAN(buf_rb_tree, &vp->v_rbdirty_tree, NULL, softdep_fsync_mountdev_bp, vp); + lwkt_reltoken(&vlock); drain_output(vp, 1); FREE_LOCK(&lk); } @@ -4262,6 +4270,7 @@ int softdep_sync_metadata(struct vnode *vp, struct thread *td) { struct softdep_sync_metadata_info info; + lwkt_tokref vlock; int error, waitfor; /* @@ -4307,10 +4316,13 @@ top: * all potential buffers on the dirty list will be visible. */ drain_output(vp, 1); + info.vp = vp; info.waitfor = waitfor; + lwkt_gettoken(&vlock, &vp->v_token); error = RB_SCAN(buf_rb_tree, &vp->v_rbdirty_tree, NULL, softdep_sync_metadata_bp, &info); + lwkt_reltoken(&vlock); if (error < 0) { FREE_LOCK(&lk); return(-error); /* error code */ diff --git a/sys/vfs/ufs/ufs_vnops.c b/sys/vfs/ufs/ufs_vnops.c index 352b3be6d4..98b08923ba 100644 --- a/sys/vfs/ufs/ufs_vnops.c +++ b/sys/vfs/ufs/ufs_vnops.c @@ -2280,7 +2280,7 @@ ufs_kqfilter(struct vop_kqfilter_args *ap) { struct vnode *vp = ap->a_vp; struct knote *kn = ap->a_kn; - lwkt_tokref ilock; + lwkt_tokref vlock; switch (kn->kn_filter) { case EVFILT_READ: @@ -2298,9 +2298,9 @@ ufs_kqfilter(struct vop_kqfilter_args *ap) kn->kn_hook = (caddr_t)vp; - lwkt_gettoken(&ilock, &vp->v_token); + lwkt_gettoken(&vlock, &vp->v_token); SLIST_INSERT_HEAD(&vp->v_pollinfo.vpi_selinfo.si_note, kn, kn_selnext); - lwkt_reltoken(&ilock); + lwkt_reltoken(&vlock); return (0); } @@ -2309,12 +2309,12 @@ static void filt_ufsdetach(struct knote *kn) { struct vnode *vp = (struct vnode *)kn->kn_hook; - lwkt_tokref ilock; + lwkt_tokref vlock; - lwkt_gettoken(&ilock, &vp->v_token); + lwkt_gettoken(&vlock, &vp->v_token); SLIST_REMOVE(&vp->v_pollinfo.vpi_selinfo.si_note, kn, knote, kn_selnext); - lwkt_reltoken(&ilock); + lwkt_reltoken(&vlock); } /*ARGSUSED*/ -- 2.41.0