From: Matthew Dillon Date: Mon, 23 Mar 2015 03:26:08 +0000 (-0700) Subject: hammer2 - locking revamp X-Git-Tag: v4.2.0rc~467 X-Git-Url: https://gitweb.dragonflybsd.org/dragonfly.git/commitdiff_plain/94491fa098a9b62dd517fe5e5f0c8bf3315b9d5a hammer2 - locking revamp * Temporarily remove hammer2_ccms.c from the build and remove the ccms container. The CCMS container will be used again when we get cache coherent in, but for now it isn't needed. * Replace the ccms state lock with a mtx lock and move into hammer2_chain_core. Note that the mtx lock being used here has abort and async locking support and these features will be required by HAMMER2. * Replace the ccms spin lock with a normal spin lock and move into hammer2_chain_core. * Refactor the OS locking interface to use hammer2_* prefixes for easier porting. * Use a shared spin lock for the ONFLUSH bit update instead of an exclusive spin lock. --- diff --git a/sys/vfs/hammer2/Makefile b/sys/vfs/hammer2/Makefile index a06504cfe3..560f507db8 100644 --- a/sys/vfs/hammer2/Makefile +++ b/sys/vfs/hammer2/Makefile @@ -5,7 +5,7 @@ CFLAGS+= -DINVARIANTS -DSMP KMOD= hammer2 -SRCS= hammer2_vfsops.c hammer2_vnops.c hammer2_inode.c hammer2_ccms.c +SRCS= hammer2_vfsops.c hammer2_vnops.c hammer2_inode.c SRCS+= hammer2_chain.c hammer2_flush.c hammer2_freemap.c hammer2_cluster.c SRCS+= hammer2_ioctl.c hammer2_msgops.c hammer2_subr.c hammer2_bulkscan.c SRCS+= hammer2_lz4.c hammer2_io.c hammer2_iocom.c diff --git a/sys/vfs/hammer2/hammer2.h b/sys/vfs/hammer2/hammer2.h index 0ea201d62e..03a9b04e0c 100644 --- a/sys/vfs/hammer2/hammer2.h +++ b/sys/vfs/hammer2/hammer2.h @@ -81,18 +81,17 @@ #include #include #include -#include #include #include #include +#include #include #include #include "hammer2_disk.h" #include "hammer2_mount.h" #include "hammer2_ioctl.h" -#include "hammer2_ccms.h" struct hammer2_io; struct hammer2_iocb; @@ -105,6 +104,64 @@ struct hammer2_span; struct hammer2_state; struct hammer2_msg; +/* + * Mutex and lock shims. Hammer2 requires support for asynchronous and + * abortable locks, and both exclusive and shared spinlocks. Normal + * synchronous non-abortable locks can be substituted for spinlocks. + */ +typedef mtx_t hammer2_mtx_t; +typedef mtx_link_t hammer2_mtx_link_t; +typedef mtx_state_t hammer2_mtx_state_t; + +typedef struct spinlock hammer2_spin_t; + +#define hammer2_mtx_ex mtx_lock_ex_quick +#define hammer2_mtx_sh mtx_lock_sh_quick +#define hammer2_mtx_unlock mtx_unlock +#define hammer2_mtx_owned mtx_owned +#define hammer2_mtx_init mtx_init +#define hammer2_mtx_temp_release mtx_lock_temp_release +#define hammer2_mtx_temp_restore mtx_lock_temp_restore +#define hammer2_mtx_refs mtx_lockrefs + +#define hammer2_spin_init spin_init +#define hammer2_spin_sh spin_lock_shared +#define hammer2_spin_ex spin_lock +#define hammer2_spin_unsh spin_unlock_shared +#define hammer2_spin_unex spin_unlock + +/* + * General lock support + */ +static __inline +int +hammer2_mtx_upgrade(hammer2_mtx_t *mtx) +{ + int wasexclusive; + + if (mtx_islocked_ex(mtx)) { + wasexclusive = 1; + } else { + mtx_unlock(mtx); + mtx_lock_ex_quick(mtx); + wasexclusive = 0; + } + return wasexclusive; +} + +/* + * Downgrade an inode lock from exclusive to shared only if the inode + * lock was previously shared. If the inode lock was previously exclusive, + * this is a NOP. + */ +static __inline +void +hammer2_mtx_downgrade(hammer2_mtx_t *mtx, int wasexclusive) +{ + if (wasexclusive == 0) + mtx_downgrade(mtx); +} + /* * The xid tracks internal transactional updates. * @@ -185,7 +242,8 @@ TAILQ_HEAD(h2_iocb_list, hammer2_iocb); (HAMMER2_PBUFSIZE / sizeof(hammer2_blockref_t) / sizeof(uint32_t)) struct hammer2_chain_core { - struct ccms_cst cst; + hammer2_mtx_t lock; + hammer2_spin_t spin; struct hammer2_chain_tree rbtree; /* sub-chains */ int live_zero; /* blockref array opt */ u_int flags; @@ -409,20 +467,23 @@ RB_PROTOTYPE(hammer2_chain_tree, hammer2_chain, rbnode, hammer2_chain_cmp); /* * HAMMER2 cluster - A set of chains representing the same entity. * - * The hammer2_pfsmount structure embeds a hammer2_cluster. All other - * hammer2_cluster use cases use temporary allocations. + * hammer2_cluster typically represents a temporary set of representitive + * chains. The one exception is that a hammer2_cluster is embedded in + * hammer2_inode. This embedded cluster is ONLY used to track the + * representitive chains and cannot be directly locked. * - * The cluster API mimics the chain API. Except as used in the pfsmount, - * the cluster structure is a temporary 'working copy' of a set of chains - * representing targets compatible with the operation. However, for - * performance reasons the cluster API does not necessarily issue concurrent - * requests to the underlying chain API for all compatible chains all the - * time. This may sometimes necessitate revisiting parent cluster nodes - * to 'flesh out' (validate more chains). + * A cluster is temporary (and thus per-thread) for locking purposes, + * allowing us to embed the asynchronous storage required for + * cluster operations in the cluster itself. That is, except for the + * embeddeding hammer2_inode, the cluster structure will always represent + * a 'working copy'. * - * If an insufficient number of chains remain in a working copy, the operation - * may have to be downgraded, retried, or stall until the requisit number - * of chains are available. + * Because the cluster is a 'working copy' and is usually subject to cluster + * quorum rules, it is quite possible for us to end up with an insufficient + * number of live chains to execute an operation. If an insufficient number + * of chains remain in a working copy, the operation may have to be + * downgraded, retried, or stall until the requisit number of chains are + * available. */ #define HAMMER2_MAXCLUSTER 8 @@ -434,12 +495,12 @@ struct hammer2_cluster { int nchains; hammer2_iocb_t iocb; hammer2_chain_t *focus; /* current focus (or mod) */ + hammer2_mtx_link_t asynclnk[HAMMER2_MAXCLUSTER]; hammer2_chain_t *array[HAMMER2_MAXCLUSTER]; - char missed[HAMMER2_MAXCLUSTER]; int cache_index[HAMMER2_MAXCLUSTER]; }; -typedef struct hammer2_cluster hammer2_cluster_t; +typedef struct hammer2_cluster hammer2_cluster_t; #define HAMMER2_CLUSTER_INODE 0x00000001 /* embedded in inode */ #define HAMMER2_CLUSTER_NOSYNC 0x00000002 /* not in sync (cumulative) */ @@ -450,16 +511,13 @@ RB_HEAD(hammer2_inode_tree, hammer2_inode); /* * A hammer2 inode. * - * NOTE: The inode's attribute CST which is also used to lock the inode - * is embedded in the chain (chain.cst) and aliased w/ attr_cst. - * * NOTE: The inode-embedded cluster is never used directly for I/O (since * it may be shared). Instead it will be replicated-in and synchronized * back out if changed. */ struct hammer2_inode { RB_ENTRY(hammer2_inode) rbnode; /* inumber lookup (HL) */ - ccms_cst_t topo_cst; /* directory topology cst */ + hammer2_mtx_t lock; /* inode lock */ struct hammer2_pfsmount *pmp; /* PFS mount */ struct hammer2_inode *pip; /* parent inode */ struct vnode *vp; @@ -638,7 +696,6 @@ struct hammer2_pfsmount { hammer2_inode_t *ihidden; /* PFS hidden directory */ struct lock lock; /* PFS lock for certain ops */ hammer2_off_t inode_count; /* copy of inode_count */ - ccms_domain_t ccms_dom; struct netexport export; /* nfs export */ int ronly; /* read-only mount */ struct malloc_type *minode; @@ -655,7 +712,7 @@ struct hammer2_pfsmount { struct h2_unlk_list unlinkq; /* last-close unlink */ thread_t wthread_td; /* write thread td */ struct bio_queue_head wthread_bioq; /* logical buffer bioq */ - struct mtx wthread_mtx; /* interlock */ + hammer2_mtx_t wthread_mtx; /* interlock */ int wthread_destroy;/* termination sequencing */ }; @@ -773,8 +830,6 @@ extern struct objcache *cache_buffer_write; extern int destroy; extern int write_thread_wakeup; -extern mtx_t thread_protect; - /* * hammer2_subr.c */ @@ -787,10 +842,11 @@ hammer2_cluster_t *hammer2_inode_lock_nex(hammer2_inode_t *ip, int how); hammer2_cluster_t *hammer2_inode_lock_sh(hammer2_inode_t *ip); void hammer2_inode_unlock_ex(hammer2_inode_t *ip, hammer2_cluster_t *chain); void hammer2_inode_unlock_sh(hammer2_inode_t *ip, hammer2_cluster_t *chain); -ccms_state_t hammer2_inode_lock_temp_release(hammer2_inode_t *ip); -void hammer2_inode_lock_temp_restore(hammer2_inode_t *ip, ccms_state_t ostate); -ccms_state_t hammer2_inode_lock_upgrade(hammer2_inode_t *ip); -void hammer2_inode_lock_downgrade(hammer2_inode_t *ip, ccms_state_t ostate); +hammer2_mtx_state_t hammer2_inode_lock_temp_release(hammer2_inode_t *ip); +void hammer2_inode_lock_temp_restore(hammer2_inode_t *ip, + hammer2_mtx_state_t ostate); +int hammer2_inode_lock_upgrade(hammer2_inode_t *ip); +void hammer2_inode_lock_downgrade(hammer2_inode_t *ip, int); void hammer2_mount_exlock(hammer2_mount_t *hmp); void hammer2_mount_shlock(hammer2_mount_t *hmp); diff --git a/sys/vfs/hammer2/hammer2_ccms.c b/sys/vfs/hammer2/hammer2_ccms.c index d4f810113a..e202814b8f 100644 --- a/sys/vfs/hammer2/hammer2_ccms.c +++ b/sys/vfs/hammer2/hammer2_ccms.c @@ -51,31 +51,11 @@ int ccms_debug = 0; -/* - * Initialize a new CCMS dataspace. Create a new RB tree with a single - * element covering the entire 64 bit offset range. This simplifies - * algorithms enormously by removing a number of special cases. - */ -void -ccms_domain_init(ccms_domain_t *dom) -{ - bzero(dom, sizeof(*dom)); - /*kmalloc_create(&dom->mcst, "CCMS-cst");*/ - /*dom->root.domain = dom;*/ -} - -void -ccms_domain_uninit(ccms_domain_t *dom) -{ - /*kmalloc_destroy(&dom->mcst);*/ -} - void -ccms_cst_init(ccms_cst_t *cst, void *handle) +ccms_cst_init(ccms_cst_t *cst) { bzero(cst, sizeof(*cst)); spin_init(&cst->spin, "ccmscst"); - cst->handle = handle; } void @@ -85,95 +65,8 @@ ccms_cst_uninit(ccms_cst_t *cst) if (cst->state != CCMS_STATE_INVALID) { /* XXX */ } - cst->handle = NULL; -} - -#if 0 -/* - * Acquire an operational CCMS lock on multiple CSTs. - * - * This code is in the critical path and highly streamlined. - */ -void -ccms_lock_get(ccms_lock_t *lock) -{ - ccms_inode_t *cino = lock->cino; - -again: - lock->flags &= ~CCMS_LOCK_FAILED; - - /* - * Acquire all local locks first, then resolve them against the - * remote cache state. Order is important here. - */ - if (lock->req_t) { - KKASSERT(lock->req_d <= lock->req_t); - KKASSERT(lock->req_a <= lock->req_t); - ccms_thread_lock(&cino->topo_cst, lock->req_t); - } - if (lock->req_a) - ccms_thread_lock(&cino->attr_cst, lock->req_a); - if (lock->req_d) - ccms_thread_lock(&cino->data_cst[0], lock->req_d); - - /* - * Once the local locks are established the CST grant state cannot - * be pulled out from under us. However, it is entirely possible - * to deadlock on it so when CST grant state cannot be obtained - * trivially we have to unwind our local locks, then get the state, - * and then loop. - */ - if (lock->req_t > cino->topo_cst.state) { - ccms_rstate_get(lock, &cino->topo_cst, lock->req_t); - } else if (cino->topo_cst.state == CCMS_STATE_INVALID) { - ccms_rstate_get(lock, &cino->topo_cst, CCMS_STATE_ALLOWED); - } else if (cino->topo_cst.state == CCMS_STATE_SHARED && - (lock->req_d > CCMS_STATE_SHARED || - lock->req_a > CCMS_STATE_SHARED)) { - ccms_rstate_get(lock, &cino->topo_cst, CCMS_STATE_ALLOWED); - } - /* else the rstate is compatible */ - - if (lock->req_a > cino->attr_cst.state) - ccms_rstate_get(lock, &cino->attr_cst, lock->req_a); - - if (lock->req_d > cino->data_cst[0].state) - ccms_rstate_get(lock, &cino->data_cst[0], lock->req_d); - - /* - * If the ccms_rstate_get() code deadlocks (or even if it just - * blocks), it will release all local locks and set the FAILED - * bit. The routine will still acquire the requested remote grants - * before returning but since the local locks are lost at that - * point the remote grants are no longer protected and we have to - * retry. - */ - if (lock->flags & CCMS_LOCK_FAILED) { - goto again; - } } -/* - * Release a previously acquired CCMS lock. - */ -void -ccms_lock_put(ccms_lock_t *lock) -{ - ccms_inode_t *cino = lock->cino; - - if (lock->req_d) { - ccms_thread_unlock(&cino->data_cst[0]); - } - if (lock->req_a) { - ccms_thread_unlock(&cino->attr_cst); - } - if (lock->req_t) { - ccms_thread_unlock(&cino->topo_cst); - } -} - -#endif - /************************************************************************ * CST SUPPORT FUNCTIONS * ************************************************************************/ @@ -201,7 +94,7 @@ ccms_thread_lock(ccms_cst_t *cst, ccms_state_t state) * Otherwise use the spinlock to interlock the operation and sleep * as necessary. */ - spin_lock(&cst->spin); + hammer2_spin_ex(&cst->spin); if (state == CCMS_STATE_SHARED) { while (cst->count < 0 || cst->upgrade) { cst->blocked = 1; @@ -217,10 +110,10 @@ ccms_thread_lock(ccms_cst_t *cst, ccms_state_t state) cst->count = -1; cst->td = curthread; } else { - spin_unlock(&cst->spin); + hammer2_spin_unex(&cst->spin); panic("ccms_thread_lock: bad state %d\n", state); } - spin_unlock(&cst->spin); + hammer2_spin_unex(&cst->spin); } /* @@ -236,26 +129,26 @@ ccms_thread_lock_nonblock(ccms_cst_t *cst, ccms_state_t state) return(0); } - spin_lock(&cst->spin); + hammer2_spin_ex(&cst->spin); if (state == CCMS_STATE_SHARED) { if (cst->count < 0 || cst->upgrade) { - spin_unlock(&cst->spin); + hammer2_spin_unex(&cst->spin); return (EBUSY); } ++cst->count; KKASSERT(cst->td == NULL); } else if (state == CCMS_STATE_EXCLUSIVE) { if (cst->count != 0 || cst->upgrade) { - spin_unlock(&cst->spin); + hammer2_spin_unex(&cst->spin); return (EBUSY); } cst->count = -1; cst->td = curthread; } else { - spin_unlock(&cst->spin); + hammer2_spin_unex(&cst->spin); panic("ccms_thread_lock_nonblock: bad state %d\n", state); } - spin_unlock(&cst->spin); + hammer2_spin_unex(&cst->spin); LOCKENTER; return(0); } @@ -300,7 +193,7 @@ ccms_thread_lock_upgrade(ccms_cst_t *cst) * Convert a shared lock to exclusive. */ if (cst->count > 0) { - spin_lock(&cst->spin); + hammer2_spin_ex(&cst->spin); ++cst->upgrade; --cst->count; while (cst->count) { @@ -309,7 +202,7 @@ ccms_thread_lock_upgrade(ccms_cst_t *cst) } cst->count = -1; cst->td = curthread; - spin_unlock(&cst->spin); + hammer2_spin_unex(&cst->spin); return(CCMS_STATE_SHARED); } panic("ccms_thread_lock_upgrade: not locked"); @@ -323,16 +216,16 @@ ccms_thread_lock_downgrade(ccms_cst_t *cst, ccms_state_t ostate) if (ostate == CCMS_STATE_SHARED) { KKASSERT(cst->td == curthread); KKASSERT(cst->count == -1); - spin_lock(&cst->spin); + hammer2_spin_ex(&cst->spin); --cst->upgrade; cst->count = 1; cst->td = NULL; if (cst->blocked) { cst->blocked = 0; - spin_unlock(&cst->spin); + hammer2_spin_unex(&cst->spin); wakeup(cst); } else { - spin_unlock(&cst->spin); + hammer2_spin_unex(&cst->spin); } } /* else nothing to do if excl->excl */ @@ -354,29 +247,29 @@ ccms_thread_unlock(ccms_cst_t *cst) ++cst->count; return; } - spin_lock(&cst->spin); + hammer2_spin_ex(&cst->spin); KKASSERT(cst->count == -1); cst->count = 0; cst->td = NULL; if (cst->blocked) { cst->blocked = 0; - spin_unlock(&cst->spin); + hammer2_spin_unex(&cst->spin); wakeup(cst); return; } - spin_unlock(&cst->spin); + hammer2_spin_unex(&cst->spin); } else if (cst->count > 0) { /* * Shared */ - spin_lock(&cst->spin); + hammer2_spin_ex(&cst->spin); if (--cst->count == 0 && cst->blocked) { cst->blocked = 0; - spin_unlock(&cst->spin); + hammer2_spin_unex(&cst->spin); wakeup(cst); return; } - spin_unlock(&cst->spin); + hammer2_spin_unex(&cst->spin); } else { panic("ccms_thread_unlock: bad zero count\n"); } @@ -399,118 +292,24 @@ ccms_thread_unlock_upgraded(ccms_cst_t *cst, ccms_state_t ostate) LOCKEXIT; KKASSERT(cst->td == curthread); KKASSERT(cst->count == -1); - spin_lock(&cst->spin); + hammer2_spin_ex(&cst->spin); --cst->upgrade; cst->count = 0; cst->td = NULL; if (cst->blocked) { cst->blocked = 0; - spin_unlock(&cst->spin); + hammer2_spin_unex(&cst->spin); wakeup(cst); } else { - spin_unlock(&cst->spin); + hammer2_spin_unex(&cst->spin); } } else { ccms_thread_unlock(cst); } } -#if 0 -/* - * Release a local thread lock with special handling of the last lock - * reference. - * - * If no upgrades are in progress then the last reference to the lock will - * upgrade the lock to exclusive (if it was shared) and return 0 without - * unlocking it. - * - * If more than one reference remains, or upgrades are in progress, - * we drop the reference and return non-zero to indicate that more - * locks are present or pending. - */ -int -ccms_thread_unlock_zero(ccms_cst_t *cst) -{ - if (cst->count < 0) { - /* - * Exclusive owned by us, no races possible as long as it - * remains negative. Return 0 and leave us locked on the - * last lock. - */ - KKASSERT(cst->td == curthread); - if (cst->count == -1) { - spin_lock(&cst->spin); - if (cst->upgrade) { - cst->count = 0; - if (cst->blocked) { - cst->blocked = 0; - spin_unlock(&cst->spin); - wakeup(cst); - } else { - spin_unlock(&cst->spin); - } - return(1); - } - spin_unlock(&cst->spin); - return(0); - } - ++cst->count; - } else { - /* - * Convert the last shared lock to an exclusive lock - * and return 0. - * - * If there are upgrades pending the cst is unlocked and - * the upgrade waiters are woken up. The upgrade count - * prevents new exclusive holders for the duration. - */ - spin_lock(&cst->spin); - KKASSERT(cst->count > 0); - if (cst->count == 1) { - if (cst->upgrade) { - cst->count = 0; - if (cst->blocked) { - cst->blocked = 0; - spin_unlock(&cst->spin); - wakeup(cst); - } else { - spin_unlock(&cst->spin); - } - return(1); - } else { - cst->count = -1; - cst->td = curthread; - spin_unlock(&cst->spin); - return(0); - } - } - --cst->count; - spin_unlock(&cst->spin); - } - return(1); -} -#endif - int ccms_thread_lock_owned(ccms_cst_t *cst) { return(cst->count < 0 && cst->td == curthread); } - - -#if 0 -/* - * Acquire remote grant state. This routine can be used to upgrade or - * downgrade the state. If it blocks it will release any local locks - * acquired via (lock) but then it will continue getting the requested - * remote grant. - */ -static -void -ccms_rstate_get(ccms_lock_t *lock, ccms_cst_t *cst, ccms_state_t state) -{ - /* XXX */ - cst->state = state; -} - -#endif diff --git a/sys/vfs/hammer2/hammer2_ccms.h b/sys/vfs/hammer2/hammer2_ccms.h index cbbb7f7af7..116c8cdf54 100644 --- a/sys/vfs/hammer2/hammer2_ccms.h +++ b/sys/vfs/hammer2/hammer2_ccms.h @@ -88,7 +88,6 @@ typedef uint8_t ccms_state_t; typedef uint8_t ccms_type_t; struct ccms_cst; -struct ccms_lock; /* * CCMS_STATE_T - CCMS cache states. @@ -97,18 +96,14 @@ struct ccms_lock; * * ALLOWED - Cache state allows any recursive state to be acquired. * - * SHARED - Cache state allows shared access. If this is a topo_cst - * only INVALID or SHARED recursive states are allowed. + * SHARED - Cache state allows shared access. * - * EXCLUSIVE - Cache state allows exclusive access. If this is a - * topo_cst then INVALID, SHARED, or EXCLUSIVE recursive - * state is allowed. + * EXCLUSIVE - Cache state allows exclusive access. * * CCMS Implements an extended MESI model. The extensions are implemented * as CCMS_TYPE_T flags. */ #define CCMS_STATE_INVALID 0 /* unknown cache state */ -#define CCMS_STATE_ALLOWED 1 /* allow subsystem (topo only) */ #define CCMS_STATE_SHARED 2 /* clean, shared, read-only */ #define CCMS_STATE_EXCLUSIVE 3 /* clean, exclusive, read-only */ @@ -142,38 +137,6 @@ struct ccms_lock; #define CCMS_TYPE_QSALVE 0x10 #define CCMS_TYPE_RECURSIVE 0x80 -/* - * CCMS_LOCK - High level active lock - * - * This represents a high level locking request, such as used by - * read, write, and attribute operations. Initialize the ccms_lock - * structure and call ccms_lock_get(). - * - * When a CCMS lock is established the cache state of the underlying elements - * is adjusted to meet the requirements of the lock. The cache state - * requirements are infered by the lock type. CCMS locks can block on - * third party interactions if the underlying remote cache state is not - * compatible. - * - * CCMS data locks imply a shared CCMS inode lock. A CCMS topology lock does - * not imply a data or inode lock but topology locks can have far-reaching - * effects such as block ccms_locks on multiple inodes. - */ -struct ccms_lock { - TAILQ_ENTRY(ccms_lock) entry; - ccms_state_t req_t; - ccms_state_t req_a; - ccms_state_t req_d; - uint8_t flags; - struct ccms_cst *topo_cst; - struct ccms_cst *attr_cst; - struct ccms_cst *data_cst; - ccms_key_t key_beg; /* applies to dstate */ - ccms_key_t key_end; /* applies to dstate */ -}; - -#define CCMS_LOCK_FAILED 0x01 - /* * CCMS_CST - Low level locking state, persistent cache state * @@ -194,43 +157,25 @@ struct ccms_lock { */ struct ccms_cst { struct spinlock spin; /* thread spinlock */ - void *handle; /* opaque VFS handle */ ccms_state_t state; /* granted or inherited state */ ccms_type_t type; /* CST type and flags */ uint8_t unused02; uint8_t unused03; - ccms_tid_t path_id; /* rendezvous inode id */ - ccms_tid_t tid; /* [meta]data versioning id */ - ccms_key_t key_beg; /* key range (inclusive) */ - ccms_key_t key_end; /* key range (inclusive) */ - int32_t upgrade; /* upgrades pending */ int32_t count; /* active shared/exclusive count */ int32_t blocked; /* wakeup blocked on release */ thread_t td; /* if excl lock (count < 0) */ }; -/* - * Domain management, contains a pseudo-root for the CCMS topology. - */ -struct ccms_domain { - int cst_count; /* dynamic cst count */ - int cst_limit; /* dynamic cst limit */ -}; - -typedef struct ccms_lock ccms_lock_t; typedef struct ccms_cst ccms_cst_t; -typedef struct ccms_domain ccms_domain_t; /* * Kernel API */ #ifdef _KERNEL -void ccms_domain_init(ccms_domain_t *dom); -void ccms_domain_uninit(ccms_domain_t *dom); -void ccms_cst_init(ccms_cst_t *cst, void *handle); +void ccms_cst_init(ccms_cst_t *cst); void ccms_cst_uninit(ccms_cst_t *cst); void ccms_thread_lock(ccms_cst_t *cst, ccms_state_t state); @@ -241,13 +186,9 @@ ccms_state_t ccms_thread_lock_upgrade(ccms_cst_t *cst); void ccms_thread_lock_downgrade(ccms_cst_t *cst, ccms_state_t ostate); void ccms_thread_unlock(ccms_cst_t *cst); void ccms_thread_unlock_upgraded(ccms_cst_t *cst, ccms_state_t ostate); -/*int ccms_thread_unlock_zero(ccms_cst_t *cst);*/ int ccms_thread_lock_owned(ccms_cst_t *cst); void ccms_thread_lock_setown(ccms_cst_t *cst); -void ccms_lock_get(ccms_lock_t *lock); -void ccms_lock_put(ccms_lock_t *lock); - #endif #endif diff --git a/sys/vfs/hammer2/hammer2_chain.c b/sys/vfs/hammer2/hammer2_chain.c index 43e9d1e648..a67f270e08 100644 --- a/sys/vfs/hammer2/hammer2_chain.c +++ b/sys/vfs/hammer2/hammer2_chain.c @@ -137,16 +137,16 @@ hammer2_chain_setflush(hammer2_trans_t *trans, hammer2_chain_t *chain) hammer2_chain_t *parent; if ((chain->flags & HAMMER2_CHAIN_ONFLUSH) == 0) { - spin_lock(&chain->core.cst.spin); + hammer2_spin_sh(&chain->core.spin); while ((chain->flags & HAMMER2_CHAIN_ONFLUSH) == 0) { atomic_set_int(&chain->flags, HAMMER2_CHAIN_ONFLUSH); if ((parent = chain->parent) == NULL) break; - spin_lock(&parent->core.cst.spin); - spin_unlock(&chain->core.cst.spin); + hammer2_spin_sh(&parent->core.spin); + hammer2_spin_unsh(&chain->core.spin); chain = parent; } - spin_unlock(&chain->core.cst.spin); + hammer2_spin_unsh(&chain->core.spin); } } @@ -236,7 +236,7 @@ hammer2_chain_core_alloc(hammer2_trans_t *trans, hammer2_chain_t *chain) * sub-tree). */ RB_INIT(&core->rbtree); /* live chains */ - ccms_cst_init(&core->cst, chain); + hammer2_mtx_init(&core->lock, "h2chn"); } /* @@ -270,7 +270,7 @@ hammer2_chain_insert(hammer2_chain_t *parent, hammer2_chain_t *chain, int error = 0; if (flags & HAMMER2_CHAIN_INSERT_SPIN) - spin_lock(&parent->core.cst.spin); + hammer2_spin_ex(&parent->core.spin); /* * Interlocked by spinlock, check for race @@ -300,7 +300,7 @@ hammer2_chain_insert(hammer2_chain_t *parent, hammer2_chain_t *chain, atomic_add_int(&parent->core.live_count, 1); failed: if (flags & HAMMER2_CHAIN_INSERT_SPIN) - spin_unlock(&parent->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); return error; } @@ -355,7 +355,7 @@ hammer2_chain_drop(hammer2_chain_t *chain) * The chain cannot be freed if it has a non-empty core (children) or * it is not at the head of ownerq. * - * The cst spinlock is allowed nest child-to-parent (not parent-to-child). + * The core spinlock is allowed nest child-to-parent (not parent-to-child). */ static hammer2_chain_t * @@ -372,7 +372,7 @@ hammer2_chain_lastdrop(hammer2_chain_t *chain) * in core->rbtree are associated with other chains contemporary * with ours but not with our chain directly. */ - spin_lock(&chain->core.cst.spin); + hammer2_spin_ex(&chain->core.spin); /* * We can't free non-stale chains with children until we are @@ -388,10 +388,10 @@ hammer2_chain_lastdrop(hammer2_chain_t *chain) */ if (chain->core.chain_count) { if (atomic_cmpset_int(&chain->refs, 1, 0)) { - spin_unlock(&chain->core.cst.spin); + hammer2_spin_unex(&chain->core.spin); chain = NULL; /* success */ } else { - spin_unlock(&chain->core.cst.spin); + hammer2_spin_unex(&chain->core.spin); } return(chain); } @@ -415,11 +415,11 @@ hammer2_chain_lastdrop(hammer2_chain_t *chain) * spinlocks as bottom-up recursive, so this is safe). */ if ((parent = chain->parent) != NULL) { - spin_lock(&parent->core.cst.spin); + hammer2_spin_ex(&parent->core.spin); if (atomic_cmpset_int(&chain->refs, 1, 0) == 0) { /* 1->0 transition failed */ - spin_unlock(&parent->core.cst.spin); - spin_unlock(&chain->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); + hammer2_spin_unex(&chain->core.spin); return(chain); /* retry */ } @@ -446,7 +446,7 @@ hammer2_chain_lastdrop(hammer2_chain_t *chain) rdrop = NULL; } } - spin_unlock(&parent->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); parent = NULL; /* safety */ } @@ -456,11 +456,9 @@ hammer2_chain_lastdrop(hammer2_chain_t *chain) * We still have the core spinlock, and core's chain_count is 0. * Any parent spinlock is gone. */ - spin_unlock(&chain->core.cst.spin); + hammer2_spin_unex(&chain->core.spin); KKASSERT(RB_EMPTY(&chain->core.rbtree) && chain->core.chain_count == 0); - KKASSERT(chain->core.cst.count == 0); - KKASSERT(chain->core.cst.upgrade == 0); /* * All spin locks are gone, finish freeing stuff. @@ -558,7 +556,7 @@ hammer2_chain_lock(hammer2_chain_t *chain, int how) { hammer2_mount_t *hmp; hammer2_blockref_t *bref; - ccms_state_t ostate; + hammer2_mtx_state_t ostate; char *bdata; int error; @@ -576,9 +574,9 @@ hammer2_chain_lock(hammer2_chain_t *chain, int how) * Get the appropriate lock. */ if (how & HAMMER2_RESOLVE_SHARED) - ccms_thread_lock(&chain->core.cst, CCMS_STATE_SHARED); + hammer2_mtx_sh(&chain->core.lock, "h2chnx"); else - ccms_thread_lock(&chain->core.cst, CCMS_STATE_EXCLUSIVE); + hammer2_mtx_ex(&chain->core.lock, "h2chns"); /* * If we already have a valid data pointer no further action is @@ -614,9 +612,9 @@ hammer2_chain_lock(hammer2_chain_t *chain, int how) * buffer cache. If another thread got to it before us we * can just return. */ - ostate = ccms_thread_lock_upgrade(&chain->core.cst); + ostate = hammer2_mtx_upgrade(&chain->core.lock); if (chain->data) { - ccms_thread_lock_downgrade(&chain->core.cst, ostate); + hammer2_mtx_downgrade(&chain->core.lock, ostate); return (0); } @@ -652,7 +650,7 @@ hammer2_chain_lock(hammer2_chain_t *chain, int how) kprintf("hammer2_chain_lock: I/O error %016jx: %d\n", (intmax_t)bref->data_off, error); hammer2_io_bqrelse(&chain->dio); - ccms_thread_lock_downgrade(&chain->core.cst, ostate); + hammer2_mtx_downgrade(&chain->core.lock, ostate); return (error); } @@ -724,7 +722,7 @@ hammer2_chain_lock(hammer2_chain_t *chain, int how) chain->data = (void *)bdata; break; } - ccms_thread_lock_downgrade(&chain->core.cst, ostate); + hammer2_mtx_downgrade(&chain->core.lock, ostate); return (0); } @@ -737,14 +735,11 @@ hammer2_chain_lock(hammer2_chain_t *chain, int how) void hammer2_chain_unlock(hammer2_chain_t *chain) { - ccms_state_t ostate; + hammer2_mtx_state_t ostate; long *counterp; u_int lockcnt; /* - * The core->cst lock can be shared across several chains so we - * need to track the per-chain lockcnt separately. - * * If multiple locks are present (or being attempted) on this * particular chain we can just unlock, drop refs, and return. * @@ -757,7 +752,7 @@ hammer2_chain_unlock(hammer2_chain_t *chain) if (lockcnt > 1) { if (atomic_cmpset_int(&chain->lockcnt, lockcnt, lockcnt - 1)) { - ccms_thread_unlock(&chain->core.cst); + hammer2_mtx_unlock(&chain->core.lock); hammer2_chain_drop(chain); return; } @@ -776,13 +771,13 @@ hammer2_chain_unlock(hammer2_chain_t *chain) * leaving the data/io intact * * Otherwise if lockcnt is still 0 it is possible for it to become - * non-zero and race, but since we hold the core->cst lock - * exclusively all that will happen is that the chain will be - * reloaded after we unload it. + * non-zero and race, but since we hold the core->lock exclusively + * all that will happen is that the chain will be reloaded after we + * unload it. */ - ostate = ccms_thread_lock_upgrade(&chain->core.cst); + ostate = hammer2_mtx_upgrade(&chain->core.lock); if (chain->lockcnt) { - ccms_thread_unlock_upgraded(&chain->core.cst, ostate); + hammer2_mtx_unlock(&chain->core.lock); hammer2_chain_drop(chain); return; } @@ -796,7 +791,7 @@ hammer2_chain_unlock(hammer2_chain_t *chain) if (chain->dio == NULL) { if ((chain->flags & HAMMER2_CHAIN_MODIFIED) == 0) hammer2_chain_drop_data(chain, 0); - ccms_thread_unlock_upgraded(&chain->core.cst, ostate); + hammer2_mtx_unlock(&chain->core.lock); hammer2_chain_drop(chain); return; } @@ -871,7 +866,7 @@ hammer2_chain_unlock(hammer2_chain_t *chain) } else { hammer2_io_bqrelse(&chain->dio); } - ccms_thread_unlock_upgraded(&chain->core.cst, ostate); + hammer2_mtx_unlock(&chain->core.lock); hammer2_chain_drop(chain); } @@ -893,7 +888,7 @@ void hammer2_chain_countbrefs(hammer2_chain_t *chain, hammer2_blockref_t *base, int count) { - spin_lock(&chain->core.cst.spin); + hammer2_spin_ex(&chain->core.spin); if ((chain->core.flags & HAMMER2_CORE_COUNTEDBREFS) == 0) { if (base) { while (--count >= 0) { @@ -913,7 +908,7 @@ hammer2_chain_countbrefs(hammer2_chain_t *chain, /* else do not modify live_count */ atomic_set_int(&chain->core.flags, HAMMER2_CORE_COUNTEDBREFS); } - spin_unlock(&chain->core.cst.spin); + hammer2_spin_unex(&chain->core.spin); } /* @@ -1508,10 +1503,10 @@ hammer2_chain_getparent(hammer2_chain_t **parentp, int how) * is locked below to avoid a deadlock. */ oparent = *parentp; - spin_lock(&oparent->core.cst.spin); + hammer2_spin_ex(&oparent->core.spin); nparent = oparent->parent; hammer2_chain_ref(nparent); - spin_unlock(&oparent->core.cst.spin); + hammer2_spin_unex(&oparent->core.spin); if (oparent) { hammer2_chain_unlock(oparent); oparent = NULL; @@ -1699,7 +1694,7 @@ again: /* * Combined search */ - spin_lock(&parent->core.cst.spin); + hammer2_spin_ex(&parent->core.spin); chain = hammer2_combined_find(parent, base, count, cache_indexp, key_nextp, key_beg, key_end, @@ -1710,7 +1705,7 @@ again: * Exhausted parent chain, iterate. */ if (bref == NULL) { - spin_unlock(&parent->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); if (key_beg == key_end) /* short cut single-key case */ return (NULL); @@ -1739,7 +1734,7 @@ again: */ if (chain == NULL) { bcopy = *bref; - spin_unlock(&parent->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); chain = hammer2_chain_get(parent, generation, &bcopy); if (chain == NULL) { @@ -1753,7 +1748,7 @@ again: } } else { hammer2_chain_ref(chain); - spin_unlock(&parent->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); } /* @@ -2016,7 +2011,7 @@ again: hammer2_chain_countbrefs(parent, base, count); next_key = 0; - spin_lock(&parent->core.cst.spin); + hammer2_spin_ex(&parent->core.spin); chain = hammer2_combined_find(parent, base, count, cache_indexp, &next_key, key, HAMMER2_KEY_MAX, @@ -2027,7 +2022,7 @@ again: * Exhausted parent chain, we're done. */ if (bref == NULL) { - spin_unlock(&parent->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); KKASSERT(chain == NULL); goto done; } @@ -2037,7 +2032,7 @@ again: */ if (chain == NULL) { bcopy = *bref; - spin_unlock(&parent->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); chain = hammer2_chain_get(parent, generation, &bcopy); if (chain == NULL) { kprintf("retry scan parent %p keys %016jx\n", @@ -2051,7 +2046,7 @@ again: } } else { hammer2_chain_ref(chain); - spin_unlock(&parent->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); } /* @@ -2140,7 +2135,7 @@ hammer2_chain_create(hammer2_trans_t *trans, hammer2_chain_t **parentp, * Topology may be crossing a PFS boundary. */ parent = *parentp; - KKASSERT(ccms_thread_lock_owned(&parent->core.cst)); + KKASSERT(hammer2_mtx_owned(&parent->core.lock)); hmp = parent->hmp; chain = *chainp; @@ -2166,7 +2161,7 @@ hammer2_chain_create(hammer2_trans_t *trans, hammer2_chain_t **parentp, * to 1 by chain_alloc() for us, but lockcnt is not). */ chain->lockcnt = 1; - ccms_thread_lock(&chain->core.cst, CCMS_STATE_EXCLUSIVE); + hammer2_mtx_ex(&chain->core.lock, "h2chnx"); allocated = 1; /* @@ -2463,7 +2458,7 @@ hammer2_chain_rename(hammer2_trans_t *trans, hammer2_blockref_t *bref, * Having both chains locked is extremely important for atomicy. */ if (parentp && (parent = *parentp) != NULL) { - KKASSERT(ccms_thread_lock_owned(&parent->core.cst)); + KKASSERT(hammer2_mtx_owned(&parent->core.lock)); KKASSERT(parent->refs > 0); hammer2_chain_create(trans, parentp, &chain, chain->pmp, @@ -2508,7 +2503,7 @@ _hammer2_chain_delete_helper(hammer2_trans_t *trans, * Calculate blockmap pointer */ KKASSERT(chain->flags & HAMMER2_CHAIN_ONRBTREE); - spin_lock(&parent->core.cst.spin); + hammer2_spin_ex(&parent->core.spin); atomic_set_int(&chain->flags, HAMMER2_CHAIN_DELETED); atomic_add_int(&parent->core.live_count, -1); @@ -2588,7 +2583,7 @@ _hammer2_chain_delete_helper(hammer2_trans_t *trans, hammer2_base_delete(trans, parent, base, count, &cache_index, chain); } - spin_unlock(&parent->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); } else if (chain->flags & HAMMER2_CHAIN_ONRBTREE) { /* * Chain is not blockmapped but a parent is present. @@ -2599,7 +2594,7 @@ _hammer2_chain_delete_helper(hammer2_trans_t *trans, * synchronized, the chain's *_count_up fields contain * inode adjustment statistics which must be undone. */ - spin_lock(&parent->core.cst.spin); + hammer2_spin_ex(&parent->core.spin); if (chain->bref.type == HAMMER2_BREF_TYPE_INODE && (flags & HAMMER2_DELETE_NOSTATS) == 0) { KKASSERT(chain->data != NULL); @@ -2615,7 +2610,7 @@ _hammer2_chain_delete_helper(hammer2_trans_t *trans, atomic_clear_int(&chain->flags, HAMMER2_CHAIN_ONRBTREE); --parent->core.chain_count; chain->parent = NULL; - spin_unlock(&parent->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); } else { /* * Chain is not blockmapped and has no parent. This @@ -2724,7 +2719,7 @@ hammer2_chain_create_indirect(hammer2_trans_t *trans, hammer2_chain_t *parent, */ hmp = parent->hmp; *errorp = 0; - KKASSERT(ccms_thread_lock_owned(&parent->core.cst)); + KKASSERT(hammer2_mtx_owned(&parent->core.lock)); /*hammer2_chain_modify(trans, &parent, HAMMER2_MODIFY_OPTDATA);*/ if (parent->flags & HAMMER2_CHAIN_INITIAL) { @@ -2856,13 +2851,13 @@ hammer2_chain_create_indirect(hammer2_trans_t *trans, hammer2_chain_t *parent, key_beg = 0; key_end = HAMMER2_KEY_MAX; cache_index = 0; - spin_lock(&parent->core.cst.spin); + hammer2_spin_ex(&parent->core.spin); loops = 0; reason = 0; for (;;) { if (++loops > 100000) { - spin_unlock(&parent->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); panic("excessive loops r=%d p=%p base/count %p:%d %016jx\n", reason, parent, base, count, key_next); } @@ -2905,7 +2900,7 @@ hammer2_chain_create_indirect(hammer2_trans_t *trans, hammer2_chain_t *parent, * Use chain already present in the RBTREE */ hammer2_chain_ref(chain); - spin_unlock(&parent->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); hammer2_chain_lock(chain, HAMMER2_RESOLVE_NEVER | HAMMER2_RESOLVE_NOREF); } else { @@ -2914,18 +2909,18 @@ hammer2_chain_create_indirect(hammer2_trans_t *trans, hammer2_chain_t *parent, * on insertion race. */ bcopy = *bref; - spin_unlock(&parent->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); chain = hammer2_chain_get(parent, generation, &bcopy); if (chain == NULL) { reason = 1; - spin_lock(&parent->core.cst.spin); + hammer2_spin_ex(&parent->core.spin); continue; } if (bcmp(&bcopy, bref, sizeof(bcopy))) { kprintf("REASON 2\n"); reason = 2; hammer2_chain_drop(chain); - spin_lock(&parent->core.cst.spin); + hammer2_spin_ex(&parent->core.spin); continue; } hammer2_chain_lock(chain, HAMMER2_RESOLVE_NEVER | @@ -2966,7 +2961,7 @@ hammer2_chain_create_indirect(hammer2_trans_t *trans, hammer2_chain_t *parent, KKASSERT(parent->refs > 0); chain = NULL; next_key: - spin_lock(&parent->core.cst.spin); + hammer2_spin_ex(&parent->core.spin); next_key_spinlocked: if (--maxloops == 0) panic("hammer2_chain_create_indirect: maxloops"); @@ -2976,7 +2971,7 @@ next_key_spinlocked: key_beg = key_next; /* loop */ } - spin_unlock(&parent->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); /* * Insert the new indirect block into the parent now that we've @@ -3062,7 +3057,7 @@ hammer2_chain_indkey_freemap(hammer2_chain_t *parent, hammer2_key_t *keyp, key_beg = 0; key_end = HAMMER2_KEY_MAX; cache_index = 0; - spin_lock(&parent->core.cst.spin); + hammer2_spin_ex(&parent->core.spin); for (;;) { if (--maxloops == 0) { @@ -3108,7 +3103,7 @@ hammer2_chain_indkey_freemap(hammer2_chain_t *parent, hammer2_key_t *keyp, break; key_beg = key_next; } - spin_unlock(&parent->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); /* * Return the keybits for a higher-level FREEMAP_NODE covering @@ -3172,7 +3167,7 @@ hammer2_chain_indkey_normal(hammer2_chain_t *parent, hammer2_key_t *keyp, key_beg = 0; key_end = HAMMER2_KEY_MAX; cache_index = 0; - spin_lock(&parent->core.cst.spin); + hammer2_spin_ex(&parent->core.spin); for (;;) { if (--maxloops == 0) { @@ -3259,7 +3254,7 @@ hammer2_chain_indkey_normal(hammer2_chain_t *parent, hammer2_key_t *keyp, break; key_beg = key_next; } - spin_unlock(&parent->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); bref = NULL; /* now invalid (safety) */ /* @@ -3318,7 +3313,7 @@ void hammer2_chain_delete(hammer2_trans_t *trans, hammer2_chain_t *parent, hammer2_chain_t *chain, int flags) { - KKASSERT(ccms_thread_lock_owned(&chain->core.cst)); + KKASSERT(hammer2_mtx_owned(&chain->core.lock)); /* * Nothing to do if already marked. @@ -3566,7 +3561,7 @@ hammer2_base_delete(hammer2_trans_t *trans, hammer2_chain_t *parent, base[i].key != elm->key || ((chain->flags & HAMMER2_CHAIN_BMAPUPD) == 0 && base[i].keybits != elm->keybits)) { - spin_unlock(&parent->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); panic("delete base %p element not found at %d/%d elm %p\n", base, i, count, elm); return; @@ -3645,7 +3640,7 @@ hammer2_base_insert(hammer2_trans_t *trans __unused, hammer2_chain_t *parent, xkey = elm->key + ((hammer2_key_t)1 << elm->keybits) - 1; if (i != count && (base[i].key < elm->key || xkey >= base[i].key)) { - spin_unlock(&parent->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); panic("insert base %p overlapping elements at %d elm %p\n", base, i, elm); } diff --git a/sys/vfs/hammer2/hammer2_flush.c b/sys/vfs/hammer2/hammer2_flush.c index 57fda67c25..812b912fc6 100644 --- a/sys/vfs/hammer2/hammer2_flush.c +++ b/sys/vfs/hammer2/hammer2_flush.c @@ -537,10 +537,10 @@ hammer2_flush_core(hammer2_flush_info_t *info, hammer2_chain_t *chain, */ atomic_clear_int(&chain->flags, HAMMER2_CHAIN_ONFLUSH); info->parent = chain; - spin_lock(&chain->core.cst.spin); + hammer2_spin_ex(&chain->core.spin); RB_SCAN(hammer2_chain_tree, &chain->core.rbtree, NULL, hammer2_flush_recurse, info); - spin_unlock(&chain->core.cst.spin); + hammer2_spin_unex(&chain->core.spin); info->parent = parent; if (info->diddeferral) hammer2_chain_setflush(info->trans, chain); @@ -968,7 +968,7 @@ hammer2_flush_recurse(hammer2_chain_t *child, void *data) * unlock it in order to lock the child. */ hammer2_chain_ref(child); - spin_unlock(&parent->core.cst.spin); + hammer2_spin_unex(&parent->core.spin); hammer2_chain_unlock(parent); hammer2_chain_lock(child, HAMMER2_RESOLVE_MAYBE); @@ -1002,7 +1002,7 @@ hammer2_flush_recurse(hammer2_chain_t *child, void *data) hammer2_chain_lock(parent, HAMMER2_RESOLVE_MAYBE); hammer2_chain_drop(child); KKASSERT(info->parent == parent); - spin_lock(&parent->core.cst.spin); + hammer2_spin_ex(&parent->core.spin); return (0); } diff --git a/sys/vfs/hammer2/hammer2_inode.c b/sys/vfs/hammer2/hammer2_inode.c index b10e145939..12b7c2f9d4 100644 --- a/sys/vfs/hammer2/hammer2_inode.c +++ b/sys/vfs/hammer2/hammer2_inode.c @@ -97,7 +97,7 @@ hammer2_inode_lock_nex(hammer2_inode_t *ip, int how) int i; hammer2_inode_ref(ip); - ccms_thread_lock(&ip->topo_cst, CCMS_STATE_EXCLUSIVE); + hammer2_mtx_ex(&ip->lock, "h2ino"); cluster = hammer2_cluster_copy(&ip->cluster, HAMMER2_CLUSTER_COPY_NOCHAINS); @@ -142,7 +142,7 @@ hammer2_inode_unlock_ex(hammer2_inode_t *ip, hammer2_cluster_t *cluster) { if (cluster) hammer2_cluster_unlock(cluster); - ccms_thread_unlock(&ip->topo_cst); + hammer2_mtx_unlock(&ip->lock); hammer2_inode_drop(ip); } @@ -168,7 +168,7 @@ hammer2_inode_lock_sh(hammer2_inode_t *ip) hammer2_inode_ref(ip); cluster = hammer2_cluster_copy(&ip->cluster, HAMMER2_CLUSTER_COPY_NOCHAINS); - ccms_thread_lock(&ip->topo_cst, CCMS_STATE_SHARED); + hammer2_mtx_sh(&ip->lock, "h2ino"); cluster->focus = NULL; @@ -208,32 +208,63 @@ hammer2_inode_unlock_sh(hammer2_inode_t *ip, hammer2_cluster_t *cluster) { if (cluster) hammer2_cluster_unlock(cluster); - ccms_thread_unlock(&ip->topo_cst); + hammer2_mtx_unlock(&ip->lock); hammer2_inode_drop(ip); } -ccms_state_t +/* + * Temporarily release a lock held shared or exclusive. Caller must + * hold the lock shared or exclusive on call and lock will be released + * on return. + * + * Restore a lock that was temporarily released. + */ +hammer2_mtx_state_t hammer2_inode_lock_temp_release(hammer2_inode_t *ip) { - return(ccms_thread_lock_temp_release(&ip->topo_cst)); + return hammer2_mtx_temp_release(&ip->lock); } void -hammer2_inode_lock_temp_restore(hammer2_inode_t *ip, ccms_state_t ostate) +hammer2_inode_lock_temp_restore(hammer2_inode_t *ip, hammer2_mtx_state_t ostate) { - ccms_thread_lock_temp_restore(&ip->topo_cst, ostate); + hammer2_mtx_temp_restore(&ip->lock, "h2ino", ostate); } -ccms_state_t +/* + * Upgrade a shared inode lock to exclusive and return. If the inode lock + * is already held exclusively this is a NOP. + * + * The caller MUST hold the inode lock either shared or exclusive on call + * and will own the lock exclusively on return. + * + * Returns non-zero if the lock was already exclusive prior to the upgrade. + */ +int hammer2_inode_lock_upgrade(hammer2_inode_t *ip) { - return(ccms_thread_lock_upgrade(&ip->topo_cst)); + int wasexclusive; + + if (mtx_islocked_ex(&ip->lock)) { + wasexclusive = 1; + } else { + hammer2_mtx_unlock(&ip->lock); + hammer2_mtx_ex(&ip->lock, "h2upg"); + wasexclusive = 0; + } + return wasexclusive; } +/* + * Downgrade an inode lock from exclusive to shared only if the inode + * lock was previously shared. If the inode lock was previously exclusive, + * this is a NOP. + */ void -hammer2_inode_lock_downgrade(hammer2_inode_t *ip, ccms_state_t ostate) +hammer2_inode_lock_downgrade(hammer2_inode_t *ip, int wasexclusive) { - ccms_thread_lock_downgrade(&ip->topo_cst, ostate); + if (wasexclusive == 0) + mtx_downgrade(&ip->lock); } /* @@ -248,11 +279,11 @@ hammer2_inode_lookup(hammer2_pfsmount_t *pmp, hammer2_tid_t inum) if (pmp->spmp_hmp) { ip = NULL; } else { - spin_lock(&pmp->inum_spin); + hammer2_spin_ex(&pmp->inum_spin); ip = RB_LOOKUP(hammer2_inode_tree, &pmp->inum_tree, inum); if (ip) hammer2_inode_ref(ip); - spin_unlock(&pmp->inum_spin); + hammer2_spin_unex(&pmp->inum_spin); } return(ip); } @@ -287,20 +318,23 @@ hammer2_inode_drop(hammer2_inode_t *ip) /* * Transition to zero, must interlock with * the inode inumber lookup tree (if applicable). + * It should not be possible for anyone to race + * the transition to 0. + * */ pmp = ip->pmp; KKASSERT(pmp); - spin_lock(&pmp->inum_spin); + hammer2_spin_ex(&pmp->inum_spin); if (atomic_cmpset_int(&ip->refs, 1, 0)) { - KKASSERT(ip->topo_cst.count == 0); + KKASSERT(hammer2_mtx_refs(&ip->lock) == 0); if (ip->flags & HAMMER2_INODE_ONRBTREE) { atomic_clear_int(&ip->flags, HAMMER2_INODE_ONRBTREE); RB_REMOVE(hammer2_inode_tree, &pmp->inum_tree, ip); } - spin_unlock(&pmp->inum_spin); + hammer2_spin_unex(&pmp->inum_spin); pip = ip->pip; ip->pip = NULL; @@ -322,7 +356,7 @@ hammer2_inode_drop(hammer2_inode_t *ip) ip = pip; /* continue with pip (can be NULL) */ } else { - spin_unlock(&ip->pmp->inum_spin); + hammer2_spin_unex(&ip->pmp->inum_spin); } } else { /* @@ -349,7 +383,6 @@ hammer2_igetv(hammer2_inode_t *ip, hammer2_cluster_t *cparent, int *errorp) const hammer2_inode_data_t *ripdata; hammer2_pfsmount_t *pmp; struct vnode *vp; - ccms_state_t ostate; pmp = ip->pmp; KKASSERT(pmp != NULL); @@ -364,6 +397,8 @@ hammer2_igetv(hammer2_inode_t *ip, hammer2_cluster_t *cparent, int *errorp) * inode must be unlocked during the vget() to avoid a * deadlock against a reclaim. */ + int wasexclusive; + vp = ip->vp; if (vp) { /* @@ -374,6 +409,8 @@ hammer2_igetv(hammer2_inode_t *ip, hammer2_cluster_t *cparent, int *errorp) * vget(). The vget() can still fail if we lost * a reclaim race on the vnode. */ + hammer2_mtx_state_t ostate; + vhold(vp); ostate = hammer2_inode_lock_temp_release(ip); if (vget(vp, LK_EXCLUSIVE)) { @@ -410,11 +447,11 @@ hammer2_igetv(hammer2_inode_t *ip, hammer2_cluster_t *cparent, int *errorp) /* * Lock the inode and check for an allocation race. */ - ostate = hammer2_inode_lock_upgrade(ip); + wasexclusive = hammer2_inode_lock_upgrade(ip); if (ip->vp != NULL) { vp->v_type = VBAD; vx_put(vp); - hammer2_inode_lock_downgrade(ip, ostate); + hammer2_inode_lock_downgrade(ip, wasexclusive); continue; } @@ -463,7 +500,7 @@ hammer2_igetv(hammer2_inode_t *ip, hammer2_cluster_t *cparent, int *errorp) vp->v_data = ip; ip->vp = vp; hammer2_inode_ref(ip); /* vp association */ - hammer2_inode_lock_downgrade(ip, ostate); + hammer2_inode_lock_downgrade(ip, wasexclusive); break; } @@ -512,7 +549,7 @@ again: if (nip == NULL) break; - ccms_thread_lock(&nip->topo_cst, CCMS_STATE_EXCLUSIVE); + hammer2_mtx_ex(&nip->lock, "h2ino"); /* * Handle SMP race (not applicable to the super-root spmp @@ -520,7 +557,7 @@ again: */ if (pmp->spmp_hmp == NULL && (nip->flags & HAMMER2_INODE_ONRBTREE) == 0) { - ccms_thread_unlock(&nip->topo_cst); + hammer2_mtx_unlock(&nip->lock); hammer2_inode_drop(nip); continue; } @@ -562,8 +599,8 @@ again: * hammer2_inode_lock_ex() call. */ nip->refs = 1; - ccms_cst_init(&nip->topo_cst, &nip->cluster); - ccms_thread_lock(&nip->topo_cst, CCMS_STATE_EXCLUSIVE); + hammer2_mtx_init(&nip->lock, "h2ino"); + hammer2_mtx_ex(&nip->lock, "h2ino"); /* combination of thread lock and chain lock == inode lock */ /* @@ -571,15 +608,15 @@ again: * get. Undo all the work and try again. */ if (pmp->spmp_hmp == NULL) { - spin_lock(&pmp->inum_spin); + hammer2_spin_ex(&pmp->inum_spin); if (RB_INSERT(hammer2_inode_tree, &pmp->inum_tree, nip)) { - spin_unlock(&pmp->inum_spin); - ccms_thread_unlock(&nip->topo_cst); + hammer2_spin_unex(&pmp->inum_spin); + hammer2_mtx_unlock(&nip->lock); hammer2_inode_drop(nip); goto again; } atomic_set_int(&nip->flags, HAMMER2_INODE_ONRBTREE); - spin_unlock(&pmp->inum_spin); + hammer2_spin_unex(&pmp->inum_spin); } return (nip); diff --git a/sys/vfs/hammer2/hammer2_io.c b/sys/vfs/hammer2/hammer2_io.c index ac2066f91c..ce346470a1 100644 --- a/sys/vfs/hammer2/hammer2_io.c +++ b/sys/vfs/hammer2/hammer2_io.c @@ -97,34 +97,34 @@ hammer2_io_getblk(hammer2_mount_t *hmp, off_t lbase, int lsize, /* * Access/Allocate the DIO, bump dio->refs to prevent destruction. */ - spin_lock_shared(&hmp->io_spin); + hammer2_spin_sh(&hmp->io_spin); dio = RB_LOOKUP(hammer2_io_tree, &hmp->iotree, pbase); if (dio) { if ((atomic_fetchadd_int(&dio->refs, 1) & HAMMER2_DIO_MASK) == 0) { atomic_add_int(&dio->hmp->iofree_count, -1); } - spin_unlock_shared(&hmp->io_spin); + hammer2_spin_unsh(&hmp->io_spin); } else { - spin_unlock_shared(&hmp->io_spin); + hammer2_spin_unsh(&hmp->io_spin); dio = kmalloc(sizeof(*dio), M_HAMMER2, M_INTWAIT | M_ZERO); dio->hmp = hmp; dio->pbase = pbase; dio->psize = psize; dio->refs = 1; - spin_init(&dio->spin, "h2dio"); + hammer2_spin_init(&dio->spin, "h2dio"); TAILQ_INIT(&dio->iocbq); - spin_lock(&hmp->io_spin); + hammer2_spin_ex(&hmp->io_spin); xio = RB_INSERT(hammer2_io_tree, &hmp->iotree, dio); if (xio == NULL) { atomic_add_int(&hammer2_dio_count, 1); - spin_unlock(&hmp->io_spin); + hammer2_spin_unex(&hmp->io_spin); } else { if ((atomic_fetchadd_int(&xio->refs, 1) & HAMMER2_DIO_MASK) == 0) { atomic_add_int(&xio->hmp->iofree_count, -1); } - spin_unlock(&hmp->io_spin); + hammer2_spin_unex(&hmp->io_spin); kfree(dio, M_HAMMER2); dio = xio; } @@ -160,16 +160,16 @@ hammer2_io_getblk(hammer2_mount_t *hmp, off_t lbase, int lsize, * If DIO_INPROG is already set then set WAITING and * queue the iocb. */ - spin_lock(&dio->spin); + hammer2_spin_ex(&dio->spin); if (atomic_cmpset_int(&dio->refs, refs, refs | HAMMER2_DIO_WAITING)) { iocb->flags |= HAMMER2_IOCB_ONQ | HAMMER2_IOCB_INPROG; TAILQ_INSERT_TAIL(&dio->iocbq, iocb, entry); - spin_unlock(&dio->spin); + hammer2_spin_unex(&dio->spin); break; } - spin_unlock(&dio->spin); + hammer2_spin_unex(&dio->spin); /* retry */ } else { /* @@ -246,7 +246,7 @@ hammer2_io_complete(hammer2_iocb_t *iocb) nrefs = orefs & ~(HAMMER2_DIO_WAITING | HAMMER2_DIO_INPROG); if (orefs & HAMMER2_DIO_WAITING) { - spin_lock(&dio->spin); + hammer2_spin_ex(&dio->spin); cbtmp = TAILQ_FIRST(&dio->iocbq); if (cbtmp) { /* @@ -255,15 +255,15 @@ hammer2_io_complete(hammer2_iocb_t *iocb) * iocb. */ TAILQ_REMOVE(&dio->iocbq, cbtmp, entry); - spin_unlock(&dio->spin); + hammer2_spin_unex(&dio->spin); cbtmp->callback(cbtmp); /* chained */ break; } else if (atomic_cmpset_int(&dio->refs, orefs, nrefs)) { - spin_unlock(&dio->spin); + hammer2_spin_unex(&dio->spin); break; } - spin_unlock(&dio->spin); + hammer2_spin_unex(&dio->spin); /* retry */ } else if (atomic_cmpset_int(&dio->refs, orefs, nrefs)) { break; @@ -441,13 +441,13 @@ hammer2_io_putblk(hammer2_io_t **diop) struct hammer2_cleanupcb_info info; RB_INIT(&info.tmptree); - spin_lock(&hmp->io_spin); + hammer2_spin_ex(&hmp->io_spin); if (hmp->iofree_count > 1000) { info.count = hmp->iofree_count / 2; RB_SCAN(hammer2_io_tree, &hmp->iotree, NULL, hammer2_io_cleanup_callback, &info); } - spin_unlock(&hmp->io_spin); + hammer2_spin_unex(&hmp->io_spin); hammer2_io_cleanup(hmp, &info.tmptree); } } diff --git a/sys/vfs/hammer2/hammer2_subr.c b/sys/vfs/hammer2/hammer2_subr.c index 128583ee46..af479f88d9 100644 --- a/sys/vfs/hammer2/hammer2_subr.c +++ b/sys/vfs/hammer2/hammer2_subr.c @@ -45,23 +45,22 @@ /* * Mount-wide locks */ - void hammer2_mount_exlock(hammer2_mount_t *hmp) { - ccms_thread_lock(&hmp->vchain.core.cst, CCMS_STATE_EXCLUSIVE); + hammer2_mtx_ex(&hmp->vchain.core.lock, "h2mount"); } void hammer2_mount_shlock(hammer2_mount_t *hmp) { - ccms_thread_lock(&hmp->vchain.core.cst, CCMS_STATE_SHARED); + hammer2_mtx_sh(&hmp->vchain.core.lock, "h2mount"); } void hammer2_mount_unlock(hammer2_mount_t *hmp) { - ccms_thread_unlock(&hmp->vchain.core.cst); + hammer2_mtx_unlock(&hmp->vchain.core.lock); } /* diff --git a/sys/vfs/hammer2/hammer2_vfsops.c b/sys/vfs/hammer2/hammer2_vfsops.c index 7b2cbb3f7c..83c29efc4d 100644 --- a/sys/vfs/hammer2/hammer2_vfsops.c +++ b/sys/vfs/hammer2/hammer2_vfsops.c @@ -338,7 +338,7 @@ hammer2_pfsalloc(const hammer2_inode_data_t *ripdata, hammer2_tid_t alloc_tid) pmp->inode_tid = ripdata->pfs_inum + 1; pmp->pfs_clid = ripdata->pfs_clid; } - mtx_init(&pmp->wthread_mtx); + hammer2_mtx_init(&pmp->wthread_mtx, "h2wthr"); bioq_init(&pmp->wthread_bioq); return pmp; @@ -736,14 +736,18 @@ hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data, int i; int j; + /* + * Directly lock the inode->lock, do not run through + * hammer2_inode_lock*(). + */ hammer2_inode_ref(pmp->iroot); - ccms_thread_lock(&pmp->iroot->topo_cst, CCMS_STATE_EXCLUSIVE); + hammer2_mtx_ex(&pmp->iroot->lock, "h2ino"); if (pmp->iroot->cluster.nchains + cluster->nchains > HAMMER2_MAXCLUSTER) { kprintf("hammer2_mount: cluster full!\n"); - ccms_thread_unlock(&pmp->iroot->topo_cst); + hammer2_mtx_unlock(&pmp->iroot->lock); hammer2_inode_drop(pmp->iroot); hammer2_cluster_unlock(cluster); @@ -764,7 +768,7 @@ hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data, ++j; } pmp->iroot->cluster.nchains = j; - ccms_thread_unlock(&pmp->iroot->topo_cst); + hammer2_mtx_unlock(&pmp->iroot->lock); hammer2_inode_drop(pmp->iroot); hammer2_cluster_unlock(cluster); lockmgr(&hammer2_mntlk, LK_RELEASE); @@ -791,7 +795,6 @@ hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data, } cluster->pmp = pmp; - ccms_domain_init(&pmp->ccms_dom); TAILQ_INSERT_TAIL(&hammer2_pfslist, pmp, mntentry); lockmgr(&hammer2_mntlk, LK_RELEASE); @@ -887,7 +890,7 @@ hammer2_write_thread(void *arg) pmp = arg; - mtx_lock(&pmp->wthread_mtx); + hammer2_mtx_ex(&pmp->wthread_mtx, "h2wth"); while (pmp->wthread_destroy == 0) { if (bioq_first(&pmp->wthread_bioq) == NULL) { mtxsleep(&pmp->wthread_bioq, &pmp->wthread_mtx, @@ -914,7 +917,7 @@ hammer2_write_thread(void *arg) /* * else normal bio processing */ - mtx_unlock(&pmp->wthread_mtx); + hammer2_mtx_unlock(&pmp->wthread_mtx); hammer2_lwinprog_drop(pmp); @@ -954,14 +957,14 @@ hammer2_write_thread(void *arg) bp->b_error = EIO; } biodone(bio); - mtx_lock(&pmp->wthread_mtx); + hammer2_mtx_ex(&pmp->wthread_mtx, "h2wth"); } hammer2_trans_done(&trans); } pmp->wthread_destroy = -1; wakeup(&pmp->wthread_destroy); - mtx_unlock(&pmp->wthread_mtx); + hammer2_mtx_unlock(&pmp->wthread_mtx); } void @@ -970,14 +973,14 @@ hammer2_bioq_sync(hammer2_pfsmount_t *pmp) struct bio sync_bio; bzero(&sync_bio, sizeof(sync_bio)); /* dummy with no bio_buf */ - mtx_lock(&pmp->wthread_mtx); + hammer2_mtx_ex(&pmp->wthread_mtx, "h2wth"); if (pmp->wthread_destroy == 0 && TAILQ_FIRST(&pmp->wthread_bioq.queue)) { bioq_insert_tail(&pmp->wthread_bioq, &sync_bio); while ((sync_bio.bio_flags & BIO_DONE) == 0) mtxsleep(&sync_bio, &pmp->wthread_mtx, 0, "h2bioq", 0); } - mtx_unlock(&pmp->wthread_mtx); + hammer2_mtx_unlock(&pmp->wthread_mtx); } /* @@ -1589,10 +1592,8 @@ hammer2_vfs_unmount(struct mount *mp, int mntflags) goto failed; } - ccms_domain_uninit(&pmp->ccms_dom); - if (pmp->wthread_td) { - mtx_lock(&pmp->wthread_mtx); + hammer2_mtx_ex(&pmp->wthread_mtx, "h2wth"); pmp->wthread_destroy = 1; wakeup(&pmp->wthread_bioq); while (pmp->wthread_destroy != -1) { @@ -1600,7 +1601,7 @@ hammer2_vfs_unmount(struct mount *mp, int mntflags) &pmp->wthread_mtx, 0, "umount-sleep", 0); } - mtx_unlock(&pmp->wthread_mtx); + hammer2_mtx_unlock(&pmp->wthread_mtx); pmp->wthread_td = NULL; } diff --git a/sys/vfs/hammer2/hammer2_vnops.c b/sys/vfs/hammer2/hammer2_vnops.c index 3d603d3898..75a969cfd3 100644 --- a/sys/vfs/hammer2/hammer2_vnops.c +++ b/sys/vfs/hammer2/hammer2_vnops.c @@ -287,9 +287,9 @@ hammer2_vop_reclaim(struct vop_reclaim_args *ap) ipul = kmalloc(sizeof(*ipul), pmp->minode, M_WAITOK | M_ZERO); ipul->ip = ip; - spin_lock(&pmp->list_spin); + hammer2_spin_ex(&pmp->list_spin); TAILQ_INSERT_TAIL(&pmp->unlinkq, ipul, entry); - spin_unlock(&pmp->list_spin); + hammer2_spin_unex(&pmp->list_spin); hammer2_inode_unlock_ex(ip, cluster); /* unlock */ /* retain ref from vp for ipul */ } else { @@ -937,10 +937,13 @@ hammer2_read_file(hammer2_inode_t *ip, struct uio *uio, int seqcount) /* * UIO read loop. + * + * WARNING! Assumes that the kernel interlocks size changes at the + * vnode level. */ - ccms_thread_lock(&ip->topo_cst, CCMS_STATE_EXCLUSIVE); + hammer2_mtx_sh(&ip->lock, "h2ino"); size = ip->size; - ccms_thread_unlock(&ip->topo_cst); + hammer2_mtx_unlock(&ip->lock); while (uio->uio_resid > 0 && uio->uio_offset < size) { hammer2_key_t lbase; @@ -991,12 +994,15 @@ hammer2_write_file(hammer2_inode_t *ip, /* * Setup if append + * + * WARNING! Assumes that the kernel interlocks size changes at the + * vnode level. */ - ccms_thread_lock(&ip->topo_cst, CCMS_STATE_EXCLUSIVE); + hammer2_mtx_ex(&ip->lock, "h2ino"); if (ioflag & IO_APPEND) uio->uio_offset = ip->size; old_eof = ip->size; - ccms_thread_unlock(&ip->topo_cst); + hammer2_mtx_unlock(&ip->lock); /* * Extend the file if necessary. If the write fails at some point @@ -1143,10 +1149,10 @@ hammer2_write_file(hammer2_inode_t *ip, if (error && new_eof != old_eof) { hammer2_truncate_file(ip, old_eof); } else if (modified) { - ccms_thread_lock(&ip->topo_cst, CCMS_STATE_EXCLUSIVE); + hammer2_mtx_ex(&ip->lock, "h2ino"); hammer2_update_time(&ip->mtime); atomic_set_int(&ip->flags, HAMMER2_INODE_MTIME); - ccms_thread_unlock(&ip->topo_cst); + hammer2_mtx_unlock(&ip->lock); } atomic_set_int(&ip->flags, HAMMER2_INODE_MODIFIED); hammer2_knote(ip->vp, kflags); @@ -1162,6 +1168,9 @@ hammer2_write_file(hammer2_inode_t *ip, * * WARNING: nvtruncbuf() can only be safely called without the inode lock * held due to the way our write thread works. + * + * WARNING! Assumes that the kernel interlocks size changes at the + * vnode level. */ static void @@ -1177,16 +1186,19 @@ hammer2_truncate_file(hammer2_inode_t *ip, hammer2_key_t nsize) nblksize, (int)nsize & (nblksize - 1), 0); } - ccms_thread_lock(&ip->topo_cst, CCMS_STATE_EXCLUSIVE); + hammer2_mtx_ex(&ip->lock, "h2ino"); ip->size = nsize; atomic_set_int(&ip->flags, HAMMER2_INODE_RESIZED); - ccms_thread_unlock(&ip->topo_cst); + hammer2_mtx_unlock(&ip->lock); LOCKSTOP; } /* * Extend the size of a file. The inode must not be locked. * + * WARNING! Assumes that the kernel interlocks size changes at the + * vnode level. + * * NOTE: Caller handles setting HAMMER2_INODE_MODIFIED */ static @@ -1199,10 +1211,10 @@ hammer2_extend_file(hammer2_inode_t *ip, hammer2_key_t nsize) int nblksize; LOCKSTART; - ccms_thread_lock(&ip->topo_cst, CCMS_STATE_EXCLUSIVE); + hammer2_mtx_ex(&ip->lock, "h2ino"); osize = ip->size; ip->size = nsize; - ccms_thread_unlock(&ip->topo_cst); + hammer2_mtx_unlock(&ip->lock); if (ip->vp) { oblksize = hammer2_calc_logical(ip, osize, &lbase, NULL); @@ -2316,14 +2328,14 @@ hammer2_strategy_write(struct vop_strategy_args *ap) pmp = ip->pmp; hammer2_lwinprog_ref(pmp); - mtx_lock(&pmp->wthread_mtx); + hammer2_mtx_ex(&pmp->wthread_mtx, "h2wth"); if (TAILQ_EMPTY(&pmp->wthread_bioq.queue)) { bioq_insert_tail(&pmp->wthread_bioq, ap->a_bio); - mtx_unlock(&pmp->wthread_mtx); + hammer2_mtx_unlock(&pmp->wthread_mtx); wakeup(&pmp->wthread_bioq); } else { bioq_insert_tail(&pmp->wthread_bioq, ap->a_bio); - mtx_unlock(&pmp->wthread_mtx); + hammer2_mtx_unlock(&pmp->wthread_mtx); } hammer2_lwinprog_wait(pmp); @@ -2396,10 +2408,10 @@ hammer2_run_unlinkq(hammer2_trans_t *trans, hammer2_pfsmount_t *pmp) return; LOCKSTART; - spin_lock(&pmp->list_spin); + hammer2_spin_ex(&pmp->list_spin); while ((ipul = TAILQ_FIRST(&pmp->unlinkq)) != NULL) { TAILQ_REMOVE(&pmp->unlinkq, ipul, entry); - spin_unlock(&pmp->list_spin); + hammer2_spin_unex(&pmp->list_spin); ip = ipul->ip; kfree(ipul, pmp->minode); @@ -2418,9 +2430,9 @@ hammer2_run_unlinkq(hammer2_trans_t *trans, hammer2_pfsmount_t *pmp) hammer2_inode_unlock_ex(ip, cluster); /* inode lock */ hammer2_inode_drop(ip); /* ipul ref */ - spin_lock(&pmp->list_spin); + hammer2_spin_ex(&pmp->list_spin); } - spin_unlock(&pmp->list_spin); + hammer2_spin_unex(&pmp->list_spin); LOCKSTOP; }