hammer2 - locking revamp
authorMatthew Dillon <dillon@apollo.backplane.com>
Mon, 23 Mar 2015 03:26:08 +0000 (20:26 -0700)
committerMatthew Dillon <dillon@apollo.backplane.com>
Mon, 23 Mar 2015 03:26:08 +0000 (20:26 -0700)
* Temporarily remove hammer2_ccms.c from the build and remove the ccms
  container.  The CCMS container will be used again when we get cache
  coherent in, but for now it isn't needed.

* Replace the ccms state lock with a mtx lock and move
  into hammer2_chain_core.

  Note that the mtx lock being used here has abort and async locking
  support and these features will be required by HAMMER2.

* Replace the ccms spin lock with a normal spin lock and move
  into hammer2_chain_core.

* Refactor the OS locking interface to use hammer2_* prefixes for easier
  porting.

* Use a shared spin lock for the ONFLUSH bit update instead of an exclusive
  spin lock.

sys/vfs/hammer2/Makefile
sys/vfs/hammer2/hammer2.h
sys/vfs/hammer2/hammer2_ccms.c
sys/vfs/hammer2/hammer2_ccms.h
sys/vfs/hammer2/hammer2_chain.c
sys/vfs/hammer2/hammer2_flush.c
sys/vfs/hammer2/hammer2_inode.c
sys/vfs/hammer2/hammer2_io.c
sys/vfs/hammer2/hammer2_subr.c
sys/vfs/hammer2/hammer2_vfsops.c
sys/vfs/hammer2/hammer2_vnops.c

index a06504c..560f507 100644 (file)
@@ -5,7 +5,7 @@
 
 CFLAGS+= -DINVARIANTS -DSMP
 KMOD=  hammer2
-SRCS=  hammer2_vfsops.c hammer2_vnops.c hammer2_inode.c hammer2_ccms.c
+SRCS=  hammer2_vfsops.c hammer2_vnops.c hammer2_inode.c
 SRCS+= hammer2_chain.c hammer2_flush.c hammer2_freemap.c hammer2_cluster.c
 SRCS+= hammer2_ioctl.c hammer2_msgops.c hammer2_subr.c hammer2_bulkscan.c
 SRCS+=  hammer2_lz4.c hammer2_io.c hammer2_iocom.c
index 0ea201d..03a9b04 100644 (file)
 #include <sys/buf.h>
 #include <sys/queue.h>
 #include <sys/limits.h>
-#include <sys/signal2.h>
 #include <sys/dmsg.h>
 #include <sys/mutex.h>
 #include <sys/kern_syscall.h>
 
+#include <sys/signal2.h>
 #include <sys/buf2.h>
 #include <sys/mutex2.h>
 
 #include "hammer2_disk.h"
 #include "hammer2_mount.h"
 #include "hammer2_ioctl.h"
-#include "hammer2_ccms.h"
 
 struct hammer2_io;
 struct hammer2_iocb;
@@ -105,6 +104,64 @@ struct hammer2_span;
 struct hammer2_state;
 struct hammer2_msg;
 
+/*
+ * Mutex and lock shims.  Hammer2 requires support for asynchronous and
+ * abortable locks, and both exclusive and shared spinlocks.  Normal
+ * synchronous non-abortable locks can be substituted for spinlocks.
+ */
+typedef mtx_t                          hammer2_mtx_t;
+typedef mtx_link_t                     hammer2_mtx_link_t;
+typedef mtx_state_t                    hammer2_mtx_state_t;
+
+typedef struct spinlock                        hammer2_spin_t;
+
+#define hammer2_mtx_ex                 mtx_lock_ex_quick
+#define hammer2_mtx_sh                 mtx_lock_sh_quick
+#define hammer2_mtx_unlock             mtx_unlock
+#define hammer2_mtx_owned              mtx_owned
+#define hammer2_mtx_init               mtx_init
+#define hammer2_mtx_temp_release       mtx_lock_temp_release
+#define hammer2_mtx_temp_restore       mtx_lock_temp_restore
+#define hammer2_mtx_refs               mtx_lockrefs
+
+#define hammer2_spin_init              spin_init
+#define hammer2_spin_sh                        spin_lock_shared
+#define hammer2_spin_ex                        spin_lock
+#define hammer2_spin_unsh              spin_unlock_shared
+#define hammer2_spin_unex              spin_unlock
+
+/*
+ * General lock support
+ */
+static __inline
+int
+hammer2_mtx_upgrade(hammer2_mtx_t *mtx)
+{
+       int wasexclusive;
+
+       if (mtx_islocked_ex(mtx)) {
+               wasexclusive = 1;
+       } else {
+               mtx_unlock(mtx);
+               mtx_lock_ex_quick(mtx);
+               wasexclusive = 0;
+       }
+       return wasexclusive;
+}
+
+/*
+ * Downgrade an inode lock from exclusive to shared only if the inode
+ * lock was previously shared.  If the inode lock was previously exclusive,
+ * this is a NOP.
+ */
+static __inline
+void
+hammer2_mtx_downgrade(hammer2_mtx_t *mtx, int wasexclusive)
+{
+       if (wasexclusive == 0)
+               mtx_downgrade(mtx);
+}
+
 /*
  * The xid tracks internal transactional updates.
  *
@@ -185,7 +242,8 @@ TAILQ_HEAD(h2_iocb_list, hammer2_iocb);
        (HAMMER2_PBUFSIZE / sizeof(hammer2_blockref_t) / sizeof(uint32_t))
 
 struct hammer2_chain_core {
-       struct ccms_cst cst;
+       hammer2_mtx_t   lock;
+       hammer2_spin_t  spin;
        struct hammer2_chain_tree rbtree; /* sub-chains */
        int             live_zero;      /* blockref array opt */
        u_int           flags;
@@ -409,20 +467,23 @@ RB_PROTOTYPE(hammer2_chain_tree, hammer2_chain, rbnode, hammer2_chain_cmp);
 /*
  * HAMMER2 cluster - A set of chains representing the same entity.
  *
- * The hammer2_pfsmount structure embeds a hammer2_cluster.  All other
- * hammer2_cluster use cases use temporary allocations.
+ * hammer2_cluster typically represents a temporary set of representitive
+ * chains.  The one exception is that a hammer2_cluster is embedded in
+ * hammer2_inode.  This embedded cluster is ONLY used to track the
+ * representitive chains and cannot be directly locked.
  *
- * The cluster API mimics the chain API.  Except as used in the pfsmount,
- * the cluster structure is a temporary 'working copy' of a set of chains
- * representing targets compatible with the operation.  However, for
- * performance reasons the cluster API does not necessarily issue concurrent
- * requests to the underlying chain API for all compatible chains all the
- * time.  This may sometimes necessitate revisiting parent cluster nodes
- * to 'flesh out' (validate more chains).
+ * A cluster is temporary (and thus per-thread) for locking purposes,
+ * allowing us to embed the asynchronous storage required for
+ * cluster operations in the cluster itself.  That is, except for the
+ * embeddeding hammer2_inode, the cluster structure will always represent
+ * a 'working copy'.
  *
- * If an insufficient number of chains remain in a working copy, the operation
- * may have to be downgraded, retried, or stall until the requisit number
- * of chains are available.
+ * Because the cluster is a 'working copy' and is usually subject to cluster
+ * quorum rules, it is quite possible for us to end up with an insufficient
+ * number of live chains to execute an operation.  If an insufficient number
+ * of chains remain in a working copy, the operation may have to be
+ * downgraded, retried, or stall until the requisit number of chains are
+ * available.
  */
 #define HAMMER2_MAXCLUSTER     8
 
@@ -434,12 +495,12 @@ struct hammer2_cluster {
        int                     nchains;
        hammer2_iocb_t          iocb;
        hammer2_chain_t         *focus;         /* current focus (or mod) */
+       hammer2_mtx_link_t      asynclnk[HAMMER2_MAXCLUSTER];
        hammer2_chain_t         *array[HAMMER2_MAXCLUSTER];
-       char                    missed[HAMMER2_MAXCLUSTER];
        int                     cache_index[HAMMER2_MAXCLUSTER];
 };
 
-typedef struct hammer2_cluster hammer2_cluster_t;
+typedef struct hammer2_cluster hammer2_cluster_t;
 
 #define HAMMER2_CLUSTER_INODE  0x00000001      /* embedded in inode */
 #define HAMMER2_CLUSTER_NOSYNC 0x00000002      /* not in sync (cumulative) */
@@ -450,16 +511,13 @@ RB_HEAD(hammer2_inode_tree, hammer2_inode);
 /*
  * A hammer2 inode.
  *
- * NOTE: The inode's attribute CST which is also used to lock the inode
- *      is embedded in the chain (chain.cst) and aliased w/ attr_cst.
- *
  * NOTE: The inode-embedded cluster is never used directly for I/O (since
  *      it may be shared).  Instead it will be replicated-in and synchronized
  *      back out if changed.
  */
 struct hammer2_inode {
        RB_ENTRY(hammer2_inode) rbnode;         /* inumber lookup (HL) */
-       ccms_cst_t              topo_cst;       /* directory topology cst */
+       hammer2_mtx_t           lock;           /* inode lock */
        struct hammer2_pfsmount *pmp;           /* PFS mount */
        struct hammer2_inode    *pip;           /* parent inode */
        struct vnode            *vp;
@@ -638,7 +696,6 @@ struct hammer2_pfsmount {
        hammer2_inode_t         *ihidden;       /* PFS hidden directory */
        struct lock             lock;           /* PFS lock for certain ops */
        hammer2_off_t           inode_count;    /* copy of inode_count */
-       ccms_domain_t           ccms_dom;
        struct netexport        export;         /* nfs export */
        int                     ronly;          /* read-only mount */
        struct malloc_type      *minode;
@@ -655,7 +712,7 @@ struct hammer2_pfsmount {
        struct h2_unlk_list     unlinkq;        /* last-close unlink */
        thread_t                wthread_td;     /* write thread td */
        struct bio_queue_head   wthread_bioq;   /* logical buffer bioq */
-       struct mtx              wthread_mtx;    /* interlock */
+       hammer2_mtx_t           wthread_mtx;    /* interlock */
        int                     wthread_destroy;/* termination sequencing */
 };
 
@@ -773,8 +830,6 @@ extern struct objcache *cache_buffer_write;
 extern int destroy;
 extern int write_thread_wakeup;
 
-extern mtx_t thread_protect;
-
 /*
  * hammer2_subr.c
  */
@@ -787,10 +842,11 @@ hammer2_cluster_t *hammer2_inode_lock_nex(hammer2_inode_t *ip, int how);
 hammer2_cluster_t *hammer2_inode_lock_sh(hammer2_inode_t *ip);
 void hammer2_inode_unlock_ex(hammer2_inode_t *ip, hammer2_cluster_t *chain);
 void hammer2_inode_unlock_sh(hammer2_inode_t *ip, hammer2_cluster_t *chain);
-ccms_state_t hammer2_inode_lock_temp_release(hammer2_inode_t *ip);
-void hammer2_inode_lock_temp_restore(hammer2_inode_t *ip, ccms_state_t ostate);
-ccms_state_t hammer2_inode_lock_upgrade(hammer2_inode_t *ip);
-void hammer2_inode_lock_downgrade(hammer2_inode_t *ip, ccms_state_t ostate);
+hammer2_mtx_state_t hammer2_inode_lock_temp_release(hammer2_inode_t *ip);
+void hammer2_inode_lock_temp_restore(hammer2_inode_t *ip,
+                       hammer2_mtx_state_t ostate);
+int hammer2_inode_lock_upgrade(hammer2_inode_t *ip);
+void hammer2_inode_lock_downgrade(hammer2_inode_t *ip, int);
 
 void hammer2_mount_exlock(hammer2_mount_t *hmp);
 void hammer2_mount_shlock(hammer2_mount_t *hmp);
index d4f8101..e202814 100644 (file)
 
 int ccms_debug = 0;
 
-/*
- * Initialize a new CCMS dataspace.  Create a new RB tree with a single
- * element covering the entire 64 bit offset range.  This simplifies
- * algorithms enormously by removing a number of special cases.
- */
-void
-ccms_domain_init(ccms_domain_t *dom)
-{
-       bzero(dom, sizeof(*dom));
-       /*kmalloc_create(&dom->mcst, "CCMS-cst");*/
-       /*dom->root.domain = dom;*/
-}
-
-void
-ccms_domain_uninit(ccms_domain_t *dom)
-{
-       /*kmalloc_destroy(&dom->mcst);*/
-}
-
 void
-ccms_cst_init(ccms_cst_t *cst, void *handle)
+ccms_cst_init(ccms_cst_t *cst)
 {
        bzero(cst, sizeof(*cst));
        spin_init(&cst->spin, "ccmscst");
-       cst->handle = handle;
 }
 
 void
@@ -85,95 +65,8 @@ ccms_cst_uninit(ccms_cst_t *cst)
        if (cst->state != CCMS_STATE_INVALID) {
                /* XXX */
        }
-       cst->handle = NULL;
-}
-
-#if 0
-/*
- * Acquire an operational CCMS lock on multiple CSTs.
- *
- * This code is in the critical path and highly streamlined.
- */
-void
-ccms_lock_get(ccms_lock_t *lock)
-{
-       ccms_inode_t *cino = lock->cino;
-
-again:
-       lock->flags &= ~CCMS_LOCK_FAILED;
-
-       /*
-        * Acquire all local locks first, then resolve them against the
-        * remote cache state.  Order is important here.
-        */
-       if (lock->req_t) {
-               KKASSERT(lock->req_d <= lock->req_t);
-               KKASSERT(lock->req_a <= lock->req_t);
-               ccms_thread_lock(&cino->topo_cst, lock->req_t);
-       }
-       if (lock->req_a)
-               ccms_thread_lock(&cino->attr_cst, lock->req_a);
-       if (lock->req_d)
-               ccms_thread_lock(&cino->data_cst[0], lock->req_d);
-
-       /*
-        * Once the local locks are established the CST grant state cannot
-        * be pulled out from under us.  However, it is entirely possible
-        * to deadlock on it so when CST grant state cannot be obtained
-        * trivially we have to unwind our local locks, then get the state,
-        * and then loop.
-        */
-       if (lock->req_t > cino->topo_cst.state) {
-               ccms_rstate_get(lock, &cino->topo_cst, lock->req_t);
-       } else if (cino->topo_cst.state == CCMS_STATE_INVALID) {
-               ccms_rstate_get(lock, &cino->topo_cst, CCMS_STATE_ALLOWED);
-       } else if (cino->topo_cst.state == CCMS_STATE_SHARED &&
-                   (lock->req_d > CCMS_STATE_SHARED ||
-                    lock->req_a > CCMS_STATE_SHARED)) {
-               ccms_rstate_get(lock, &cino->topo_cst, CCMS_STATE_ALLOWED);
-       }
-       /* else the rstate is compatible */
-
-       if (lock->req_a > cino->attr_cst.state)
-               ccms_rstate_get(lock, &cino->attr_cst, lock->req_a);
-
-       if (lock->req_d > cino->data_cst[0].state)
-               ccms_rstate_get(lock, &cino->data_cst[0], lock->req_d);
-
-       /*
-        * If the ccms_rstate_get() code deadlocks (or even if it just
-        * blocks), it will release all local locks and set the FAILED
-        * bit.  The routine will still acquire the requested remote grants
-        * before returning but since the local locks are lost at that
-        * point the remote grants are no longer protected and we have to
-        * retry.
-        */
-       if (lock->flags & CCMS_LOCK_FAILED) {
-               goto again;
-       }
 }
 
-/*
- * Release a previously acquired CCMS lock.
- */
-void
-ccms_lock_put(ccms_lock_t *lock)
-{
-       ccms_inode_t *cino = lock->cino;
-
-       if (lock->req_d) {
-               ccms_thread_unlock(&cino->data_cst[0]);
-       }
-       if (lock->req_a) {
-               ccms_thread_unlock(&cino->attr_cst);
-       }
-       if (lock->req_t) {
-               ccms_thread_unlock(&cino->topo_cst);
-       }
-}
-
-#endif
-
 /************************************************************************
  *                         CST SUPPORT FUNCTIONS                       *
  ************************************************************************/
@@ -201,7 +94,7 @@ ccms_thread_lock(ccms_cst_t *cst, ccms_state_t state)
         * Otherwise use the spinlock to interlock the operation and sleep
         * as necessary.
         */
-       spin_lock(&cst->spin);
+       hammer2_spin_ex(&cst->spin);
        if (state == CCMS_STATE_SHARED) {
                while (cst->count < 0 || cst->upgrade) {
                        cst->blocked = 1;
@@ -217,10 +110,10 @@ ccms_thread_lock(ccms_cst_t *cst, ccms_state_t state)
                cst->count = -1;
                cst->td = curthread;
        } else {
-               spin_unlock(&cst->spin);
+               hammer2_spin_unex(&cst->spin);
                panic("ccms_thread_lock: bad state %d\n", state);
        }
-       spin_unlock(&cst->spin);
+       hammer2_spin_unex(&cst->spin);
 }
 
 /*
@@ -236,26 +129,26 @@ ccms_thread_lock_nonblock(ccms_cst_t *cst, ccms_state_t state)
                return(0);
        }
 
-       spin_lock(&cst->spin);
+       hammer2_spin_ex(&cst->spin);
        if (state == CCMS_STATE_SHARED) {
                if (cst->count < 0 || cst->upgrade) {
-                       spin_unlock(&cst->spin);
+                       hammer2_spin_unex(&cst->spin);
                        return (EBUSY);
                }
                ++cst->count;
                KKASSERT(cst->td == NULL);
        } else if (state == CCMS_STATE_EXCLUSIVE) {
                if (cst->count != 0 || cst->upgrade) {
-                       spin_unlock(&cst->spin);
+                       hammer2_spin_unex(&cst->spin);
                        return (EBUSY);
                }
                cst->count = -1;
                cst->td = curthread;
        } else {
-               spin_unlock(&cst->spin);
+               hammer2_spin_unex(&cst->spin);
                panic("ccms_thread_lock_nonblock: bad state %d\n", state);
        }
-       spin_unlock(&cst->spin);
+       hammer2_spin_unex(&cst->spin);
        LOCKENTER;
        return(0);
 }
@@ -300,7 +193,7 @@ ccms_thread_lock_upgrade(ccms_cst_t *cst)
         * Convert a shared lock to exclusive.
         */
        if (cst->count > 0) {
-               spin_lock(&cst->spin);
+               hammer2_spin_ex(&cst->spin);
                ++cst->upgrade;
                --cst->count;
                while (cst->count) {
@@ -309,7 +202,7 @@ ccms_thread_lock_upgrade(ccms_cst_t *cst)
                }
                cst->count = -1;
                cst->td = curthread;
-               spin_unlock(&cst->spin);
+               hammer2_spin_unex(&cst->spin);
                return(CCMS_STATE_SHARED);
        }
        panic("ccms_thread_lock_upgrade: not locked");
@@ -323,16 +216,16 @@ ccms_thread_lock_downgrade(ccms_cst_t *cst, ccms_state_t ostate)
        if (ostate == CCMS_STATE_SHARED) {
                KKASSERT(cst->td == curthread);
                KKASSERT(cst->count == -1);
-               spin_lock(&cst->spin);
+               hammer2_spin_ex(&cst->spin);
                --cst->upgrade;
                cst->count = 1;
                cst->td = NULL;
                if (cst->blocked) {
                        cst->blocked = 0;
-                       spin_unlock(&cst->spin);
+                       hammer2_spin_unex(&cst->spin);
                        wakeup(cst);
                } else {
-                       spin_unlock(&cst->spin);
+                       hammer2_spin_unex(&cst->spin);
                }
        }
        /* else nothing to do if excl->excl */
@@ -354,29 +247,29 @@ ccms_thread_unlock(ccms_cst_t *cst)
                        ++cst->count;
                        return;
                }
-               spin_lock(&cst->spin);
+               hammer2_spin_ex(&cst->spin);
                KKASSERT(cst->count == -1);
                cst->count = 0;
                cst->td = NULL;
                if (cst->blocked) {
                        cst->blocked = 0;
-                       spin_unlock(&cst->spin);
+                       hammer2_spin_unex(&cst->spin);
                        wakeup(cst);
                        return;
                }
-               spin_unlock(&cst->spin);
+               hammer2_spin_unex(&cst->spin);
        } else if (cst->count > 0) {
                /*
                 * Shared
                 */
-               spin_lock(&cst->spin);
+               hammer2_spin_ex(&cst->spin);
                if (--cst->count == 0 && cst->blocked) {
                        cst->blocked = 0;
-                       spin_unlock(&cst->spin);
+                       hammer2_spin_unex(&cst->spin);
                        wakeup(cst);
                        return;
                }
-               spin_unlock(&cst->spin);
+               hammer2_spin_unex(&cst->spin);
        } else {
                panic("ccms_thread_unlock: bad zero count\n");
        }
@@ -399,118 +292,24 @@ ccms_thread_unlock_upgraded(ccms_cst_t *cst, ccms_state_t ostate)
                LOCKEXIT;
                KKASSERT(cst->td == curthread);
                KKASSERT(cst->count == -1);
-               spin_lock(&cst->spin);
+               hammer2_spin_ex(&cst->spin);
                --cst->upgrade;
                cst->count = 0;
                cst->td = NULL;
                if (cst->blocked) {
                        cst->blocked = 0;
-                       spin_unlock(&cst->spin);
+                       hammer2_spin_unex(&cst->spin);
                        wakeup(cst);
                } else {
-                       spin_unlock(&cst->spin);
+                       hammer2_spin_unex(&cst->spin);
                }
        } else {
                ccms_thread_unlock(cst);
        }
 }
 
-#if 0
-/*
- * Release a local thread lock with special handling of the last lock
- * reference.
- *
- * If no upgrades are in progress then the last reference to the lock will
- * upgrade the lock to exclusive (if it was shared) and return 0 without
- * unlocking it.
- *
- * If more than one reference remains, or upgrades are in progress,
- * we drop the reference and return non-zero to indicate that more
- * locks are present or pending.
- */
-int
-ccms_thread_unlock_zero(ccms_cst_t *cst)
-{
-       if (cst->count < 0) {
-               /*
-                * Exclusive owned by us, no races possible as long as it
-                * remains negative.  Return 0 and leave us locked on the
-                * last lock.
-                */
-               KKASSERT(cst->td == curthread);
-               if (cst->count == -1) {
-                       spin_lock(&cst->spin);
-                       if (cst->upgrade) {
-                               cst->count = 0;
-                               if (cst->blocked) {
-                                       cst->blocked = 0;
-                                       spin_unlock(&cst->spin);
-                                       wakeup(cst);
-                               } else {
-                                       spin_unlock(&cst->spin);
-                               }
-                               return(1);
-                       }
-                       spin_unlock(&cst->spin);
-                       return(0);
-               }
-               ++cst->count;
-       } else {
-               /*
-                * Convert the last shared lock to an exclusive lock
-                * and return 0.
-                *
-                * If there are upgrades pending the cst is unlocked and
-                * the upgrade waiters are woken up.  The upgrade count
-                * prevents new exclusive holders for the duration.
-                */
-               spin_lock(&cst->spin);
-               KKASSERT(cst->count > 0);
-               if (cst->count == 1) {
-                       if (cst->upgrade) {
-                               cst->count = 0;
-                               if (cst->blocked) {
-                                       cst->blocked = 0;
-                                       spin_unlock(&cst->spin);
-                                       wakeup(cst);
-                               } else {
-                                       spin_unlock(&cst->spin);
-                               }
-                               return(1);
-                       } else {
-                               cst->count = -1;
-                               cst->td = curthread;
-                               spin_unlock(&cst->spin);
-                               return(0);
-                       }
-               }
-               --cst->count;
-               spin_unlock(&cst->spin);
-       }
-       return(1);
-}
-#endif
-
 int
 ccms_thread_lock_owned(ccms_cst_t *cst)
 {
        return(cst->count < 0 && cst->td == curthread);
 }
-
-
-#if 0
-/*
- * Acquire remote grant state.  This routine can be used to upgrade or
- * downgrade the state.  If it blocks it will release any local locks
- * acquired via (lock) but then it will continue getting the requested
- * remote grant.
- */
-static
-void
-ccms_rstate_get(ccms_lock_t *lock, ccms_cst_t *cst, ccms_state_t state)
-{
-       /* XXX */
-       cst->state = state;
-}
-
-#endif
index cbbb7f7..116c8cd 100644 (file)
@@ -88,7 +88,6 @@ typedef uint8_t               ccms_state_t;
 typedef uint8_t                ccms_type_t;
 
 struct ccms_cst;
-struct ccms_lock;
 
 /*
  * CCMS_STATE_T - CCMS cache states.
@@ -97,18 +96,14 @@ struct ccms_lock;
  *
  * ALLOWED   -  Cache state allows any recursive state to be acquired.
  *
- * SHARED    - Cache state allows shared access.  If this is a topo_cst
- *             only INVALID or SHARED recursive states are allowed.
+ * SHARED    - Cache state allows shared access.
  *
- * EXCLUSIVE -  Cache state allows exclusive access.  If this is a
- *             topo_cst then INVALID, SHARED, or EXCLUSIVE recursive
- *             state is allowed.
+ * EXCLUSIVE -  Cache state allows exclusive access.
  *
  * CCMS Implements an extended MESI model.  The extensions are implemented
  * as CCMS_TYPE_T flags.
  */
 #define CCMS_STATE_INVALID     0       /* unknown cache state */
-#define CCMS_STATE_ALLOWED     1       /* allow subsystem (topo only) */
 #define CCMS_STATE_SHARED      2       /* clean, shared, read-only */
 #define CCMS_STATE_EXCLUSIVE   3       /* clean, exclusive, read-only */
 
@@ -142,38 +137,6 @@ struct ccms_lock;
 #define CCMS_TYPE_QSALVE       0x10
 #define CCMS_TYPE_RECURSIVE    0x80
 
-/*
- * CCMS_LOCK - High level active lock
- *
- * This represents a high level locking request, such as used by
- * read, write, and attribute operations.  Initialize the ccms_lock
- * structure and call ccms_lock_get().
- *
- * When a CCMS lock is established the cache state of the underlying elements
- * is adjusted to meet the requirements of the lock.  The cache state
- * requirements are infered by the lock type.  CCMS locks can block on
- * third party interactions if the underlying remote cache state is not
- * compatible.
- *
- * CCMS data locks imply a shared CCMS inode lock.  A CCMS topology lock does
- * not imply a data or inode lock but topology locks can have far-reaching
- * effects such as block ccms_locks on multiple inodes.
- */
-struct ccms_lock {
-       TAILQ_ENTRY(ccms_lock) entry;
-       ccms_state_t    req_t;
-       ccms_state_t    req_a;
-       ccms_state_t    req_d;
-       uint8_t         flags;
-       struct ccms_cst *topo_cst;
-       struct ccms_cst *attr_cst;
-       struct ccms_cst *data_cst;
-       ccms_key_t      key_beg;        /* applies to dstate */
-       ccms_key_t      key_end;        /* applies to dstate */
-};
-
-#define CCMS_LOCK_FAILED       0x01
-
 /*
  * CCMS_CST - Low level locking state, persistent cache state
  *
@@ -194,43 +157,25 @@ struct ccms_lock {
  */
 struct ccms_cst {
        struct spinlock spin;           /* thread spinlock */
-       void            *handle;        /* opaque VFS handle */
        ccms_state_t    state;          /* granted or inherited state */
        ccms_type_t     type;           /* CST type and flags */
        uint8_t         unused02;
        uint8_t         unused03;
 
-       ccms_tid_t      path_id;        /* rendezvous inode id */
-       ccms_tid_t      tid;            /* [meta]data versioning id */
-       ccms_key_t      key_beg;        /* key range (inclusive) */
-       ccms_key_t      key_end;        /* key range (inclusive) */
-
        int32_t         upgrade;        /* upgrades pending */
        int32_t         count;          /* active shared/exclusive count */
        int32_t         blocked;        /* wakeup blocked on release */
        thread_t        td;             /* if excl lock (count < 0) */
 };
 
-/*
- * Domain management, contains a pseudo-root for the CCMS topology.
- */
-struct ccms_domain {
-       int                     cst_count;      /* dynamic cst count */
-       int                     cst_limit;      /* dynamic cst limit */
-};
-
-typedef struct ccms_lock       ccms_lock_t;
 typedef struct ccms_cst                ccms_cst_t;
-typedef struct ccms_domain     ccms_domain_t;
 
 /*
  * Kernel API
  */
 #ifdef _KERNEL
 
-void ccms_domain_init(ccms_domain_t *dom);
-void ccms_domain_uninit(ccms_domain_t *dom);
-void ccms_cst_init(ccms_cst_t *cst, void *handle);
+void ccms_cst_init(ccms_cst_t *cst);
 void ccms_cst_uninit(ccms_cst_t *cst);
 
 void ccms_thread_lock(ccms_cst_t *cst, ccms_state_t state);
@@ -241,13 +186,9 @@ ccms_state_t ccms_thread_lock_upgrade(ccms_cst_t *cst);
 void ccms_thread_lock_downgrade(ccms_cst_t *cst, ccms_state_t ostate);
 void ccms_thread_unlock(ccms_cst_t *cst);
 void ccms_thread_unlock_upgraded(ccms_cst_t *cst, ccms_state_t ostate);
-/*int ccms_thread_unlock_zero(ccms_cst_t *cst);*/
 int ccms_thread_lock_owned(ccms_cst_t *cst);
 void ccms_thread_lock_setown(ccms_cst_t *cst);
 
-void ccms_lock_get(ccms_lock_t *lock);
-void ccms_lock_put(ccms_lock_t *lock);
-
 #endif
 
 #endif
index 43e9d1e..a67f270 100644 (file)
@@ -137,16 +137,16 @@ hammer2_chain_setflush(hammer2_trans_t *trans, hammer2_chain_t *chain)
        hammer2_chain_t *parent;
 
        if ((chain->flags & HAMMER2_CHAIN_ONFLUSH) == 0) {
-               spin_lock(&chain->core.cst.spin);
+               hammer2_spin_sh(&chain->core.spin);
                while ((chain->flags & HAMMER2_CHAIN_ONFLUSH) == 0) {
                        atomic_set_int(&chain->flags, HAMMER2_CHAIN_ONFLUSH);
                        if ((parent = chain->parent) == NULL)
                                break;
-                       spin_lock(&parent->core.cst.spin);
-                       spin_unlock(&chain->core.cst.spin);
+                       hammer2_spin_sh(&parent->core.spin);
+                       hammer2_spin_unsh(&chain->core.spin);
                        chain = parent;
                }
-               spin_unlock(&chain->core.cst.spin);
+               hammer2_spin_unsh(&chain->core.spin);
        }
 }
 
@@ -236,7 +236,7 @@ hammer2_chain_core_alloc(hammer2_trans_t *trans, hammer2_chain_t *chain)
         * sub-tree).
         */
        RB_INIT(&core->rbtree); /* live chains */
-       ccms_cst_init(&core->cst, chain);
+       hammer2_mtx_init(&core->lock, "h2chn");
 }
 
 /*
@@ -270,7 +270,7 @@ hammer2_chain_insert(hammer2_chain_t *parent, hammer2_chain_t *chain,
        int error = 0;
 
        if (flags & HAMMER2_CHAIN_INSERT_SPIN)
-               spin_lock(&parent->core.cst.spin);
+               hammer2_spin_ex(&parent->core.spin);
 
        /*
         * Interlocked by spinlock, check for race
@@ -300,7 +300,7 @@ hammer2_chain_insert(hammer2_chain_t *parent, hammer2_chain_t *chain,
                atomic_add_int(&parent->core.live_count, 1);
 failed:
        if (flags & HAMMER2_CHAIN_INSERT_SPIN)
-               spin_unlock(&parent->core.cst.spin);
+               hammer2_spin_unex(&parent->core.spin);
        return error;
 }
 
@@ -355,7 +355,7 @@ hammer2_chain_drop(hammer2_chain_t *chain)
  * The chain cannot be freed if it has a non-empty core (children) or
  * it is not at the head of ownerq.
  *
- * The cst spinlock is allowed nest child-to-parent (not parent-to-child).
+ * The core spinlock is allowed nest child-to-parent (not parent-to-child).
  */
 static
 hammer2_chain_t *
@@ -372,7 +372,7 @@ hammer2_chain_lastdrop(hammer2_chain_t *chain)
         * in core->rbtree are associated with other chains contemporary
         * with ours but not with our chain directly.
         */
-       spin_lock(&chain->core.cst.spin);
+       hammer2_spin_ex(&chain->core.spin);
 
        /*
         * We can't free non-stale chains with children until we are
@@ -388,10 +388,10 @@ hammer2_chain_lastdrop(hammer2_chain_t *chain)
         */
        if (chain->core.chain_count) {
                if (atomic_cmpset_int(&chain->refs, 1, 0)) {
-                       spin_unlock(&chain->core.cst.spin);
+                       hammer2_spin_unex(&chain->core.spin);
                        chain = NULL;   /* success */
                } else {
-                       spin_unlock(&chain->core.cst.spin);
+                       hammer2_spin_unex(&chain->core.spin);
                }
                return(chain);
        }
@@ -415,11 +415,11 @@ hammer2_chain_lastdrop(hammer2_chain_t *chain)
         *  spinlocks as bottom-up recursive, so this is safe).
         */
        if ((parent = chain->parent) != NULL) {
-               spin_lock(&parent->core.cst.spin);
+               hammer2_spin_ex(&parent->core.spin);
                if (atomic_cmpset_int(&chain->refs, 1, 0) == 0) {
                        /* 1->0 transition failed */
-                       spin_unlock(&parent->core.cst.spin);
-                       spin_unlock(&chain->core.cst.spin);
+                       hammer2_spin_unex(&parent->core.spin);
+                       hammer2_spin_unex(&chain->core.spin);
                        return(chain);  /* retry */
                }
 
@@ -446,7 +446,7 @@ hammer2_chain_lastdrop(hammer2_chain_t *chain)
                                rdrop = NULL;
                        }
                }
-               spin_unlock(&parent->core.cst.spin);
+               hammer2_spin_unex(&parent->core.spin);
                parent = NULL;  /* safety */
        }
 
@@ -456,11 +456,9 @@ hammer2_chain_lastdrop(hammer2_chain_t *chain)
         * We still have the core spinlock, and core's chain_count is 0.
         * Any parent spinlock is gone.
         */
-       spin_unlock(&chain->core.cst.spin);
+       hammer2_spin_unex(&chain->core.spin);
        KKASSERT(RB_EMPTY(&chain->core.rbtree) &&
                 chain->core.chain_count == 0);
-       KKASSERT(chain->core.cst.count == 0);
-       KKASSERT(chain->core.cst.upgrade == 0);
 
        /*
         * All spin locks are gone, finish freeing stuff.
@@ -558,7 +556,7 @@ hammer2_chain_lock(hammer2_chain_t *chain, int how)
 {
        hammer2_mount_t *hmp;
        hammer2_blockref_t *bref;
-       ccms_state_t ostate;
+       hammer2_mtx_state_t ostate;
        char *bdata;
        int error;
 
@@ -576,9 +574,9 @@ hammer2_chain_lock(hammer2_chain_t *chain, int how)
         * Get the appropriate lock.
         */
        if (how & HAMMER2_RESOLVE_SHARED)
-               ccms_thread_lock(&chain->core.cst, CCMS_STATE_SHARED);
+               hammer2_mtx_sh(&chain->core.lock, "h2chnx");
        else
-               ccms_thread_lock(&chain->core.cst, CCMS_STATE_EXCLUSIVE);
+               hammer2_mtx_ex(&chain->core.lock, "h2chns");
 
        /*
         * If we already have a valid data pointer no further action is
@@ -614,9 +612,9 @@ hammer2_chain_lock(hammer2_chain_t *chain, int how)
         * buffer cache.  If another thread got to it before us we
         * can just return.
         */
-       ostate = ccms_thread_lock_upgrade(&chain->core.cst);
+       ostate = hammer2_mtx_upgrade(&chain->core.lock);
        if (chain->data) {
-               ccms_thread_lock_downgrade(&chain->core.cst, ostate);
+               hammer2_mtx_downgrade(&chain->core.lock, ostate);
                return (0);
        }
 
@@ -652,7 +650,7 @@ hammer2_chain_lock(hammer2_chain_t *chain, int how)
                kprintf("hammer2_chain_lock: I/O error %016jx: %d\n",
                        (intmax_t)bref->data_off, error);
                hammer2_io_bqrelse(&chain->dio);
-               ccms_thread_lock_downgrade(&chain->core.cst, ostate);
+               hammer2_mtx_downgrade(&chain->core.lock, ostate);
                return (error);
        }
 
@@ -724,7 +722,7 @@ hammer2_chain_lock(hammer2_chain_t *chain, int how)
                chain->data = (void *)bdata;
                break;
        }
-       ccms_thread_lock_downgrade(&chain->core.cst, ostate);
+       hammer2_mtx_downgrade(&chain->core.lock, ostate);
        return (0);
 }
 
@@ -737,14 +735,11 @@ hammer2_chain_lock(hammer2_chain_t *chain, int how)
 void
 hammer2_chain_unlock(hammer2_chain_t *chain)
 {
-       ccms_state_t ostate;
+       hammer2_mtx_state_t ostate;
        long *counterp;
        u_int lockcnt;
 
        /*
-        * The core->cst lock can be shared across several chains so we
-        * need to track the per-chain lockcnt separately.
-        *
         * If multiple locks are present (or being attempted) on this
         * particular chain we can just unlock, drop refs, and return.
         *
@@ -757,7 +752,7 @@ hammer2_chain_unlock(hammer2_chain_t *chain)
                if (lockcnt > 1) {
                        if (atomic_cmpset_int(&chain->lockcnt,
                                              lockcnt, lockcnt - 1)) {
-                               ccms_thread_unlock(&chain->core.cst);
+                               hammer2_mtx_unlock(&chain->core.lock);
                                hammer2_chain_drop(chain);
                                return;
                        }
@@ -776,13 +771,13 @@ hammer2_chain_unlock(hammer2_chain_t *chain)
         * leaving the data/io intact
         *
         * Otherwise if lockcnt is still 0 it is possible for it to become
-        * non-zero and race, but since we hold the core->cst lock
-        * exclusively all that will happen is that the chain will be
-        * reloaded after we unload it.
+        * non-zero and race, but since we hold the core->lock exclusively
+        * all that will happen is that the chain will be reloaded after we
+        * unload it.
         */
-       ostate = ccms_thread_lock_upgrade(&chain->core.cst);
+       ostate = hammer2_mtx_upgrade(&chain->core.lock);
        if (chain->lockcnt) {
-               ccms_thread_unlock_upgraded(&chain->core.cst, ostate);
+               hammer2_mtx_unlock(&chain->core.lock);
                hammer2_chain_drop(chain);
                return;
        }
@@ -796,7 +791,7 @@ hammer2_chain_unlock(hammer2_chain_t *chain)
        if (chain->dio == NULL) {
                if ((chain->flags & HAMMER2_CHAIN_MODIFIED) == 0)
                        hammer2_chain_drop_data(chain, 0);
-               ccms_thread_unlock_upgraded(&chain->core.cst, ostate);
+               hammer2_mtx_unlock(&chain->core.lock);
                hammer2_chain_drop(chain);
                return;
        }
@@ -871,7 +866,7 @@ hammer2_chain_unlock(hammer2_chain_t *chain)
        } else {
                hammer2_io_bqrelse(&chain->dio);
        }
-       ccms_thread_unlock_upgraded(&chain->core.cst, ostate);
+       hammer2_mtx_unlock(&chain->core.lock);
        hammer2_chain_drop(chain);
 }
 
@@ -893,7 +888,7 @@ void
 hammer2_chain_countbrefs(hammer2_chain_t *chain,
                         hammer2_blockref_t *base, int count)
 {
-       spin_lock(&chain->core.cst.spin);
+       hammer2_spin_ex(&chain->core.spin);
         if ((chain->core.flags & HAMMER2_CORE_COUNTEDBREFS) == 0) {
                if (base) {
                        while (--count >= 0) {
@@ -913,7 +908,7 @@ hammer2_chain_countbrefs(hammer2_chain_t *chain,
                /* else do not modify live_count */
                atomic_set_int(&chain->core.flags, HAMMER2_CORE_COUNTEDBREFS);
        }
-       spin_unlock(&chain->core.cst.spin);
+       hammer2_spin_unex(&chain->core.spin);
 }
 
 /*
@@ -1508,10 +1503,10 @@ hammer2_chain_getparent(hammer2_chain_t **parentp, int how)
         * is locked below to avoid a deadlock.
         */
        oparent = *parentp;
-       spin_lock(&oparent->core.cst.spin);
+       hammer2_spin_ex(&oparent->core.spin);
        nparent = oparent->parent;
        hammer2_chain_ref(nparent);
-       spin_unlock(&oparent->core.cst.spin);
+       hammer2_spin_unex(&oparent->core.spin);
        if (oparent) {
                hammer2_chain_unlock(oparent);
                oparent = NULL;
@@ -1699,7 +1694,7 @@ again:
        /*
         * Combined search
         */
-       spin_lock(&parent->core.cst.spin);
+       hammer2_spin_ex(&parent->core.spin);
        chain = hammer2_combined_find(parent, base, count,
                                      cache_indexp, key_nextp,
                                      key_beg, key_end,
@@ -1710,7 +1705,7 @@ again:
         * Exhausted parent chain, iterate.
         */
        if (bref == NULL) {
-               spin_unlock(&parent->core.cst.spin);
+               hammer2_spin_unex(&parent->core.spin);
                if (key_beg == key_end) /* short cut single-key case */
                        return (NULL);
 
@@ -1739,7 +1734,7 @@ again:
         */
        if (chain == NULL) {
                bcopy = *bref;
-               spin_unlock(&parent->core.cst.spin);
+               hammer2_spin_unex(&parent->core.spin);
                chain = hammer2_chain_get(parent, generation,
                                          &bcopy);
                if (chain == NULL) {
@@ -1753,7 +1748,7 @@ again:
                }
        } else {
                hammer2_chain_ref(chain);
-               spin_unlock(&parent->core.cst.spin);
+               hammer2_spin_unex(&parent->core.spin);
        }
 
        /*
@@ -2016,7 +2011,7 @@ again:
                hammer2_chain_countbrefs(parent, base, count);
 
        next_key = 0;
-       spin_lock(&parent->core.cst.spin);
+       hammer2_spin_ex(&parent->core.spin);
        chain = hammer2_combined_find(parent, base, count,
                                      cache_indexp, &next_key,
                                      key, HAMMER2_KEY_MAX,
@@ -2027,7 +2022,7 @@ again:
         * Exhausted parent chain, we're done.
         */
        if (bref == NULL) {
-               spin_unlock(&parent->core.cst.spin);
+               hammer2_spin_unex(&parent->core.spin);
                KKASSERT(chain == NULL);
                goto done;
        }
@@ -2037,7 +2032,7 @@ again:
         */
        if (chain == NULL) {
                bcopy = *bref;
-               spin_unlock(&parent->core.cst.spin);
+               hammer2_spin_unex(&parent->core.spin);
                chain = hammer2_chain_get(parent, generation, &bcopy);
                if (chain == NULL) {
                        kprintf("retry scan parent %p keys %016jx\n",
@@ -2051,7 +2046,7 @@ again:
                }
        } else {
                hammer2_chain_ref(chain);
-               spin_unlock(&parent->core.cst.spin);
+               hammer2_spin_unex(&parent->core.spin);
        }
 
        /*
@@ -2140,7 +2135,7 @@ hammer2_chain_create(hammer2_trans_t *trans, hammer2_chain_t **parentp,
         * Topology may be crossing a PFS boundary.
         */
        parent = *parentp;
-       KKASSERT(ccms_thread_lock_owned(&parent->core.cst));
+       KKASSERT(hammer2_mtx_owned(&parent->core.lock));
        hmp = parent->hmp;
        chain = *chainp;
 
@@ -2166,7 +2161,7 @@ hammer2_chain_create(hammer2_trans_t *trans, hammer2_chain_t **parentp,
                 * to 1 by chain_alloc() for us, but lockcnt is not).
                 */
                chain->lockcnt = 1;
-               ccms_thread_lock(&chain->core.cst, CCMS_STATE_EXCLUSIVE);
+               hammer2_mtx_ex(&chain->core.lock, "h2chnx");
                allocated = 1;
 
                /*
@@ -2463,7 +2458,7 @@ hammer2_chain_rename(hammer2_trans_t *trans, hammer2_blockref_t *bref,
         * Having both chains locked is extremely important for atomicy.
         */
        if (parentp && (parent = *parentp) != NULL) {
-               KKASSERT(ccms_thread_lock_owned(&parent->core.cst));
+               KKASSERT(hammer2_mtx_owned(&parent->core.lock));
                KKASSERT(parent->refs > 0);
 
                hammer2_chain_create(trans, parentp, &chain, chain->pmp,
@@ -2508,7 +2503,7 @@ _hammer2_chain_delete_helper(hammer2_trans_t *trans,
                 * Calculate blockmap pointer
                 */
                KKASSERT(chain->flags & HAMMER2_CHAIN_ONRBTREE);
-               spin_lock(&parent->core.cst.spin);
+               hammer2_spin_ex(&parent->core.spin);
 
                atomic_set_int(&chain->flags, HAMMER2_CHAIN_DELETED);
                atomic_add_int(&parent->core.live_count, -1);
@@ -2588,7 +2583,7 @@ _hammer2_chain_delete_helper(hammer2_trans_t *trans,
                        hammer2_base_delete(trans, parent, base, count,
                                            &cache_index, chain);
                }
-               spin_unlock(&parent->core.cst.spin);
+               hammer2_spin_unex(&parent->core.spin);
        } else if (chain->flags & HAMMER2_CHAIN_ONRBTREE) {
                /*
                 * Chain is not blockmapped but a parent is present.
@@ -2599,7 +2594,7 @@ _hammer2_chain_delete_helper(hammer2_trans_t *trans,
                 * synchronized, the chain's *_count_up fields contain
                 * inode adjustment statistics which must be undone.
                 */
-               spin_lock(&parent->core.cst.spin);
+               hammer2_spin_ex(&parent->core.spin);
                if (chain->bref.type == HAMMER2_BREF_TYPE_INODE &&
                    (flags & HAMMER2_DELETE_NOSTATS) == 0) {
                        KKASSERT(chain->data != NULL);
@@ -2615,7 +2610,7 @@ _hammer2_chain_delete_helper(hammer2_trans_t *trans,
                atomic_clear_int(&chain->flags, HAMMER2_CHAIN_ONRBTREE);
                --parent->core.chain_count;
                chain->parent = NULL;
-               spin_unlock(&parent->core.cst.spin);
+               hammer2_spin_unex(&parent->core.spin);
        } else {
                /*
                 * Chain is not blockmapped and has no parent.  This
@@ -2724,7 +2719,7 @@ hammer2_chain_create_indirect(hammer2_trans_t *trans, hammer2_chain_t *parent,
         */
        hmp = parent->hmp;
        *errorp = 0;
-       KKASSERT(ccms_thread_lock_owned(&parent->core.cst));
+       KKASSERT(hammer2_mtx_owned(&parent->core.lock));
 
        /*hammer2_chain_modify(trans, &parent, HAMMER2_MODIFY_OPTDATA);*/
        if (parent->flags & HAMMER2_CHAIN_INITIAL) {
@@ -2856,13 +2851,13 @@ hammer2_chain_create_indirect(hammer2_trans_t *trans, hammer2_chain_t *parent,
        key_beg = 0;
        key_end = HAMMER2_KEY_MAX;
        cache_index = 0;
-       spin_lock(&parent->core.cst.spin);
+       hammer2_spin_ex(&parent->core.spin);
        loops = 0;
        reason = 0;
 
        for (;;) {
                if (++loops > 100000) {
-                   spin_unlock(&parent->core.cst.spin);
+                   hammer2_spin_unex(&parent->core.spin);
                    panic("excessive loops r=%d p=%p base/count %p:%d %016jx\n",
                          reason, parent, base, count, key_next);
                }
@@ -2905,7 +2900,7 @@ hammer2_chain_create_indirect(hammer2_trans_t *trans, hammer2_chain_t *parent,
                         * Use chain already present in the RBTREE
                         */
                        hammer2_chain_ref(chain);
-                       spin_unlock(&parent->core.cst.spin);
+                       hammer2_spin_unex(&parent->core.spin);
                        hammer2_chain_lock(chain, HAMMER2_RESOLVE_NEVER |
                                                  HAMMER2_RESOLVE_NOREF);
                } else {
@@ -2914,18 +2909,18 @@ hammer2_chain_create_indirect(hammer2_trans_t *trans, hammer2_chain_t *parent,
                         * on insertion race.
                         */
                        bcopy = *bref;
-                       spin_unlock(&parent->core.cst.spin);
+                       hammer2_spin_unex(&parent->core.spin);
                        chain = hammer2_chain_get(parent, generation, &bcopy);
                        if (chain == NULL) {
                                reason = 1;
-                               spin_lock(&parent->core.cst.spin);
+                               hammer2_spin_ex(&parent->core.spin);
                                continue;
                        }
                        if (bcmp(&bcopy, bref, sizeof(bcopy))) {
                                kprintf("REASON 2\n");
                                reason = 2;
                                hammer2_chain_drop(chain);
-                               spin_lock(&parent->core.cst.spin);
+                               hammer2_spin_ex(&parent->core.spin);
                                continue;
                        }
                        hammer2_chain_lock(chain, HAMMER2_RESOLVE_NEVER |
@@ -2966,7 +2961,7 @@ hammer2_chain_create_indirect(hammer2_trans_t *trans, hammer2_chain_t *parent,
                KKASSERT(parent->refs > 0);
                chain = NULL;
 next_key:
-               spin_lock(&parent->core.cst.spin);
+               hammer2_spin_ex(&parent->core.spin);
 next_key_spinlocked:
                if (--maxloops == 0)
                        panic("hammer2_chain_create_indirect: maxloops");
@@ -2976,7 +2971,7 @@ next_key_spinlocked:
                key_beg = key_next;
                /* loop */
        }
-       spin_unlock(&parent->core.cst.spin);
+       hammer2_spin_unex(&parent->core.spin);
 
        /*
         * Insert the new indirect block into the parent now that we've
@@ -3062,7 +3057,7 @@ hammer2_chain_indkey_freemap(hammer2_chain_t *parent, hammer2_key_t *keyp,
        key_beg = 0;
        key_end = HAMMER2_KEY_MAX;
        cache_index = 0;
-       spin_lock(&parent->core.cst.spin);
+       hammer2_spin_ex(&parent->core.spin);
 
        for (;;) {
                if (--maxloops == 0) {
@@ -3108,7 +3103,7 @@ hammer2_chain_indkey_freemap(hammer2_chain_t *parent, hammer2_key_t *keyp,
                        break;
                key_beg = key_next;
        }
-       spin_unlock(&parent->core.cst.spin);
+       hammer2_spin_unex(&parent->core.spin);
 
        /*
         * Return the keybits for a higher-level FREEMAP_NODE covering
@@ -3172,7 +3167,7 @@ hammer2_chain_indkey_normal(hammer2_chain_t *parent, hammer2_key_t *keyp,
        key_beg = 0;
        key_end = HAMMER2_KEY_MAX;
        cache_index = 0;
-       spin_lock(&parent->core.cst.spin);
+       hammer2_spin_ex(&parent->core.spin);
 
        for (;;) {
                if (--maxloops == 0) {
@@ -3259,7 +3254,7 @@ hammer2_chain_indkey_normal(hammer2_chain_t *parent, hammer2_key_t *keyp,
                        break;
                key_beg = key_next;
        }
-       spin_unlock(&parent->core.cst.spin);
+       hammer2_spin_unex(&parent->core.spin);
        bref = NULL;    /* now invalid (safety) */
 
        /*
@@ -3318,7 +3313,7 @@ void
 hammer2_chain_delete(hammer2_trans_t *trans, hammer2_chain_t *parent,
                     hammer2_chain_t *chain, int flags)
 {
-       KKASSERT(ccms_thread_lock_owned(&chain->core.cst));
+       KKASSERT(hammer2_mtx_owned(&chain->core.lock));
 
        /*
         * Nothing to do if already marked.
@@ -3566,7 +3561,7 @@ hammer2_base_delete(hammer2_trans_t *trans, hammer2_chain_t *parent,
            base[i].key != elm->key ||
            ((chain->flags & HAMMER2_CHAIN_BMAPUPD) == 0 &&
             base[i].keybits != elm->keybits)) {
-               spin_unlock(&parent->core.cst.spin);
+               hammer2_spin_unex(&parent->core.spin);
                panic("delete base %p element not found at %d/%d elm %p\n",
                      base, i, count, elm);
                return;
@@ -3645,7 +3640,7 @@ hammer2_base_insert(hammer2_trans_t *trans __unused, hammer2_chain_t *parent,
 
        xkey = elm->key + ((hammer2_key_t)1 << elm->keybits) - 1;
        if (i != count && (base[i].key < elm->key || xkey >= base[i].key)) {
-               spin_unlock(&parent->core.cst.spin);
+               hammer2_spin_unex(&parent->core.spin);
                panic("insert base %p overlapping elements at %d elm %p\n",
                      base, i, elm);
        }
index 57fda67..812b912 100644 (file)
@@ -537,10 +537,10 @@ hammer2_flush_core(hammer2_flush_info_t *info, hammer2_chain_t *chain,
                 */
                atomic_clear_int(&chain->flags, HAMMER2_CHAIN_ONFLUSH);
                info->parent = chain;
-               spin_lock(&chain->core.cst.spin);
+               hammer2_spin_ex(&chain->core.spin);
                RB_SCAN(hammer2_chain_tree, &chain->core.rbtree,
                        NULL, hammer2_flush_recurse, info);
-               spin_unlock(&chain->core.cst.spin);
+               hammer2_spin_unex(&chain->core.spin);
                info->parent = parent;
                if (info->diddeferral)
                        hammer2_chain_setflush(info->trans, chain);
@@ -968,7 +968,7 @@ hammer2_flush_recurse(hammer2_chain_t *child, void *data)
         * unlock it in order to lock the child.
         */
        hammer2_chain_ref(child);
-       spin_unlock(&parent->core.cst.spin);
+       hammer2_spin_unex(&parent->core.spin);
 
        hammer2_chain_unlock(parent);
        hammer2_chain_lock(child, HAMMER2_RESOLVE_MAYBE);
@@ -1002,7 +1002,7 @@ hammer2_flush_recurse(hammer2_chain_t *child, void *data)
        hammer2_chain_lock(parent, HAMMER2_RESOLVE_MAYBE);
        hammer2_chain_drop(child);
        KKASSERT(info->parent == parent);
-       spin_lock(&parent->core.cst.spin);
+       hammer2_spin_ex(&parent->core.spin);
 
        return (0);
 }
index b10e145..12b7c2f 100644 (file)
@@ -97,7 +97,7 @@ hammer2_inode_lock_nex(hammer2_inode_t *ip, int how)
        int i;
 
        hammer2_inode_ref(ip);
-       ccms_thread_lock(&ip->topo_cst, CCMS_STATE_EXCLUSIVE);
+       hammer2_mtx_ex(&ip->lock, "h2ino");
        cluster = hammer2_cluster_copy(&ip->cluster,
                                       HAMMER2_CLUSTER_COPY_NOCHAINS);
 
@@ -142,7 +142,7 @@ hammer2_inode_unlock_ex(hammer2_inode_t *ip, hammer2_cluster_t *cluster)
 {
        if (cluster)
                hammer2_cluster_unlock(cluster);
-       ccms_thread_unlock(&ip->topo_cst);
+       hammer2_mtx_unlock(&ip->lock);
        hammer2_inode_drop(ip);
 }
 
@@ -168,7 +168,7 @@ hammer2_inode_lock_sh(hammer2_inode_t *ip)
        hammer2_inode_ref(ip);
        cluster = hammer2_cluster_copy(&ip->cluster,
                                       HAMMER2_CLUSTER_COPY_NOCHAINS);
-       ccms_thread_lock(&ip->topo_cst, CCMS_STATE_SHARED);
+       hammer2_mtx_sh(&ip->lock, "h2ino");
 
        cluster->focus = NULL;
 
@@ -208,32 +208,63 @@ hammer2_inode_unlock_sh(hammer2_inode_t *ip, hammer2_cluster_t *cluster)
 {
        if (cluster)
                hammer2_cluster_unlock(cluster);
-       ccms_thread_unlock(&ip->topo_cst);
+       hammer2_mtx_unlock(&ip->lock);
        hammer2_inode_drop(ip);
 }
 
-ccms_state_t
+/*
+ * Temporarily release a lock held shared or exclusive.  Caller must
+ * hold the lock shared or exclusive on call and lock will be released
+ * on return.
+ *
+ * Restore a lock that was temporarily released.
+ */
+hammer2_mtx_state_t
 hammer2_inode_lock_temp_release(hammer2_inode_t *ip)
 {
-       return(ccms_thread_lock_temp_release(&ip->topo_cst));
+       return hammer2_mtx_temp_release(&ip->lock);
 }
 
 void
-hammer2_inode_lock_temp_restore(hammer2_inode_t *ip, ccms_state_t ostate)
+hammer2_inode_lock_temp_restore(hammer2_inode_t *ip, hammer2_mtx_state_t ostate)
 {
-       ccms_thread_lock_temp_restore(&ip->topo_cst, ostate);
+       hammer2_mtx_temp_restore(&ip->lock, "h2ino", ostate);
 }
 
-ccms_state_t
+/*
+ * Upgrade a shared inode lock to exclusive and return.  If the inode lock
+ * is already held exclusively this is a NOP.
+ *
+ * The caller MUST hold the inode lock either shared or exclusive on call
+ * and will own the lock exclusively on return.
+ *
+ * Returns non-zero if the lock was already exclusive prior to the upgrade.
+ */
+int
 hammer2_inode_lock_upgrade(hammer2_inode_t *ip)
 {
-       return(ccms_thread_lock_upgrade(&ip->topo_cst));
+       int wasexclusive;
+
+       if (mtx_islocked_ex(&ip->lock)) {
+               wasexclusive = 1;
+       } else {
+               hammer2_mtx_unlock(&ip->lock);
+               hammer2_mtx_ex(&ip->lock, "h2upg");
+               wasexclusive = 0;
+       }
+       return wasexclusive;
 }
 
+/*
+ * Downgrade an inode lock from exclusive to shared only if the inode
+ * lock was previously shared.  If the inode lock was previously exclusive,
+ * this is a NOP.
+ */
 void
-hammer2_inode_lock_downgrade(hammer2_inode_t *ip, ccms_state_t ostate)
+hammer2_inode_lock_downgrade(hammer2_inode_t *ip, int wasexclusive)
 {
-       ccms_thread_lock_downgrade(&ip->topo_cst, ostate);
+       if (wasexclusive == 0)
+               mtx_downgrade(&ip->lock);
 }
 
 /*
@@ -248,11 +279,11 @@ hammer2_inode_lookup(hammer2_pfsmount_t *pmp, hammer2_tid_t inum)
        if (pmp->spmp_hmp) {
                ip = NULL;
        } else {
-               spin_lock(&pmp->inum_spin);
+               hammer2_spin_ex(&pmp->inum_spin);
                ip = RB_LOOKUP(hammer2_inode_tree, &pmp->inum_tree, inum);
                if (ip)
                        hammer2_inode_ref(ip);
-               spin_unlock(&pmp->inum_spin);
+               hammer2_spin_unex(&pmp->inum_spin);
        }
        return(ip);
 }
@@ -287,20 +318,23 @@ hammer2_inode_drop(hammer2_inode_t *ip)
                        /*
                         * Transition to zero, must interlock with
                         * the inode inumber lookup tree (if applicable).
+                        * It should not be possible for anyone to race
+                        * the transition to 0.
+                        *
                         */
                        pmp = ip->pmp;
                        KKASSERT(pmp);
-                       spin_lock(&pmp->inum_spin);
+                       hammer2_spin_ex(&pmp->inum_spin);
 
                        if (atomic_cmpset_int(&ip->refs, 1, 0)) {
-                               KKASSERT(ip->topo_cst.count == 0);
+                               KKASSERT(hammer2_mtx_refs(&ip->lock) == 0);
                                if (ip->flags & HAMMER2_INODE_ONRBTREE) {
                                        atomic_clear_int(&ip->flags,
                                                     HAMMER2_INODE_ONRBTREE);
                                        RB_REMOVE(hammer2_inode_tree,
                                                  &pmp->inum_tree, ip);
                                }
-                               spin_unlock(&pmp->inum_spin);
+                               hammer2_spin_unex(&pmp->inum_spin);
 
                                pip = ip->pip;
                                ip->pip = NULL;
@@ -322,7 +356,7 @@ hammer2_inode_drop(hammer2_inode_t *ip)
                                ip = pip;
                                /* continue with pip (can be NULL) */
                        } else {
-                               spin_unlock(&ip->pmp->inum_spin);
+                               hammer2_spin_unex(&ip->pmp->inum_spin);
                        }
                } else {
                        /*
@@ -349,7 +383,6 @@ hammer2_igetv(hammer2_inode_t *ip, hammer2_cluster_t *cparent, int *errorp)
        const hammer2_inode_data_t *ripdata;
        hammer2_pfsmount_t *pmp;
        struct vnode *vp;
-       ccms_state_t ostate;
 
        pmp = ip->pmp;
        KKASSERT(pmp != NULL);
@@ -364,6 +397,8 @@ hammer2_igetv(hammer2_inode_t *ip, hammer2_cluster_t *cparent, int *errorp)
                 * inode must be unlocked during the vget() to avoid a
                 * deadlock against a reclaim.
                 */
+               int wasexclusive;
+
                vp = ip->vp;
                if (vp) {
                        /*
@@ -374,6 +409,8 @@ hammer2_igetv(hammer2_inode_t *ip, hammer2_cluster_t *cparent, int *errorp)
                         * vget().  The vget() can still fail if we lost
                         * a reclaim race on the vnode.
                         */
+                       hammer2_mtx_state_t ostate;
+
                        vhold(vp);
                        ostate = hammer2_inode_lock_temp_release(ip);
                        if (vget(vp, LK_EXCLUSIVE)) {
@@ -410,11 +447,11 @@ hammer2_igetv(hammer2_inode_t *ip, hammer2_cluster_t *cparent, int *errorp)
                /*
                 * Lock the inode and check for an allocation race.
                 */
-               ostate = hammer2_inode_lock_upgrade(ip);
+               wasexclusive = hammer2_inode_lock_upgrade(ip);
                if (ip->vp != NULL) {
                        vp->v_type = VBAD;
                        vx_put(vp);
-                       hammer2_inode_lock_downgrade(ip, ostate);
+                       hammer2_inode_lock_downgrade(ip, wasexclusive);
                        continue;
                }
 
@@ -463,7 +500,7 @@ hammer2_igetv(hammer2_inode_t *ip, hammer2_cluster_t *cparent, int *errorp)
                vp->v_data = ip;
                ip->vp = vp;
                hammer2_inode_ref(ip);          /* vp association */
-               hammer2_inode_lock_downgrade(ip, ostate);
+               hammer2_inode_lock_downgrade(ip, wasexclusive);
                break;
        }
 
@@ -512,7 +549,7 @@ again:
                if (nip == NULL)
                        break;
 
-               ccms_thread_lock(&nip->topo_cst, CCMS_STATE_EXCLUSIVE);
+               hammer2_mtx_ex(&nip->lock, "h2ino");
 
                /*
                 * Handle SMP race (not applicable to the super-root spmp
@@ -520,7 +557,7 @@ again:
                 */
                if (pmp->spmp_hmp == NULL &&
                    (nip->flags & HAMMER2_INODE_ONRBTREE) == 0) {
-                       ccms_thread_unlock(&nip->topo_cst);
+                       hammer2_mtx_unlock(&nip->lock);
                        hammer2_inode_drop(nip);
                        continue;
                }
@@ -562,8 +599,8 @@ again:
         * hammer2_inode_lock_ex() call.
         */
        nip->refs = 1;
-       ccms_cst_init(&nip->topo_cst, &nip->cluster);
-       ccms_thread_lock(&nip->topo_cst, CCMS_STATE_EXCLUSIVE);
+       hammer2_mtx_init(&nip->lock, "h2ino");
+       hammer2_mtx_ex(&nip->lock, "h2ino");
        /* combination of thread lock and chain lock == inode lock */
 
        /*
@@ -571,15 +608,15 @@ again:
         * get.  Undo all the work and try again.
         */
        if (pmp->spmp_hmp == NULL) {
-               spin_lock(&pmp->inum_spin);
+               hammer2_spin_ex(&pmp->inum_spin);
                if (RB_INSERT(hammer2_inode_tree, &pmp->inum_tree, nip)) {
-                       spin_unlock(&pmp->inum_spin);
-                       ccms_thread_unlock(&nip->topo_cst);
+                       hammer2_spin_unex(&pmp->inum_spin);
+                       hammer2_mtx_unlock(&nip->lock);
                        hammer2_inode_drop(nip);
                        goto again;
                }
                atomic_set_int(&nip->flags, HAMMER2_INODE_ONRBTREE);
-               spin_unlock(&pmp->inum_spin);
+               hammer2_spin_unex(&pmp->inum_spin);
        }
 
        return (nip);
index ac2066f..ce34647 100644 (file)
@@ -97,34 +97,34 @@ hammer2_io_getblk(hammer2_mount_t *hmp, off_t lbase, int lsize,
        /*
         * Access/Allocate the DIO, bump dio->refs to prevent destruction.
         */
-       spin_lock_shared(&hmp->io_spin);
+       hammer2_spin_sh(&hmp->io_spin);
        dio = RB_LOOKUP(hammer2_io_tree, &hmp->iotree, pbase);
        if (dio) {
                if ((atomic_fetchadd_int(&dio->refs, 1) &
                     HAMMER2_DIO_MASK) == 0) {
                        atomic_add_int(&dio->hmp->iofree_count, -1);
                }
-               spin_unlock_shared(&hmp->io_spin);
+               hammer2_spin_unsh(&hmp->io_spin);
        } else {
-               spin_unlock_shared(&hmp->io_spin);
+               hammer2_spin_unsh(&hmp->io_spin);
                dio = kmalloc(sizeof(*dio), M_HAMMER2, M_INTWAIT | M_ZERO);
                dio->hmp = hmp;
                dio->pbase = pbase;
                dio->psize = psize;
                dio->refs = 1;
-               spin_init(&dio->spin, "h2dio");
+               hammer2_spin_init(&dio->spin, "h2dio");
                TAILQ_INIT(&dio->iocbq);
-               spin_lock(&hmp->io_spin);
+               hammer2_spin_ex(&hmp->io_spin);
                xio = RB_INSERT(hammer2_io_tree, &hmp->iotree, dio);
                if (xio == NULL) {
                        atomic_add_int(&hammer2_dio_count, 1);
-                       spin_unlock(&hmp->io_spin);
+                       hammer2_spin_unex(&hmp->io_spin);
                } else {
                        if ((atomic_fetchadd_int(&xio->refs, 1) &
                             HAMMER2_DIO_MASK) == 0) {
                                atomic_add_int(&xio->hmp->iofree_count, -1);
                        }
-                       spin_unlock(&hmp->io_spin);
+                       hammer2_spin_unex(&hmp->io_spin);
                        kfree(dio, M_HAMMER2);
                        dio = xio;
                }
@@ -160,16 +160,16 @@ hammer2_io_getblk(hammer2_mount_t *hmp, off_t lbase, int lsize,
                         * If DIO_INPROG is already set then set WAITING and
                         * queue the iocb.
                         */
-                       spin_lock(&dio->spin);
+                       hammer2_spin_ex(&dio->spin);
                        if (atomic_cmpset_int(&dio->refs, refs,
                                              refs | HAMMER2_DIO_WAITING)) {
                                iocb->flags |= HAMMER2_IOCB_ONQ |
                                               HAMMER2_IOCB_INPROG;
                                TAILQ_INSERT_TAIL(&dio->iocbq, iocb, entry);
-                               spin_unlock(&dio->spin);
+                               hammer2_spin_unex(&dio->spin);
                                break;
                        }
-                       spin_unlock(&dio->spin);
+                       hammer2_spin_unex(&dio->spin);
                        /* retry */
                } else {
                        /*
@@ -246,7 +246,7 @@ hammer2_io_complete(hammer2_iocb_t *iocb)
                nrefs = orefs & ~(HAMMER2_DIO_WAITING | HAMMER2_DIO_INPROG);
 
                if (orefs & HAMMER2_DIO_WAITING) {
-                       spin_lock(&dio->spin);
+                       hammer2_spin_ex(&dio->spin);
                        cbtmp = TAILQ_FIRST(&dio->iocbq);
                        if (cbtmp) {
                                /*
@@ -255,15 +255,15 @@ hammer2_io_complete(hammer2_iocb_t *iocb)
                                 *       iocb.
                                 */
                                TAILQ_REMOVE(&dio->iocbq, cbtmp, entry);
-                               spin_unlock(&dio->spin);
+                               hammer2_spin_unex(&dio->spin);
                                cbtmp->callback(cbtmp); /* chained */
                                break;
                        } else if (atomic_cmpset_int(&dio->refs,
                                                     orefs, nrefs)) {
-                               spin_unlock(&dio->spin);
+                               hammer2_spin_unex(&dio->spin);
                                break;
                        }
-                       spin_unlock(&dio->spin);
+                       hammer2_spin_unex(&dio->spin);
                        /* retry */
                } else if (atomic_cmpset_int(&dio->refs, orefs, nrefs)) {
                        break;
@@ -441,13 +441,13 @@ hammer2_io_putblk(hammer2_io_t **diop)
                struct hammer2_cleanupcb_info info;
 
                RB_INIT(&info.tmptree);
-               spin_lock(&hmp->io_spin);
+               hammer2_spin_ex(&hmp->io_spin);
                if (hmp->iofree_count > 1000) {
                        info.count = hmp->iofree_count / 2;
                        RB_SCAN(hammer2_io_tree, &hmp->iotree, NULL,
                                hammer2_io_cleanup_callback, &info);
                }
-               spin_unlock(&hmp->io_spin);
+               hammer2_spin_unex(&hmp->io_spin);
                hammer2_io_cleanup(hmp, &info.tmptree);
        }
 }
index 128583e..af479f8 100644 (file)
 /*
  * Mount-wide locks
  */
-
 void
 hammer2_mount_exlock(hammer2_mount_t *hmp)
 {
-       ccms_thread_lock(&hmp->vchain.core.cst, CCMS_STATE_EXCLUSIVE);
+       hammer2_mtx_ex(&hmp->vchain.core.lock, "h2mount");
 }
 
 void
 hammer2_mount_shlock(hammer2_mount_t *hmp)
 {
-       ccms_thread_lock(&hmp->vchain.core.cst, CCMS_STATE_SHARED);
+       hammer2_mtx_sh(&hmp->vchain.core.lock, "h2mount");
 }
 
 void
 hammer2_mount_unlock(hammer2_mount_t *hmp)
 {
-       ccms_thread_unlock(&hmp->vchain.core.cst);
+       hammer2_mtx_unlock(&hmp->vchain.core.lock);
 }
 
 /*
index 7b2cbb3..83c29ef 100644 (file)
@@ -338,7 +338,7 @@ hammer2_pfsalloc(const hammer2_inode_data_t *ripdata, hammer2_tid_t alloc_tid)
                pmp->inode_tid = ripdata->pfs_inum + 1;
                pmp->pfs_clid = ripdata->pfs_clid;
        }
-       mtx_init(&pmp->wthread_mtx);
+       hammer2_mtx_init(&pmp->wthread_mtx, "h2wthr");
        bioq_init(&pmp->wthread_bioq);
 
        return pmp;
@@ -736,14 +736,18 @@ hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
                int i;
                int j;
 
+               /*
+                * Directly lock the inode->lock, do not run through
+                * hammer2_inode_lock*().
+                */
                hammer2_inode_ref(pmp->iroot);
-               ccms_thread_lock(&pmp->iroot->topo_cst, CCMS_STATE_EXCLUSIVE);
+               hammer2_mtx_ex(&pmp->iroot->lock, "h2ino");
 
                if (pmp->iroot->cluster.nchains + cluster->nchains >
                    HAMMER2_MAXCLUSTER) {
                        kprintf("hammer2_mount: cluster full!\n");
 
-                       ccms_thread_unlock(&pmp->iroot->topo_cst);
+                       hammer2_mtx_unlock(&pmp->iroot->lock);
                        hammer2_inode_drop(pmp->iroot);
 
                        hammer2_cluster_unlock(cluster);
@@ -764,7 +768,7 @@ hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
                        ++j;
                }
                pmp->iroot->cluster.nchains = j;
-               ccms_thread_unlock(&pmp->iroot->topo_cst);
+               hammer2_mtx_unlock(&pmp->iroot->lock);
                hammer2_inode_drop(pmp->iroot);
                hammer2_cluster_unlock(cluster);
                lockmgr(&hammer2_mntlk, LK_RELEASE);
@@ -791,7 +795,6 @@ hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
        }
        cluster->pmp = pmp;
 
-       ccms_domain_init(&pmp->ccms_dom);
        TAILQ_INSERT_TAIL(&hammer2_pfslist, pmp, mntentry);
        lockmgr(&hammer2_mntlk, LK_RELEASE);
 
@@ -887,7 +890,7 @@ hammer2_write_thread(void *arg)
        
        pmp = arg;
        
-       mtx_lock(&pmp->wthread_mtx);
+       hammer2_mtx_ex(&pmp->wthread_mtx, "h2wth");
        while (pmp->wthread_destroy == 0) {
                if (bioq_first(&pmp->wthread_bioq) == NULL) {
                        mtxsleep(&pmp->wthread_bioq, &pmp->wthread_mtx,
@@ -914,7 +917,7 @@ hammer2_write_thread(void *arg)
                        /*
                         * else normal bio processing
                         */
-                       mtx_unlock(&pmp->wthread_mtx);
+                       hammer2_mtx_unlock(&pmp->wthread_mtx);
 
                        hammer2_lwinprog_drop(pmp);
                        
@@ -954,14 +957,14 @@ hammer2_write_thread(void *arg)
                                bp->b_error = EIO;
                        }
                        biodone(bio);
-                       mtx_lock(&pmp->wthread_mtx);
+                       hammer2_mtx_ex(&pmp->wthread_mtx, "h2wth");
                }
                hammer2_trans_done(&trans);
        }
        pmp->wthread_destroy = -1;
        wakeup(&pmp->wthread_destroy);
        
-       mtx_unlock(&pmp->wthread_mtx);
+       hammer2_mtx_unlock(&pmp->wthread_mtx);
 }
 
 void
@@ -970,14 +973,14 @@ hammer2_bioq_sync(hammer2_pfsmount_t *pmp)
        struct bio sync_bio;
 
        bzero(&sync_bio, sizeof(sync_bio));     /* dummy with no bio_buf */
-       mtx_lock(&pmp->wthread_mtx);
+       hammer2_mtx_ex(&pmp->wthread_mtx, "h2wth");
        if (pmp->wthread_destroy == 0 &&
            TAILQ_FIRST(&pmp->wthread_bioq.queue)) {
                bioq_insert_tail(&pmp->wthread_bioq, &sync_bio);
                while ((sync_bio.bio_flags & BIO_DONE) == 0)
                        mtxsleep(&sync_bio, &pmp->wthread_mtx, 0, "h2bioq", 0);
        }
-       mtx_unlock(&pmp->wthread_mtx);
+       hammer2_mtx_unlock(&pmp->wthread_mtx);
 }
 
 /* 
@@ -1589,10 +1592,8 @@ hammer2_vfs_unmount(struct mount *mp, int mntflags)
                        goto failed;
        }
 
-       ccms_domain_uninit(&pmp->ccms_dom);
-
        if (pmp->wthread_td) {
-               mtx_lock(&pmp->wthread_mtx);
+               hammer2_mtx_ex(&pmp->wthread_mtx, "h2wth");
                pmp->wthread_destroy = 1;
                wakeup(&pmp->wthread_bioq);
                while (pmp->wthread_destroy != -1) {
@@ -1600,7 +1601,7 @@ hammer2_vfs_unmount(struct mount *mp, int mntflags)
                                &pmp->wthread_mtx, 0,
                                "umount-sleep", 0);
                }
-               mtx_unlock(&pmp->wthread_mtx);
+               hammer2_mtx_unlock(&pmp->wthread_mtx);
                pmp->wthread_td = NULL;
        }
 
index 3d603d3..75a969c 100644 (file)
@@ -287,9 +287,9 @@ hammer2_vop_reclaim(struct vop_reclaim_args *ap)
                ipul = kmalloc(sizeof(*ipul), pmp->minode, M_WAITOK | M_ZERO);
                ipul->ip = ip;
 
-               spin_lock(&pmp->list_spin);
+               hammer2_spin_ex(&pmp->list_spin);
                TAILQ_INSERT_TAIL(&pmp->unlinkq, ipul, entry);
-               spin_unlock(&pmp->list_spin);
+               hammer2_spin_unex(&pmp->list_spin);
                hammer2_inode_unlock_ex(ip, cluster);   /* unlock */
                /* retain ref from vp for ipul */
        } else {
@@ -937,10 +937,13 @@ hammer2_read_file(hammer2_inode_t *ip, struct uio *uio, int seqcount)
 
        /*
         * UIO read loop.
+        *
+        * WARNING! Assumes that the kernel interlocks size changes at the
+        *          vnode level.
         */
-       ccms_thread_lock(&ip->topo_cst, CCMS_STATE_EXCLUSIVE);
+       hammer2_mtx_sh(&ip->lock, "h2ino");
        size = ip->size;
-       ccms_thread_unlock(&ip->topo_cst);
+       hammer2_mtx_unlock(&ip->lock);
 
        while (uio->uio_resid > 0 && uio->uio_offset < size) {
                hammer2_key_t lbase;
@@ -991,12 +994,15 @@ hammer2_write_file(hammer2_inode_t *ip,
 
        /*
         * Setup if append
+        *
+        * WARNING! Assumes that the kernel interlocks size changes at the
+        *          vnode level.
         */
-       ccms_thread_lock(&ip->topo_cst, CCMS_STATE_EXCLUSIVE);
+       hammer2_mtx_ex(&ip->lock, "h2ino");
        if (ioflag & IO_APPEND)
                uio->uio_offset = ip->size;
        old_eof = ip->size;
-       ccms_thread_unlock(&ip->topo_cst);
+       hammer2_mtx_unlock(&ip->lock);
 
        /*
         * Extend the file if necessary.  If the write fails at some point
@@ -1143,10 +1149,10 @@ hammer2_write_file(hammer2_inode_t *ip,
        if (error && new_eof != old_eof) {
                hammer2_truncate_file(ip, old_eof);
        } else if (modified) {
-               ccms_thread_lock(&ip->topo_cst, CCMS_STATE_EXCLUSIVE);
+               hammer2_mtx_ex(&ip->lock, "h2ino");
                hammer2_update_time(&ip->mtime);
                atomic_set_int(&ip->flags, HAMMER2_INODE_MTIME);
-               ccms_thread_unlock(&ip->topo_cst);
+               hammer2_mtx_unlock(&ip->lock);
        }
        atomic_set_int(&ip->flags, HAMMER2_INODE_MODIFIED);
        hammer2_knote(ip->vp, kflags);
@@ -1162,6 +1168,9 @@ hammer2_write_file(hammer2_inode_t *ip,
  *
  * WARNING: nvtruncbuf() can only be safely called without the inode lock
  *         held due to the way our write thread works.
+ *
+ * WARNING! Assumes that the kernel interlocks size changes at the
+ *         vnode level.
  */
 static
 void
@@ -1177,16 +1186,19 @@ hammer2_truncate_file(hammer2_inode_t *ip, hammer2_key_t nsize)
                           nblksize, (int)nsize & (nblksize - 1),
                           0);
        }
-       ccms_thread_lock(&ip->topo_cst, CCMS_STATE_EXCLUSIVE);
+       hammer2_mtx_ex(&ip->lock, "h2ino");
        ip->size = nsize;
        atomic_set_int(&ip->flags, HAMMER2_INODE_RESIZED);
-       ccms_thread_unlock(&ip->topo_cst);
+       hammer2_mtx_unlock(&ip->lock);
        LOCKSTOP;
 }
 
 /*
  * Extend the size of a file.  The inode must not be locked.
  *
+ * WARNING! Assumes that the kernel interlocks size changes at the
+ *         vnode level.
+ *
  * NOTE: Caller handles setting HAMMER2_INODE_MODIFIED
  */
 static
@@ -1199,10 +1211,10 @@ hammer2_extend_file(hammer2_inode_t *ip, hammer2_key_t nsize)
        int nblksize;
 
        LOCKSTART;
-       ccms_thread_lock(&ip->topo_cst, CCMS_STATE_EXCLUSIVE);
+       hammer2_mtx_ex(&ip->lock, "h2ino");
        osize = ip->size;
        ip->size = nsize;
-       ccms_thread_unlock(&ip->topo_cst);
+       hammer2_mtx_unlock(&ip->lock);
 
        if (ip->vp) {
                oblksize = hammer2_calc_logical(ip, osize, &lbase, NULL);
@@ -2316,14 +2328,14 @@ hammer2_strategy_write(struct vop_strategy_args *ap)
        pmp = ip->pmp;
        
        hammer2_lwinprog_ref(pmp);
-       mtx_lock(&pmp->wthread_mtx);
+       hammer2_mtx_ex(&pmp->wthread_mtx, "h2wth");
        if (TAILQ_EMPTY(&pmp->wthread_bioq.queue)) {
                bioq_insert_tail(&pmp->wthread_bioq, ap->a_bio);
-               mtx_unlock(&pmp->wthread_mtx);
+               hammer2_mtx_unlock(&pmp->wthread_mtx);
                wakeup(&pmp->wthread_bioq);
        } else {
                bioq_insert_tail(&pmp->wthread_bioq, ap->a_bio);
-               mtx_unlock(&pmp->wthread_mtx);
+               hammer2_mtx_unlock(&pmp->wthread_mtx);
        }
        hammer2_lwinprog_wait(pmp);
 
@@ -2396,10 +2408,10 @@ hammer2_run_unlinkq(hammer2_trans_t *trans, hammer2_pfsmount_t *pmp)
                return;
 
        LOCKSTART;
-       spin_lock(&pmp->list_spin);
+       hammer2_spin_ex(&pmp->list_spin);
        while ((ipul = TAILQ_FIRST(&pmp->unlinkq)) != NULL) {
                TAILQ_REMOVE(&pmp->unlinkq, ipul, entry);
-               spin_unlock(&pmp->list_spin);
+               hammer2_spin_unex(&pmp->list_spin);
                ip = ipul->ip;
                kfree(ipul, pmp->minode);
 
@@ -2418,9 +2430,9 @@ hammer2_run_unlinkq(hammer2_trans_t *trans, hammer2_pfsmount_t *pmp)
                hammer2_inode_unlock_ex(ip, cluster);   /* inode lock */
                hammer2_inode_drop(ip);                 /* ipul ref */
 
-               spin_lock(&pmp->list_spin);
+               hammer2_spin_ex(&pmp->list_spin);
        }
-       spin_unlock(&pmp->list_spin);
+       hammer2_spin_unex(&pmp->list_spin);
        LOCKSTOP;
 }