hammer2 - Integrate CCMS thread lock into hammer2 chain structure
authorMatthew Dillon <dillon@apollo.backplane.com>
Fri, 8 Jun 2012 06:52:01 +0000 (23:52 -0700)
committerMatthew Dillon <dillon@apollo.backplane.com>
Fri, 8 Jun 2012 06:52:01 +0000 (23:52 -0700)
* Integrate the CCMS thread lock into the hammer2 chain structure.

* Implement shared and exclusive modes (hammer2 was only really using
  exclusive mode before).  Rework all the chain and inode locking functions
  to use CCMS via chain->cst.

  This also required changing the SPLAY trees into RB trees.

* Start reworking non-modifying VNOPS to use shared CCMS locks.

* Rework the hammer2_chain_drop() function to avoid deadlocks due to the
  mixed shared/exclusive locks we now support.

* Major performance improvements for concurrent access.  SHARED locks now
  extend to hammer2_chain and hammer2_inode structural accesses, recursions,
  and cached data (buffer cache) accesses.

  In particular, multiple threads can now access the same bp via a
  hammer2_chain locked shared.  The bp's themselves are still exclusive
  only (the kernel APIs haven't changed), but the hammer2_chain structure
  can now share the bp's data across several threads accessing it via the
  chain.

sys/vfs/hammer2/Makefile
sys/vfs/hammer2/hammer2.h
sys/vfs/hammer2/hammer2_ccms.c
sys/vfs/hammer2/hammer2_ccms.h
sys/vfs/hammer2/hammer2_chain.c
sys/vfs/hammer2/hammer2_inode.c
sys/vfs/hammer2/hammer2_subr.c
sys/vfs/hammer2/hammer2_vfsops.c
sys/vfs/hammer2/hammer2_vnops.c

index 67f60b1..1dc3fd8 100644 (file)
@@ -3,7 +3,7 @@
 #
 .PATH: ${.CURDIR}
 
-CFLAGS+= -DINVARIANTS
+CFLAGS+= -DINVARIANTS -DSMP
 KMOD=  hammer2
 SRCS=  hammer2_vfsops.c hammer2_vnops.c hammer2_inode.c hammer2_ccms.c
 SRCS+= hammer2_chain.c hammer2_freemap.c hammer2_subr.c hammer2_icrc.c
index 029e50a..37449fe 100644 (file)
@@ -98,6 +98,7 @@ struct hammer2_pfsmount;
  * not match the blockref at (parent, index).
  */
 RB_HEAD(hammer2_chain_tree, hammer2_chain);
+TAILQ_HEAD(flush_deferral_list, hammer2_chain);
 
 struct hammer2_chain {
        ccms_cst_t      cst;                    /* attr or data cst */
@@ -119,7 +120,6 @@ struct hammer2_chain {
        u_int           bytes;          /* physical size of data */
        int             index;          /* index in parent */
        u_int           refs;
-       u_int           busy;           /* soft-busy */
        u_int           flags;
 };
 
@@ -157,6 +157,7 @@ RB_PROTOTYPE(hammer2_chain_tree, hammer2_chain, rbnode, hammer2_chain_cmp);
  */
 #define HAMMER2_LOOKUP_NOLOCK          0x00000001      /* ref only */
 #define HAMMER2_LOOKUP_NODATA          0x00000002      /* data left NULL */
+#define HAMMER2_LOOKUP_SHARED          0x00000100
 
 /*
  * Flags passed to hammer2_chain_modify() and hammer2_chain_resize()
@@ -178,6 +179,9 @@ RB_PROTOTYPE(hammer2_chain_tree, hammer2_chain, rbnode, hammer2_chain_cmp);
 #define HAMMER2_RESOLVE_NEVER          1
 #define HAMMER2_RESOLVE_MAYBE          2
 #define HAMMER2_RESOLVE_ALWAYS         3
+#define HAMMER2_RESOLVE_MASK           0x0F
+
+#define HAMMER2_RESOLVE_SHARED         0x10
 
 /*
  * Cluster different types of storage together for allocations
@@ -242,12 +246,6 @@ struct hammer2_inode {
 
 typedef struct hammer2_inode hammer2_inode_t;
 
-#if defined(_KERNEL)
-
-#define attr_cst       chain.cst
-
-#endif
-
 /*
  * A hammer2 indirect block
  */
index 4d3c8b4..771d921 100644 (file)
@@ -73,6 +73,7 @@ void
 ccms_cst_init(ccms_cst_t *cst, void *handle)
 {
        bzero(cst, sizeof(*cst));
+       spin_init(&cst->spin);
        cst->handle = handle;
 }
 
@@ -184,20 +185,30 @@ ccms_lock_put(ccms_lock_t *lock)
 void
 ccms_thread_lock(ccms_cst_t *cst, ccms_state_t state)
 {
+       /*
+        * Regardless of the type of lock requested if the current thread
+        * already holds an exclusive lock we bump the exclusive count and
+        * return.  This requires no spinlock.
+        */
        if (cst->count < 0 && cst->td == curthread) {
                --cst->count;
                return;
        }
 
+       /*
+        * Otherwise use the spinlock to interlock the operation and sleep
+        * as necessary.
+        */
        spin_lock(&cst->spin);
        if (state == CCMS_STATE_SHARED) {
-               while (cst->count < 0) {
+               while (cst->count < 0 || cst->upgrade) {
                        cst->blocked = 1;
                        ssleep(cst, &cst->spin, 0, "ccmslck", hz);
                }
                ++cst->count;
+               KKASSERT(cst->td == NULL);
        } else if (state == CCMS_STATE_EXCLUSIVE) {
-               while (cst->count != 0) {
+               while (cst->count != 0 || cst->upgrade) {
                        cst->blocked = 1;
                        ssleep(cst, &cst->spin, 0, "ccmslck", hz);
                }
@@ -224,13 +235,14 @@ ccms_thread_lock_nonblock(ccms_cst_t *cst, ccms_state_t state)
 
        spin_lock(&cst->spin);
        if (state == CCMS_STATE_SHARED) {
-               if (cst->count < 0) {
+               if (cst->count < 0 || cst->upgrade) {
                        spin_unlock(&cst->spin);
                        return (EBUSY);
                }
                ++cst->count;
+               KKASSERT(cst->td == NULL);
        } else if (state == CCMS_STATE_EXCLUSIVE) {
-               if (cst->count != 0) {
+               if (cst->count != 0 || cst->upgrade) {
                        spin_unlock(&cst->spin);
                        return (EBUSY);
                }
@@ -244,6 +256,63 @@ ccms_thread_lock_nonblock(ccms_cst_t *cst, ccms_state_t state)
        return(0);
 }
 
+/*
+ * Temporarily upgrade a thread lock for making local structural changes.
+ * No new shared or exclusive locks can be acquired by others while we are
+ * upgrading, but other upgraders are allowed.
+ */
+ccms_state_t
+ccms_thread_lock_upgrade(ccms_cst_t *cst)
+{
+       /*
+        * Nothing to do if already exclusive
+        */
+       if (cst->count < 0) {
+               KKASSERT(cst->td == curthread);
+               return(CCMS_STATE_EXCLUSIVE);
+       }
+
+       /*
+        * Convert a shared lock to exclusive.
+        */
+       if (cst->count > 0) {
+               spin_lock(&cst->spin);
+               ++cst->upgrade;
+               --cst->count;
+               while (cst->count) {
+                       cst->blocked = 1;
+                       ssleep(cst, &cst->spin, 0, "ccmsupg", hz);
+               }
+               cst->count = -1;
+               cst->td = curthread;
+               spin_unlock(&cst->spin);
+               return(CCMS_STATE_SHARED);
+       }
+       panic("ccms_thread_lock_upgrade: not locked");
+       /* NOT REACHED */
+       return(0);
+}
+
+void
+ccms_thread_lock_restore(ccms_cst_t *cst, ccms_state_t ostate)
+{
+       if (ostate == CCMS_STATE_SHARED) {
+               KKASSERT(cst->td == curthread);
+               KKASSERT(cst->count == -1);
+               spin_lock(&cst->spin);
+               --cst->upgrade;
+               cst->count = 1;
+               cst->td = NULL;
+               if (cst->blocked) {
+                       cst->blocked = 0;
+                       spin_unlock(&cst->spin);
+                       wakeup(cst);
+               } else {
+                       spin_unlock(&cst->spin);
+               }
+       }
+}
+
 /*
  * Release a local thread lock
  */
@@ -251,6 +320,7 @@ void
 ccms_thread_unlock(ccms_cst_t *cst)
 {
        if (cst->count < 0) {
+               KKASSERT(cst->td == curthread);
                if (cst->count < -1) {
                        ++cst->count;
                        return;
@@ -284,27 +354,69 @@ ccms_thread_unlock(ccms_cst_t *cst)
  * Release a local thread lock with special handling of the last lock
  * reference.
  *
- * On the last lock reference the lock, if shared, will be upgraded to
- * an exclusive lock and we return 0 without unlocking it.
+ * If no upgrades are in progress then the last reference to the lock will
+ * upgrade the lock to exclusive (if it was shared) and return 0 without
+ * unlocking it.
  *
- * If more than one reference remains we drop the reference and return
- * non-zero.
+ * If more than one reference remains, or upgrades are in progress,
+ * we drop the reference and return non-zero to indicate that more
+ * locks are present or pending.
  */
 int
 ccms_thread_unlock_zero(ccms_cst_t *cst)
 {
        if (cst->count < 0) {
-               if (cst->count == -1)
+               /*
+                * Exclusive owned by us, no races possible as long as it
+                * remains negative.  Return 0 and leave us locked on the
+                * last lock.
+                */
+               KKASSERT(cst->td == curthread);
+               if (cst->count == -1) {
+                       spin_lock(&cst->spin);
+                       if (cst->upgrade) {
+                               cst->count = 0;
+                               if (cst->blocked) {
+                                       cst->blocked = 0;
+                                       spin_unlock(&cst->spin);
+                                       wakeup(cst);
+                               } else {
+                                       spin_unlock(&cst->spin);
+                               }
+                               return(1);
+                       }
+                       spin_unlock(&cst->spin);
                        return(0);
+               }
                ++cst->count;
        } else {
-               KKASSERT(cst->count > 0);
+               /*
+                * Convert the last shared lock to an exclusive lock
+                * and return 0.
+                *
+                * If there are upgrades pending the cst is unlocked and
+                * the upgrade waiters are woken up.  The upgrade count
+                * prevents new exclusive holders for the duration.
+                */
                spin_lock(&cst->spin);
+               KKASSERT(cst->count > 0);
                if (cst->count == 1) {
-                       cst->count = -1;
-                       cst->td = curthread;
-                       spin_unlock(&cst->spin);
-                       return(0);
+                       if (cst->upgrade) {
+                               cst->count = 0;
+                               if (cst->blocked) {
+                                       cst->blocked = 0;
+                                       spin_unlock(&cst->spin);
+                                       wakeup(cst);
+                               } else {
+                                       spin_unlock(&cst->spin);
+                               }
+                               return(1);
+                       } else {
+                               cst->count = -1;
+                               cst->td = curthread;
+                               spin_unlock(&cst->spin);
+                               return(0);
+                       }
                }
                --cst->count;
                spin_unlock(&cst->spin);
@@ -312,6 +424,13 @@ ccms_thread_unlock_zero(ccms_cst_t *cst)
        return(1);
 }
 
+int
+ccms_thread_lock_owned(ccms_cst_t *cst)
+{
+       return(cst->count < 0 && cst->td == curthread);
+}
+
+
 #if 0
 /*
  * Acquire remote grant state.  This routine can be used to upgrade or
index 510f13a..5da3799 100644 (file)
@@ -198,6 +198,7 @@ struct ccms_cst {
        ccms_key_t      key_beg;        /* key range (inclusive) */
        ccms_key_t      key_end;        /* key range (inclusive) */
 
+       int32_t         upgrade;        /* upgrades pending */
        int32_t         count;          /* active shared/exclusive count */
        int32_t         blocked;        /* wakeup blocked on release */
        thread_t        td;             /* if excl lock (count < 0) */
@@ -227,8 +228,11 @@ void ccms_cst_uninit(ccms_cst_t *cst);
 
 void ccms_thread_lock(ccms_cst_t *cst, ccms_state_t state);
 int ccms_thread_lock_nonblock(ccms_cst_t *cst, ccms_state_t state);
+ccms_state_t ccms_thread_lock_upgrade(ccms_cst_t *cst);
+void ccms_thread_lock_restore(ccms_cst_t *cst, ccms_state_t ostate);
 void ccms_thread_unlock(ccms_cst_t *cst);
 int ccms_thread_unlock_zero(ccms_cst_t *cst);
+int ccms_thread_lock_owned(ccms_cst_t *cst);
 
 void ccms_lock_get(ccms_lock_t *lock);
 void ccms_lock_put(ccms_lock_t *lock);
index ee6259f..673c0e7 100644 (file)
@@ -149,6 +149,66 @@ hammer2_chain_alloc(hammer2_mount_t *hmp, hammer2_blockref_t *bref)
        return (chain);
 }
 
+/*
+ * Deallocate a chain (the step before freeing it).  Remove the chain from
+ * its parent's tree.
+ *
+ * Caller must hold the parent and the chain exclusively locked, and
+ * chain->refs must be 0.
+ *
+ * This function unlocks, removes, and destroys chain, and will recursively
+ * destroy any sub-chains under chain (whos refs must also be 0 at this
+ * point).
+ *
+ * parent can be NULL.
+ */
+static void
+hammer2_chain_dealloc(hammer2_mount_t *hmp, hammer2_chain_t *chain)
+{
+       hammer2_inode_t *ip;
+       hammer2_chain_t *parent;
+       hammer2_chain_t *child;
+
+       KKASSERT(chain->refs == 0);
+       KKASSERT((chain->flags &
+                 (HAMMER2_CHAIN_MOVED | HAMMER2_CHAIN_MODIFIED)) == 0);
+
+       parent = chain->parent;
+       chain->parent = NULL;
+       if (chain->bref.type == HAMMER2_BREF_TYPE_INODE)
+               ip = chain->u.ip;
+       else
+               ip = NULL;
+
+       /*
+        * If the sub-tree is not empty all the elements on it must have
+        * 0 refs and be deallocatable.
+        */
+       while ((child = RB_ROOT(&chain->rbhead)) != NULL) {
+               ccms_thread_lock(&child->cst, CCMS_STATE_EXCLUSIVE);
+               hammer2_chain_dealloc(hmp, child);
+       }
+
+       /*
+        * If the DELETED flag is not set the chain must be removed from
+        * its parent's tree.
+        */
+       if ((chain->flags & HAMMER2_CHAIN_DELETED) == 0) {
+               RB_REMOVE(hammer2_chain_tree, &parent->rbhead, chain);
+               atomic_set_int(&chain->flags, HAMMER2_CHAIN_DELETED);
+               if (ip)
+                       ip->pip = NULL;
+       }
+
+       /*
+        * When cleaning out a hammer2_inode we must
+        * also clean out the related ccms_inode.
+        */
+       if (ip)
+               ccms_cst_uninit(&ip->topo_cst);
+       hammer2_chain_free(hmp, chain);
+}
+
 /*
  * Free a disconnected chain element
  */
@@ -166,6 +226,9 @@ hammer2_chain_free(hammer2_mount_t *hmp, hammer2_chain_t *chain)
        KKASSERT(chain->data == NULL);
        KKASSERT(chain->bref.type != HAMMER2_BREF_TYPE_INODE ||
                 chain->u.ip->vp == NULL);
+       ccms_thread_unlock(&chain->cst);
+       KKASSERT(chain->cst.count == 0);
+       KKASSERT(chain->cst.upgrade == 0);
 
        if ((mem = chain->u.mem) != NULL) {
                chain->u.mem = NULL;
@@ -177,35 +240,61 @@ hammer2_chain_free(hammer2_mount_t *hmp, hammer2_chain_t *chain)
 }
 
 /*
- * Add a reference to a chain element (for shared access).  The chain
- * element must already have at least 1 ref controlled by the caller.
+ * Add a reference to a chain element, preventing its destruction.
+ *
+ * The parent chain must be locked shared or exclusive or otherwise be
+ * stable and already have a reference.
  */
 void
 hammer2_chain_ref(hammer2_mount_t *hmp, hammer2_chain_t *chain)
 {
-       KKASSERT(chain->refs > 0);
-       atomic_add_int(&chain->refs, 1);
+       u_int refs;
+
+       while (chain) {
+               refs = chain->refs;
+               KKASSERT(chain->refs >= 0);
+               cpu_ccfence();
+               if (refs == 0) {
+                       /*
+                        * 0 -> 1 transition must bump the refs on the parent
+                        * too.  The caller has stabilized the parent.
+                        */
+                       if (atomic_cmpset_int(&chain->refs, 0, 1)) {
+                               chain = chain->parent;
+                               KKASSERT(chain == NULL || chain->refs > 0);
+                       }
+                       /* retry or continue along the parent chain */
+               } else {
+                       /*
+                        * N -> N+1
+                        */
+                       if (atomic_cmpset_int(&chain->refs, refs, refs + 1))
+                               break;
+                       /* retry */
+               }
+       }
 }
 
 /*
  * Drop the callers reference to the chain element.  If the ref count
- * reaches zero the chain element and its related structure (typically an
- * inode or indirect block) will be freed and the parent will be
- * recursively dropped.
+ * reaches zero we attempt to recursively drop the parent.
  *
  * MOVED and MODIFIED elements hold additional references so it should not
  * be possible for the count on a modified element to drop to 0.
  *
- * The chain element must NOT be locked by the caller.
+ * The chain element must NOT be locked by the caller on the 1->0 transition.
  *
- * The parent might or might not be locked by the caller but if so it
- * will also be referenced so we shouldn't recurse upward.
+ * The parent might or might not be locked by the caller.  If we are unable
+ * to lock the parent on the 1->0 transition the destruction of the chain
+ * will be deferred but we still recurse upward and drop the ref on the
+ * parent (see the lastdrop() function)
  */
+static hammer2_chain_t *hammer2_chain_lastdrop(hammer2_mount_t *hmp,
+                                               hammer2_chain_t *chain);
+
 void
 hammer2_chain_drop(hammer2_mount_t *hmp, hammer2_chain_t *chain)
 {
-       hammer2_chain_t *parent;
-       hammer2_inode_t *ip;
        u_int refs;
 
        while (chain) {
@@ -213,62 +302,19 @@ hammer2_chain_drop(hammer2_mount_t *hmp, hammer2_chain_t *chain)
                cpu_ccfence();
                KKASSERT(refs > 0);
                if (refs == 1) {
-                       KKASSERT(chain != &hmp->vchain);
-                       parent = chain->parent;
-                       if (parent) {
-                               ccms_thread_lock(&parent->cst,
-                                               CCMS_STATE_EXCLUSIVE);
-                       }
-                       if (atomic_cmpset_int(&chain->refs, 1, 0)) {
-                               /*
-                                * Succeeded, recurse and drop parent.
-                                * These chain elements should be synchronized
-                                * so no delta data or inode count updates
-                                * should be needed.
-                                */
-                               KKASSERT((chain->flags &
-                                         (HAMMER2_CHAIN_MOVED |
-                                          HAMMER2_CHAIN_MODIFIED)) == 0);
-
-                               if (chain->bref.type == HAMMER2_BREF_TYPE_INODE)
-                                       ip = chain->u.ip;
-                               else
-                                       ip = NULL;
-
-                               /*
-                                * Delete interlock
-                                */
-                               if (!(chain->flags & HAMMER2_CHAIN_DELETED)) {
-                                       /*
-                                        * Disconnect the chain and clear
-                                        * pip if it was an inode.
-                                        */
-                                       RB_REMOVE(hammer2_chain_tree,
-                                                 &parent->rbhead, chain);
-                                       atomic_set_int(&chain->flags,
-                                                      HAMMER2_CHAIN_DELETED);
-                                       if (ip)
-                                               ip->pip = NULL;
-                                       /* parent refs dropped via recursion */
-                               }
-
-                               /*
-                                * When cleaning out a hammer2_inode we must
-                                * also clean out the related ccms_inode.
-                                */
-                               if (ip)
-                                       ccms_cst_uninit(&ip->topo_cst);
-                               chain->parent = NULL;
-                               if (parent)
-                                       ccms_thread_unlock(&parent->cst);
-                               hammer2_chain_free(hmp, chain);
-                               chain = parent;
-                               /* recurse on parent */
-                       } else {
-                               if (parent)
-                                       ccms_thread_unlock(&parent->cst);
-                               /* retry the same chain */
-                       }
+                       /*
+                        * (1) lastdrop successfully drops the chain and
+                        *     returns the parent, we recursively drop the
+                        *     parent.
+                        *
+                        * (2) lastdrop fails to transition refs from 1 to 0
+                        *     and returns the same chain, we retry.
+                        *
+                        * (3) lastdrop fails to drop the chain and returns
+                        *     NULL, leaving the ref intact for a deferred
+                        *     drop later on.
+                        */
+                       chain = hammer2_chain_lastdrop(hmp, chain);
                } else {
                        if (atomic_cmpset_int(&chain->refs, refs, refs - 1)) {
                                /*
@@ -282,6 +328,73 @@ hammer2_chain_drop(hammer2_mount_t *hmp, hammer2_chain_t *chain)
        }
 }
 
+/*
+ * On the last drop we have to stabilize chain->parent, which we can do
+ * by acquiring the chain->cst.spin lock.  If we get a full-blown lock
+ * it messes up the chain_unlock() code's ccms_thread_unlock_zero() call.
+ *
+ * Once the spinlock has been obtained we can drop the refs and become the
+ * owner of the implied ref on the parent, allowing us to return the parent.
+ */
+static
+hammer2_chain_t *
+hammer2_chain_lastdrop(hammer2_mount_t *hmp, hammer2_chain_t *chain)
+{
+       hammer2_chain_t *parent;
+
+       /*
+        * gain lock, drop refs, return chain to retry if we were unable
+        * to drop the refs from 1 to 0.
+        */
+       spin_lock(&chain->cst.spin);
+       if (atomic_cmpset_int(&chain->refs, 1, 0) == 0) {
+               spin_unlock(&chain->cst.spin);
+               return (chain);
+       }
+
+       /*
+        * Refs is 0 and we own the implied ref on the parent.  The
+        * chain can still be accessed at this point but any cycling
+        * of its refs will simply build-up more implied refs on the
+        * parent.
+        *
+        * Thus the parent pointer is valid.
+        */
+       parent = chain->parent;
+       spin_unlock(&chain->cst.spin);
+
+       /*
+        * Attempt to acquire an exclusive lock on the parent.  If this
+        * fails we just leave chain alone but still return the parent
+        * for the drop recursion.
+        */
+       if (parent &&
+           ccms_thread_lock_nonblock(&parent->cst, CCMS_STATE_EXCLUSIVE)) {
+               return (parent);
+       }
+
+       /*
+        * With an exclusive lock on the parent in-hand if chain->refs is
+        * still 0 then its impossible for anyone new to access it (or any
+        * of its children), and it can be deallocated.
+        */
+       if (chain->refs == 0) {
+               ccms_thread_lock(&chain->cst, CCMS_STATE_EXCLUSIVE);
+               hammer2_chain_dealloc(hmp, chain);
+       }
+
+       /*
+        * drop recursion, return parent so the caller can eat the implied
+        * ref we own on it.  We have to use hammer2_chain_unlock() (which
+        * also does a drop so we also need a ref on parent).
+        */
+       if (parent) {
+               hammer2_chain_ref(hmp, parent);
+               hammer2_chain_unlock(hmp, parent);
+       }
+       return (parent);
+}
+
 /*
  * Ref and lock a chain element, acquiring its data with I/O if necessary,
  * and specify how you would like the data to be resolved.
@@ -306,6 +419,8 @@ hammer2_chain_drop(hammer2_mount_t *hmp, hammer2_chain_t *chain)
  *
  * HAMMER2_RESOLVE_ALWAYS- Always resolve the data element.
  *
+ * HAMMER2_RESOLVE_SHARED- (flag) The chain is locked shared, otherwise
+ *                        it will be locked exclusive.
  *
  * NOTE: Embedded elements (volume header, inodes) are always resolved
  *      regardless.
@@ -325,18 +440,20 @@ hammer2_chain_lock(hammer2_mount_t *hmp, hammer2_chain_t *chain, int how)
        hammer2_blockref_t *bref;
        hammer2_off_t pbase;
        hammer2_off_t peof;
+       ccms_state_t ostate;
        size_t boff;
        size_t bbytes;
        int error;
        char *bdata;
 
        /*
-        * Lock the element.  Under certain conditions this might end up
-        * being a recursive lock.
+        * Ref and lock the element.  Recursive locks are allowed.
         */
-       KKASSERT(chain->refs > 0);
-       atomic_add_int(&chain->refs, 1);
-       ccms_thread_lock(&chain->cst, CCMS_STATE_EXCLUSIVE);
+       hammer2_chain_ref(hmp, chain);
+       if (how & HAMMER2_RESOLVE_SHARED)
+               ccms_thread_lock(&chain->cst, CCMS_STATE_SHARED);
+       else
+               ccms_thread_lock(&chain->cst, CCMS_STATE_EXCLUSIVE);
 
        /*
         * If we already have a valid data pointer no further action is
@@ -348,7 +465,7 @@ hammer2_chain_lock(hammer2_mount_t *hmp, hammer2_chain_t *chain, int how)
        /*
         * Do we have to resolve the data?
         */
-       switch(how) {
+       switch(how & HAMMER2_RESOLVE_MASK) {
        case HAMMER2_RESOLVE_NEVER:
                return(0);
        case HAMMER2_RESOLVE_MAYBE:
@@ -361,6 +478,17 @@ hammer2_chain_lock(hammer2_mount_t *hmp, hammer2_chain_t *chain, int how)
                break;
        }
 
+       /*
+        * Upgrade to an exclusive lock so we can safely manipulate the
+        * buffer cache.  If another thread got to it before us we
+        * can just return.
+        */
+       ostate = ccms_thread_lock_upgrade(&chain->cst);
+       if (chain->data) {
+               ccms_thread_lock_restore(&chain->cst, ostate);
+               return (0);
+       }
+
        /*
         * We must resolve to a device buffer, either by issuing I/O or
         * by creating a zero-fill element.  We do not mark the buffer
@@ -404,6 +532,7 @@ hammer2_chain_lock(hammer2_mount_t *hmp, hammer2_chain_t *chain, int how)
                        (intmax_t)pbase, error);
                bqrelse(chain->bp);
                chain->bp = NULL;
+               ccms_thread_lock_restore(&chain->cst, ostate);
                return (error);
        }
 
@@ -461,6 +590,15 @@ hammer2_chain_lock(hammer2_mount_t *hmp, hammer2_chain_t *chain, int how)
                chain->data = (void *)bdata;
                break;
        }
+
+       /*
+        * Make sure the bp is not specifically owned by this thread before
+        * restoring to a possibly shared lock, so another hammer2 thread
+        * can release it.
+        */
+       if (chain->bp)
+               BUF_KERNPROC(chain->bp);
+       ccms_thread_lock_restore(&chain->cst, ostate);
        return (0);
 }
 
@@ -943,6 +1081,7 @@ hammer2_chain_get(hammer2_mount_t *hmp, hammer2_chain_t *parent,
        hammer2_chain_t *chain;
        hammer2_chain_t dummy;
        int how;
+       ccms_state_t ostate;
 
        /*
         * Figure out how to lock.  MAYBE can be used to optimized
@@ -952,6 +1091,8 @@ hammer2_chain_get(hammer2_mount_t *hmp, hammer2_chain_t *parent,
                how = HAMMER2_RESOLVE_NEVER;
        else
                how = HAMMER2_RESOLVE_MAYBE;
+       if (flags & (HAMMER2_LOOKUP_SHARED | HAMMER2_LOOKUP_NOLOCK))
+               how |= HAMMER2_RESOLVE_SHARED;
 
        /*
         * First see if we have a (possibly modified) chain element cached
@@ -970,11 +1111,29 @@ hammer2_chain_get(hammer2_mount_t *hmp, hammer2_chain_t *parent,
                return(chain);
        }
 
+       /*
+        * Upgrade our thread lock and handle any race that may have
+        * occurred.  Leave the lock upgraded for the rest of the get.
+        * We have to do this because we will be modifying the chain
+        * structure.
+        */
+       ostate = ccms_thread_lock_upgrade(&parent->cst);
+       chain = RB_FIND(hammer2_chain_tree, &parent->rbhead, &dummy);
+       if (chain) {
+               if (flags & HAMMER2_LOOKUP_NOLOCK)
+                       hammer2_chain_ref(hmp, chain);
+               else
+                       hammer2_chain_lock(hmp, chain, how);
+               ccms_thread_lock_restore(&parent->cst, ostate);
+               return(chain);
+       }
+
        /*
         * The get function must always succeed, panic if there's no
         * data to index.
         */
        if (parent->flags & HAMMER2_CHAIN_INITIAL) {
+               ccms_thread_lock_restore(&parent->cst, ostate);
                panic("hammer2_chain_get: Missing bref(1)");
                /* NOT REACHED */
        }
@@ -1024,7 +1183,8 @@ hammer2_chain_get(hammer2_mount_t *hmp, hammer2_chain_t *parent,
        if (RB_INSERT(hammer2_chain_tree, &parent->rbhead, chain))
                panic("hammer2_chain_link: collision");
        KKASSERT(parent->refs > 0);
-       atomic_add_int(&parent->refs, 1);       /* for splay entry */
+       atomic_add_int(&parent->refs, 1);       /* for red-black entry */
+       ccms_thread_lock_restore(&parent->cst, ostate);
 
        /*
         * Additional linkage for inodes.  Reuse the parent pointer to
@@ -1101,6 +1261,13 @@ hammer2_chain_lookup(hammer2_mount_t *hmp, hammer2_chain_t **parentp,
        hammer2_key_t scan_end;
        int count = 0;
        int i;
+       int how_always = HAMMER2_RESOLVE_ALWAYS;
+       int how_maybe = HAMMER2_RESOLVE_MAYBE;
+
+       if (flags & (HAMMER2_LOOKUP_SHARED | HAMMER2_LOOKUP_NOLOCK)) {
+               how_maybe |= HAMMER2_RESOLVE_SHARED;
+               how_always |= HAMMER2_RESOLVE_SHARED;
+       }
 
        /*
         * Recurse (*parentp) upward if necessary until the parent completely
@@ -1117,7 +1284,7 @@ hammer2_chain_lookup(hammer2_mount_t *hmp, hammer2_chain_t **parentp,
                hammer2_chain_unlock(hmp, parent);      /* unlock old parent */
                parent = parent->parent;
                                                        /* lock new parent */
-               hammer2_chain_lock(hmp, parent, HAMMER2_RESOLVE_MAYBE);
+               hammer2_chain_lock(hmp, parent, how_maybe);
                hammer2_chain_drop(hmp, *parentp);      /* drop old parent */
                *parentp = parent;                      /* new parent */
        }
@@ -1140,8 +1307,7 @@ again:
                        if (flags & HAMMER2_LOOKUP_NOLOCK)
                                hammer2_chain_ref(hmp, parent);
                        else
-                               hammer2_chain_lock(hmp, parent,
-                                                  HAMMER2_RESOLVE_ALWAYS);
+                               hammer2_chain_lock(hmp, parent, how_always);
                        return (parent);
                }
                base = &parent->data->ipdata.u.blockset.blockref[0];
@@ -1217,10 +1383,10 @@ again:
                hammer2_chain_unlock(hmp, parent);
                *parentp = parent = chain;
                if (flags & HAMMER2_LOOKUP_NOLOCK) {
-                       hammer2_chain_lock(hmp, chain, HAMMER2_RESOLVE_MAYBE);
+                       hammer2_chain_lock(hmp, chain, how_maybe);
                        hammer2_chain_drop(hmp, chain); /* excess ref */
                } else if (flags & HAMMER2_LOOKUP_NODATA) {
-                       hammer2_chain_lock(hmp, chain, HAMMER2_RESOLVE_MAYBE);
+                       hammer2_chain_lock(hmp, chain, how_maybe);
                        hammer2_chain_unlock(hmp, chain);
                }
                goto again;
@@ -1256,8 +1422,12 @@ hammer2_chain_next(hammer2_mount_t *hmp, hammer2_chain_t **parentp,
        hammer2_key_t scan_beg;
        hammer2_key_t scan_end;
        int i;
+       int how_maybe = HAMMER2_RESOLVE_MAYBE;
        int count;
 
+       if (flags & (HAMMER2_LOOKUP_SHARED | HAMMER2_LOOKUP_NOLOCK))
+               how_maybe |= HAMMER2_RESOLVE_SHARED;
+
        parent = *parentp;
 
 again:
@@ -1307,7 +1477,7 @@ again:
                hammer2_chain_ref(hmp, nparent);        /* ref new parent */
                hammer2_chain_unlock(hmp, parent);      /* unlock old parent */
                                                        /* lock new parent */
-               hammer2_chain_lock(hmp, nparent, HAMMER2_RESOLVE_MAYBE);
+               hammer2_chain_lock(hmp, nparent, how_maybe);
                hammer2_chain_drop(hmp, nparent);       /* drop excess ref */
                *parentp = parent = nparent;
        }
@@ -1394,10 +1564,10 @@ again2:
                *parentp = parent = chain;
                chain = NULL;
                if (flags & HAMMER2_LOOKUP_NOLOCK) {
-                       hammer2_chain_lock(hmp, parent, HAMMER2_RESOLVE_MAYBE);
+                       hammer2_chain_lock(hmp, parent, how_maybe);
                        hammer2_chain_drop(hmp, parent);        /* excess ref */
                } else if (flags & HAMMER2_LOOKUP_NODATA) {
-                       hammer2_chain_lock(hmp, parent, HAMMER2_RESOLVE_MAYBE);
+                       hammer2_chain_lock(hmp, parent, how_maybe);
                        hammer2_chain_unlock(hmp, parent);
                }
                i = 0;
@@ -1435,6 +1605,8 @@ again2:
  *     DATA            no data area will be set-up (caller is expected
  *                     to have logical buffers, we don't want to alias
  *                     the data onto device buffers!).
+ *
+ * Requires an exclusively locked parent.
  */
 hammer2_chain_t *
 hammer2_chain_create(hammer2_mount_t *hmp, hammer2_chain_t *parent,
@@ -1449,6 +1621,8 @@ hammer2_chain_create(hammer2_mount_t *hmp, hammer2_chain_t *parent,
        int count;
        int i;
 
+       KKASSERT(ccms_thread_lock_owned(&parent->cst));
+
        if (chain == NULL) {
                /*
                 * First allocate media space and construct the dummy bref,
@@ -1718,6 +1892,8 @@ done:
  * This also has the risk of not moving enough elements to the new indirect
  * block and being forced to create several indirect blocks before the element
  * can be inserted.
+ *
+ * Must be called with an exclusively locked parent
  */
 static
 hammer2_chain_t *
@@ -1742,6 +1918,8 @@ hammer2_chain_create_indirect(hammer2_mount_t *hmp, hammer2_chain_t *parent,
         * is known to be empty.  We need to calculate the array count
         * for RB lookups either way.
         */
+       KKASSERT(ccms_thread_lock_owned(&parent->cst));
+
        hammer2_chain_modify(hmp, parent, HAMMER2_MODIFY_OPTDATA);
        if (parent->flags & HAMMER2_CHAIN_INITIAL) {
                base = NULL;
@@ -2057,6 +2235,8 @@ hammer2_chain_create_indirect(hammer2_mount_t *hmp, hammer2_chain_t *parent,
  * XXX This currently does not adhere to the MOVED flag protocol in that
  *     the removal is immediately indicated in the parent's blockref[]
  *     array.
+ *
+ * Must be called with an exclusively locked parent.
  */
 void
 hammer2_chain_delete(hammer2_mount_t *hmp, hammer2_chain_t *parent,
@@ -2068,6 +2248,7 @@ hammer2_chain_delete(hammer2_mount_t *hmp, hammer2_chain_t *parent,
 
        if (chain->parent != parent)
                panic("hammer2_chain_delete: parent mismatch");
+       KKASSERT(ccms_thread_lock_owned(&parent->cst));
 
        /*
         * Mark the parent modified so our base[] pointer remains valid
@@ -2113,7 +2294,7 @@ hammer2_chain_delete(hammer2_mount_t *hmp, hammer2_chain_t *parent,
 
        RB_REMOVE(hammer2_chain_tree, &parent->rbhead, chain);
        atomic_set_int(&chain->flags, HAMMER2_CHAIN_DELETED);
-       atomic_add_int(&parent->refs, -1);      /* for splay entry */
+       atomic_add_int(&parent->refs, -1);      /* for red-black entry */
        chain->index = -1;
        chain->parent = NULL;
 
@@ -2180,11 +2361,7 @@ hammer2_chain_delete(hammer2_mount_t *hmp, hammer2_chain_t *parent,
  * will remain referenced throughout but can temporarily lose its
  * lock during the recursion to avoid unnecessarily stalling user
  * processes.
- *
- *
  */
-TAILQ_HEAD(flush_deferral_list, hammer2_chain);
-
 struct hammer2_flush_info {
        struct flush_deferral_list flush_list;
        int             depth;
index 69e1e84..1b28298 100644 (file)
@@ -63,22 +63,19 @@ hammer2_inode_drop(hammer2_inode_t *ip)
 
 /*
  * Get the vnode associated with the given inode, allocating the vnode if
- * necessary.
+ * necessary.  The vnode will be returned exclusively locked.
+ *
+ * The caller must lock the inode (shared or exclusive).
  *
  * Great care must be taken to avoid deadlocks and vnode acquisition/reclaim
  * races.
- *
- * The vnode will be returned exclusively locked and referenced.  The
- * reference on the vnode prevents it from being reclaimed.
- *
- * The inode (ip) must be referenced by the caller and not locked to avoid
- * it getting ripped out from under us or deadlocked.
  */
 struct vnode *
 hammer2_igetv(hammer2_inode_t *ip, int *errorp)
 {
        struct vnode *vp;
        hammer2_pfsmount_t *pmp;
+       ccms_state_t ostate;
 
        pmp = ip->pmp;
        KKASSERT(pmp != NULL);
@@ -93,15 +90,6 @@ hammer2_igetv(hammer2_inode_t *ip, int *errorp)
                 */
                vp = ip->vp;
                if (vp) {
-                       /*
-                        * Lock the inode and check for a reclaim race
-                        */
-                       hammer2_inode_lock_ex(ip);
-                       if (ip->vp != vp) {
-                               hammer2_inode_unlock_ex(ip);
-                               continue;
-                       }
-
                        /*
                         * Inode must be unlocked during the vget() to avoid
                         * possible deadlocks, vnode is held to prevent
@@ -109,11 +97,14 @@ hammer2_igetv(hammer2_inode_t *ip, int *errorp)
                         * still fail if we lost a reclaim race on the vnode.
                         */
                        vhold_interlocked(vp);
-                       hammer2_inode_unlock_ex(ip);
+                       ccms_thread_unlock(&ip->chain.cst);
                        if (vget(vp, LK_EXCLUSIVE)) {
                                vdrop(vp);
+                               ccms_thread_lock(&ip->chain.cst,
+                                                CCMS_STATE_EXCLUSIVE);
                                continue;
                        }
+                       ccms_thread_lock(&ip->chain.cst, CCMS_STATE_EXCLUSIVE);
                        vdrop(vp);
                        /* vp still locked and ref from vget */
                        *errorp = 0;
@@ -134,11 +125,11 @@ hammer2_igetv(hammer2_inode_t *ip, int *errorp)
                /*
                 * Lock the inode and check for an allocation race.
                 */
-               hammer2_inode_lock_ex(ip);
+               ostate = ccms_thread_lock_upgrade(&ip->chain.cst);
                if (ip->vp != NULL) {
                        vp->v_type = VBAD;
                        vx_put(vp);
-                       hammer2_inode_unlock_ex(ip);
+                       ccms_thread_lock_restore(&ip->chain.cst, ostate);
                        continue;
                }
 
@@ -176,7 +167,7 @@ hammer2_igetv(hammer2_inode_t *ip, int *errorp)
                vp->v_data = ip;
                ip->vp = vp;
                hammer2_chain_ref(ip->hmp, &ip->chain); /* vp association */
-               hammer2_inode_unlock_ex(ip);
+               ccms_thread_lock_restore(&ip->chain.cst, ostate);
                break;
        }
 
@@ -410,11 +401,12 @@ hammer2_inode_duplicate(hammer2_inode_t *dip, hammer2_inode_t *oip,
         * We cannot leave oip with any in-memory chains because (for a
         * hardlink), oip will become a OBJTYPE_HARDLINK which is just a
         * pointer to the real hardlink's inum and can't have any sub-chains.
+        * XXX might be 0-ref chains left.
         */
        hammer2_inode_lock_ex(oip);
        hammer2_chain_flush(hmp, &oip->chain, 0);
        hammer2_inode_unlock_ex(oip);
-       KKASSERT(RB_EMPTY(&oip->chain.rbhead));
+       /*KKASSERT(RB_EMPTY(&oip->chain.rbhead));*/
 
        nip = chain->u.ip;
        hammer2_chain_modify(hmp, chain, 0);
index af88da3..b31d390 100644 (file)
@@ -77,15 +77,17 @@ void
 hammer2_inode_lock_sh(hammer2_inode_t *ip)
 {
        KKASSERT(ip->chain.refs > 0);
-       ccms_thread_lock(&ip->chain.cst, CCMS_STATE_SHARED);
+       hammer2_chain_lock(ip->hmp, &ip->chain, HAMMER2_RESOLVE_ALWAYS |
+                                               HAMMER2_RESOLVE_SHARED);
 }
 
 void
 hammer2_inode_unlock_sh(hammer2_inode_t *ip)
 {
-       ccms_thread_unlock(&ip->chain.cst);
+       hammer2_chain_unlock(ip->hmp, &ip->chain);
 }
 
+#if 0
 /*
  * Soft-busy an inode.
  *
@@ -96,7 +98,7 @@ void
 hammer2_inode_busy(hammer2_inode_t *ip)
 {
        if (ip->chain.busy++ == 0)
-               hammer2_chain_ref(ip->hmp, &ip->chain);
+               hammer2_chain_ref(ip->hmp, &ip->chain, 0);
 }
 
 void
@@ -106,6 +108,8 @@ hammer2_inode_unbusy(hammer2_inode_t *ip)
                hammer2_chain_drop(ip->hmp, &ip->chain);
 }
 
+#endif
+
 /*
  * Mount-wide locks
  */
index 3d6147f..8278642 100644 (file)
@@ -606,21 +606,27 @@ int
 hammer2_vfs_root(struct mount *mp, struct vnode **vpp)
 {
        hammer2_pfsmount_t *pmp;
+       hammer2_mount_t *hmp;
        int error;
        struct vnode *vp;
 
        pmp = MPTOPMP(mp);
-       hammer2_mount_exlock(pmp->hmp);
+       hmp = pmp->hmp;
+       hammer2_mount_exlock(hmp);
        if (pmp->iroot == NULL) {
                *vpp = NULL;
                error = EINVAL;
        } else {
+               hammer2_chain_lock(hmp, &pmp->iroot->chain,
+                                  HAMMER2_RESOLVE_ALWAYS |
+                                  HAMMER2_RESOLVE_SHARED);
                vp = hammer2_igetv(pmp->iroot, &error);
+               hammer2_chain_unlock(hmp, &pmp->iroot->chain);
                *vpp = vp;
                if (vp == NULL)
                        kprintf("vnodefail\n");
        }
-       hammer2_mount_unlock(pmp->hmp);
+       hammer2_mount_unlock(hmp);
 
        return (error);
 }
index d46be99..f51dc65 100644 (file)
@@ -472,15 +472,18 @@ hammer2_vop_readdir(struct vop_readdir_args *ap)
        lkey = saveoff | HAMMER2_DIRHASH_VISIBLE;
 
        parent = &ip->chain;
-       error = hammer2_chain_lock(hmp, parent, HAMMER2_RESOLVE_ALWAYS);
+       error = hammer2_chain_lock(hmp, parent, HAMMER2_RESOLVE_ALWAYS |
+                                               HAMMER2_RESOLVE_SHARED);
        if (error) {
                hammer2_chain_unlock(hmp, parent);
                goto done;
        }
-       chain = hammer2_chain_lookup(hmp, &parent, lkey, lkey, 0);
+       chain = hammer2_chain_lookup(hmp, &parent, lkey, lkey,
+                                    HAMMER2_LOOKUP_SHARED);
        if (chain == NULL) {
                chain = hammer2_chain_lookup(hmp, &parent,
-                                            lkey, (hammer2_key_t)-1, 0);
+                                            lkey, (hammer2_key_t)-1,
+                                            HAMMER2_LOOKUP_SHARED);
        }
        while (chain) {
                if (chain->bref.type == HAMMER2_BREF_TYPE_INODE) {
@@ -509,7 +512,8 @@ hammer2_vop_readdir(struct vop_readdir_args *ap)
                 */
                chain = hammer2_chain_next(hmp, &parent, chain,
                                           HAMMER2_DIRHASH_VISIBLE,
-                                          (hammer2_key_t)-1, 0);
+                                          (hammer2_key_t)-1,
+                                          HAMMER2_LOOKUP_SHARED);
                if (chain) {
                        saveoff = (chain->bref.key &
                                   HAMMER2_DIRHASH_USERMSK) + 1;
@@ -651,7 +655,6 @@ hammer2_vop_write(struct vop_write_args *ap)
         */
        hammer2_inode_lock_ex(ip);
        error = hammer2_write_file(ip, uio, ap->a_ioflag, seqcount);
-
        hammer2_inode_unlock_ex(ip);
        return (error);
 }
@@ -1327,10 +1330,11 @@ hammer2_vop_nresolve(struct vop_nresolve_args *ap)
         * Note: In DragonFly the kernel handles '.' and '..'.
         */
        parent = &dip->chain;
-       hammer2_chain_lock(hmp, parent, HAMMER2_RESOLVE_ALWAYS);
+       hammer2_chain_lock(hmp, parent, HAMMER2_RESOLVE_ALWAYS |
+                                       HAMMER2_RESOLVE_SHARED);
        chain = hammer2_chain_lookup(hmp, &parent,
                                     lhc, lhc + HAMMER2_DIRHASH_LOMASK,
-                                    0);
+                                    HAMMER2_LOOKUP_SHARED);
        while (chain) {
                if (chain->bref.type == HAMMER2_BREF_TYPE_INODE &&
                    chain->u.ip &&
@@ -1340,7 +1344,7 @@ hammer2_vop_nresolve(struct vop_nresolve_args *ap)
                }
                chain = hammer2_chain_next(hmp, &parent, chain,
                                           lhc, lhc + HAMMER2_DIRHASH_LOMASK,
-                                          0);
+                                          HAMMER2_LOOKUP_SHARED);
        }
        hammer2_chain_unlock(hmp, parent);
 
@@ -1351,6 +1355,8 @@ hammer2_vop_nresolve(struct vop_nresolve_args *ap)
         * to locate the real file via a hardlink.  ip will be referenced but
         * not locked in that situation.  chain is passed in locked and
         * returned locked.
+        *
+        * XXX what kind of chain lock?
         */
        ip = NULL;
        if (chain && chain->u.ip->ip_data.type == HAMMER2_OBJTYPE_HARDLINK) {
@@ -1368,6 +1374,8 @@ hammer2_vop_nresolve(struct vop_nresolve_args *ap)
        /*
         * Deconsolidate any hardlink whos nlinks == 1.  Ignore errors.
         * If an error occurs chain and ip are left alone.
+        *
+        * XXX upgrade shared lock?
         */
        if (ip && chain && chain->u.ip->ip_data.nlinks == 1 && !hmp->ronly) {
                kprintf("hammer2: need to unconsolidate hardlink for %s\n",
@@ -1507,10 +1515,12 @@ hammer2_vop_bmap(struct vop_bmap_args *ap)
        loff = ap->a_loffset & HAMMER2_OFF_MASK_LO;
 
        parent = &ip->chain;
-       hammer2_chain_lock(hmp, parent, HAMMER2_RESOLVE_ALWAYS);
+       hammer2_chain_lock(hmp, parent, HAMMER2_RESOLVE_ALWAYS |
+                                       HAMMER2_RESOLVE_SHARED);
        chain = hammer2_chain_lookup(hmp, &parent,
                                     lbeg, lend,
-                                    HAMMER2_LOOKUP_NODATA);
+                                    HAMMER2_LOOKUP_NODATA |
+                                    HAMMER2_LOOKUP_SHARED);
        if (chain == NULL) {
                *ap->a_doffsetp = ZFOFFSET;
                hammer2_chain_unlock(hmp, parent);
@@ -1526,7 +1536,8 @@ hammer2_vop_bmap(struct vop_bmap_args *ap)
                }
                chain = hammer2_chain_next(hmp, &parent, chain,
                                           lbeg, lend,
-                                          HAMMER2_LOOKUP_NODATA);
+                                          HAMMER2_LOOKUP_NODATA |
+                                          HAMMER2_LOOKUP_SHARED);
        }
        hammer2_chain_unlock(hmp, parent);
 
@@ -2024,10 +2035,12 @@ hammer2_strategy_read(struct vop_strategy_args *ap)
         */
        if (nbio->bio_offset == NOOFFSET) {
                parent = &ip->chain;
-               hammer2_chain_lock(hmp, parent, HAMMER2_RESOLVE_ALWAYS);
+               hammer2_chain_lock(hmp, parent, HAMMER2_RESOLVE_ALWAYS |
+                                               HAMMER2_RESOLVE_SHARED);
 
                chain = hammer2_chain_lookup(hmp, &parent, lbase, lbase,
-                                            HAMMER2_LOOKUP_NODATA);
+                                            HAMMER2_LOOKUP_NODATA |
+                                            HAMMER2_LOOKUP_SHARED);
                if (chain == NULL) {
                        /*
                         * Data is zero-fill