kernel - Implementat much deeper use of shared VM object locks
authorMatthew Dillon <dillon@apollo.backplane.com>
Sat, 23 Feb 2013 19:22:00 +0000 (11:22 -0800)
committerMatthew Dillon <dillon@apollo.backplane.com>
Sat, 23 Feb 2013 19:22:00 +0000 (11:22 -0800)
* Use a shared VM object lock on terminal (and likely highly shared)
  OBJT_VNODE objects.  For example, binaries in the system such as
  /bin/sh or /usr/bin/make.

  This greatly improves fork/exec and related VM faults on concurrently
  executing binaries.  Most commonly, parallel builds often exec
  hundreds of thousands of sh's and make's.

  +50% to +100% nominal improved performance under these conditions.
  +200% to +300% improved poudriere performance during the depend
  stage.

* Formalize the shared VM object lock with a new API function,
  vm_object_lock_maybe_shared(), which determines whether a VM
  object meets the requirements for obtaining a shared lock.

* Adjust the vm_fault*() APIs to track whether the VM object is
  locked shared or exclusive on entry.

* Clarify that OBJ_ONEMAPPING is only applicable to OBJT_DEFAULT
  and OBJT_SWAP objects.

* Heavy work on the exec path.  Somewhat lighter work on the exit
  path.  Tons more work could be done.

sys/kern/imgact_aout.c
sys/kern/imgact_elf.c
sys/kern/lwkt_token.c
sys/vm/swap_pager.c
sys/vm/vm_fault.c
sys/vm/vm_map.c
sys/vm/vm_map.h
sys/vm/vm_object.c
sys/vm/vm_object.h
sys/vm/vm_page.c
sys/vm/vnode_pager.c

index 67a0171..89ca8ff 100644 (file)
@@ -189,7 +189,7 @@ exec_aout_imgact(struct image_params *imgp)
                virtual_offset, text_end,
                VM_MAPTYPE_NORMAL,
                VM_PROT_READ | VM_PROT_EXECUTE, VM_PROT_ALL,
-               MAP_COPY_ON_WRITE | MAP_PREFAULT);
+               MAP_COPY_ON_WRITE | MAP_PREFAULT | MAP_PREFAULT_RELOCK);
 
        if (error) {
                vm_object_drop(object);
@@ -197,6 +197,7 @@ exec_aout_imgact(struct image_params *imgp)
                vm_map_entry_release(count);
                return (error);
        }
+
        data_end = text_end + a_out->a_data;
        if (a_out->a_data) {
                vm_object_reference_locked(object);
@@ -205,7 +206,7 @@ exec_aout_imgact(struct image_params *imgp)
                        text_end, data_end,
                        VM_MAPTYPE_NORMAL,
                        VM_PROT_ALL, VM_PROT_ALL,
-                       MAP_COPY_ON_WRITE | MAP_PREFAULT);
+                       MAP_COPY_ON_WRITE | MAP_PREFAULT | MAP_PREFAULT_RELOCK);
                if (error) {
                        vm_object_drop(object);
                        vm_map_unlock(map);
index 4a28b49..3f79537 100644 (file)
@@ -246,6 +246,7 @@ __elfN(load_section)(struct proc *p, struct vmspace *vmspace, struct vnode *vp,
        vm_offset_t map_addr;
        int error, rv, cow;
        int count;
+       int shared;
        size_t copy_len;
        vm_object_t object;
        vm_offset_t file_addr;
@@ -253,7 +254,12 @@ __elfN(load_section)(struct proc *p, struct vmspace *vmspace, struct vnode *vp,
        object = vp->v_object;
        error = 0;
 
-       vm_object_hold(object);
+       /*
+        * In most cases we will be able to use a shared lock on the
+        * object we are inserting into the map.  The lock will be
+        * upgraded in situations where new VM pages must be allocated.
+        */
+       shared = vm_object_hold_maybe_shared(object);
 
        /*
         * It's necessary to fail if the filsz + offset taken from the
@@ -288,8 +294,11 @@ __elfN(load_section)(struct proc *p, struct vmspace *vmspace, struct vnode *vp,
                vm_object_reference_locked(object);
 
                /* cow flags: don't dump readonly sections in core */
-               cow = MAP_COPY_ON_WRITE | MAP_PREFAULT |
-                   (prot & VM_PROT_WRITE ? 0 : MAP_DISABLE_COREDUMP);
+               cow = MAP_COPY_ON_WRITE | MAP_PREFAULT;
+               if ((prot & VM_PROT_WRITE) == 0)
+                       cow |= MAP_DISABLE_COREDUMP;
+               if (shared == 0)
+                       cow |= MAP_PREFAULT_RELOCK;
 
                count = vm_map_entry_reserve(MAP_RESERVE_COUNT);
                vm_map_lock(&vmspace->vm_map);
@@ -303,9 +312,14 @@ __elfN(load_section)(struct proc *p, struct vmspace *vmspace, struct vnode *vp,
                                      cow);
                vm_map_unlock(&vmspace->vm_map);
                vm_map_entry_release(count);
+
+               /*
+                * NOTE: Object must have a hold ref when calling
+                * vm_object_deallocate().
+                */
                if (rv != KERN_SUCCESS) {
-                       vm_object_deallocate(object);
                        vm_object_drop(object);
+                       vm_object_deallocate(object);
                        return (EINVAL);
                }
 
@@ -316,7 +330,6 @@ __elfN(load_section)(struct proc *p, struct vmspace *vmspace, struct vnode *vp,
                }
        }
 
-
        /*
         * We have to get the remaining bit of the file into the first part
         * of the oversized map segment.  This is normally because the .data
@@ -346,12 +359,12 @@ __elfN(load_section)(struct proc *p, struct vmspace *vmspace, struct vnode *vp,
        }
 
        if (copy_len != 0) {
-               vm_page_t m;
                struct lwbuf *lwb;
                struct lwbuf lwb_cache;
+               vm_page_t m;
 
                m = vm_fault_object_page(object, trunc_page(offset + filsz),
-                                        VM_PROT_READ, 0, &error);
+                                        VM_PROT_READ, 0, shared, &error);
                if (m) {
                        lwb = lwbuf_alloc(m, &lwb_cache);
                        error = copyout((caddr_t)lwbuf_kva(lwb),
@@ -366,6 +379,7 @@ __elfN(load_section)(struct proc *p, struct vmspace *vmspace, struct vnode *vp,
        }
 
        vm_object_drop(object);
+
        /*
         * set it to the specified protection
         */
index 12bc374..059bc18 100644 (file)
@@ -82,6 +82,7 @@ extern int lwkt_sched_debug;
 #endif
 
 static lwkt_token      pool_tokens[LWKT_NUM_POOL_TOKENS];
+struct spinlock                tok_debug_spin = SPINLOCK_INITIALIZER(&tok_debug_spin);
 
 #define TOKEN_STRING   "REF=%p TOK=%p TD=%p"
 #define TOKEN_ARGS     lwkt_tokref_t ref, lwkt_token_t tok, struct thread *td
@@ -163,6 +164,11 @@ SYSCTL_LONG(_lwkt, OID_AUTO, tty_collisions, CTLFLAG_RW,
 SYSCTL_LONG(_lwkt, OID_AUTO, vnode_collisions, CTLFLAG_RW,
     &vnode_token.t_collisions, 0, "Collision counter of vnode_token");
 
+int tokens_debug_output;
+SYSCTL_INT(_lwkt, OID_AUTO, tokens_debug_output, CTLFLAG_RW,
+    &tokens_debug_output, 0, "Generate stack trace N times");
+
+
 #ifdef DEBUG_LOCKS_LATENCY
 
 static long tokens_add_latency;
@@ -653,6 +659,17 @@ good:
        ++tok->t_collisions;
        logtoken(fail, ref);
        td->td_toks_have = td->td_toks_stop - 1;
+
+       if (tokens_debug_output > 0) {
+               --tokens_debug_output;
+               spin_lock(&tok_debug_spin);
+               kprintf("Excl Token thread %p %s %s\n",
+                       td, tok->t_desc, td->td_comm);
+               print_backtrace(6);
+               kprintf("\n");
+               spin_unlock(&tok_debug_spin);
+       }
+
        lwkt_switch();
        logtoken(succ, ref);
        KKASSERT(tok->t_ref == ref);
@@ -705,6 +722,17 @@ lwkt_gettoken_shared(lwkt_token_t tok)
        ++tok->t_collisions;
        logtoken(fail, ref);
        td->td_toks_have = td->td_toks_stop - 1;
+
+       if (tokens_debug_output > 0) {
+               --tokens_debug_output;
+               spin_lock(&tok_debug_spin);
+               kprintf("Shar Token thread %p %s %s\n",
+                       td, tok->t_desc, td->td_comm);
+               print_backtrace(6);
+               kprintf("\n");
+               spin_unlock(&tok_debug_spin);
+       }
+
        lwkt_switch();
        logtoken(succ, ref);
 }
index 3d7e9b4..dc08225 100644 (file)
@@ -1947,7 +1947,7 @@ swp_pager_fault_page(vm_object_t object, vm_pindex_t pindex)
                m = vm_fault_object_page(object, IDX_TO_OFF(pindex),
                                         VM_PROT_NONE,
                                         VM_FAULT_DIRTY | VM_FAULT_UNSWAP,
-                                        &error);
+                                        0, &error);
                if (m)
                        vm_page_unhold(m);
        }
index 6fcc5de..3adce4b 100644 (file)
@@ -118,13 +118,14 @@ struct faultstate {
        int hardfault;
        int fault_flags;
        int map_generation;
+       int shared;
        boolean_t wired;
        struct vnode *vp;
 };
 
 static int debug_cluster = 0;
 SYSCTL_INT(_vm, OID_AUTO, debug_cluster, CTLFLAG_RW, &debug_cluster, 0, "");
-static int vm_shared_fault = 1;
+int vm_shared_fault = 1;
 SYSCTL_INT(_vm, OID_AUTO, shared_fault, CTLFLAG_RW, &vm_shared_fault, 0,
           "Allow shared token on vm_object");
 static long vm_shared_hit = 0;
@@ -132,7 +133,7 @@ SYSCTL_LONG(_vm, OID_AUTO, shared_hit, CTLFLAG_RW, &vm_shared_hit, 0,
           "Successful shared faults");
 static long vm_shared_miss = 0;
 SYSCTL_LONG(_vm, OID_AUTO, shared_miss, CTLFLAG_RW, &vm_shared_miss, 0,
-          "Successful shared faults");
+          "Unsuccessful shared faults");
 
 static int vm_fault_object(struct faultstate *, vm_pindex_t, vm_prot_t);
 static int vm_fault_vpagetable(struct faultstate *, vm_pindex_t *, vpte_t, int);
@@ -453,6 +454,7 @@ RetryFault:
        fs.lookup_still_valid = TRUE;
        fs.first_m = NULL;
        fs.object = fs.first_object;    /* so unlock_and_deallocate works */
+       fs.shared = 0;
 
        /*
         * If the entry is wired we cannot change the page protection.
@@ -492,9 +494,13 @@ RetryFault:
         * page can be safely written.  However, it will force a read-only
         * mapping for a read fault if the memory is managed by a virtual
         * page table.
+        *
+        * If the fault code uses the shared object lock shortcut
+        * we must not try to burst (we can't allocate VM pages).
         */
-       /* BEFORE */
        result = vm_fault_object(&fs, first_pindex, fault_type);
+       if (fs.shared)
+               fault_flags &= ~VM_FAULT_BURST;
 
        if (result == KERN_TRY_AGAIN) {
                vm_object_drop(fs.first_object);
@@ -694,6 +700,7 @@ RetryFault:
        fs.lookup_still_valid = TRUE;
        fs.first_m = NULL;
        fs.object = fs.first_object;    /* so unlock_and_deallocate works */
+       fs.shared = 0;
 
        /*
         * If the entry is wired we cannot change the page protection.
@@ -829,7 +836,8 @@ done:
  */
 vm_page_t
 vm_fault_object_page(vm_object_t object, vm_ooffset_t offset,
-                    vm_prot_t fault_type, int fault_flags, int *errorp)
+                    vm_prot_t fault_type, int fault_flags,
+                    int shared, int *errorp)
 {
        int result;
        vm_pindex_t first_pindex;
@@ -854,6 +862,7 @@ RetryFault:
        fs.entry = &entry;
        fs.first_prot = fault_type;
        fs.wired = 0;
+       fs.shared = shared;
        /*fs.map_generation = 0; unused */
 
        /*
@@ -1125,6 +1134,8 @@ vm_fault_object(struct faultstate *fs,
                 * inclusive is chainlocked.
                 *
                 * If the object is dead, we stop here
+                *
+                * vm_shared_fault (fs->shared != 0) case: nothing special.
                 */
                if (fs->object->flags & OBJ_DEAD) {
                        vm_object_pip_wakeup(fs->first_object);
@@ -1148,6 +1159,11 @@ vm_fault_object(struct faultstate *fs,
                 * worth.  We cannot under any circumstances mess
                 * around with a vm_page_t->busy page except, perhaps,
                 * to pmap it.
+                *
+                * vm_shared_fault (fs->shared != 0) case:
+                *      error           nothing special
+                *      fs->m           relock excl if I/O needed
+                *      NULL            relock excl
                 */
                fs->m = vm_page_lookup_busy_try(fs->object, pindex,
                                                TRUE, &error);
@@ -1202,18 +1218,34 @@ vm_fault_object(struct faultstate *fs,
                        if (fs->m->object != &kernel_object) {
                                if ((fs->m->valid & VM_PAGE_BITS_ALL) !=
                                    VM_PAGE_BITS_ALL) {
+                                       if (fs->shared) {
+                                               vm_object_drop(fs->object);
+                                               vm_object_hold(fs->object);
+                                               fs->shared = 0;
+                                       }
                                        goto readrest;
                                }
                                if (fs->m->flags & PG_RAM) {
                                        if (debug_cluster)
                                                kprintf("R");
                                        vm_page_flag_clear(fs->m, PG_RAM);
+                                       if (fs->shared) {
+                                               vm_object_drop(fs->object);
+                                               vm_object_hold(fs->object);
+                                               fs->shared = 0;
+                                       }
                                        goto readrest;
                                }
                        }
                        break; /* break to PAGE HAS BEEN FOUND */
                }
 
+               if (fs->shared) {
+                       vm_object_drop(fs->object);
+                       vm_object_hold(fs->object);
+                       fs->shared = 0;
+               }
+
                /*
                 * Page is not resident, If this is the search termination
                 * or the pager might contain the page, allocate a new page.
@@ -1494,9 +1526,24 @@ skip:
                /*
                 * Move on to the next object.  The chain lock should prevent
                 * the backing_object from getting ripped out from under us.
+                *
+                * vm_shared_fault case:
+                *
+                *      If the next object is the last object and
+                *      vnode-backed (thus possibly shared), we can try a
+                *      shared object lock.  There is no 'chain' for this
+                *      last object if vnode-backed (otherwise we would
+                *      need an exclusive lock).
+                *
+                *      fs->shared mode is very fragile and only works
+                *      under certain specific conditions, and is only
+                *      handled for those conditions in our loop.  Essentially
+                *      it is designed only to be able to 'dip into' the
+                *      vnode's object and extract an already-cached page.
                 */
+               fs->shared = 0;
                if ((next_object = fs->object->backing_object) != NULL) {
-                       vm_object_hold(next_object);
+                       fs->shared = vm_object_hold_maybe_shared(next_object);
                        vm_object_chain_acquire(next_object);
                        KKASSERT(next_object == fs->object->backing_object);
                        pindex += OFF_TO_IDX(fs->object->backing_object_offset);
index 6cfc488..a0b5309 100644 (file)
@@ -158,6 +158,9 @@ static struct vm_map map_init[MAX_KMAP];
 static int randomize_mmap;
 SYSCTL_INT(_vm, OID_AUTO, randomize_mmap, CTLFLAG_RW, &randomize_mmap, 0,
     "Randomize mmap offsets");
+static int vm_map_relock_enable = 1;
+SYSCTL_INT(_vm, OID_AUTO, map_relock_enable, CTLFLAG_RW,
+          &vm_map_relock_enable, 0, "Randomize mmap offsets");
 
 static void vm_map_entry_shadow(vm_map_entry_t entry, int addref);
 static vm_map_entry_t vm_map_entry_create(vm_map_t map, int *);
@@ -987,6 +990,11 @@ vm_map_insert(vm_map_t map, int *countp,
                 * When object is non-NULL, it could be shared with another
                 * process.  We have to set or clear OBJ_ONEMAPPING 
                 * appropriately.
+                *
+                * NOTE: This flag is only applicable to DEFAULT and SWAP
+                *       objects and will already be clear in other types
+                *       of objects, so a shared object lock is ok for
+                *       VNODE objects.
                 */
                if ((object->ref_count > 1) || (object->shadow_count != 0)) {
                        vm_object_clear_flag(object, OBJ_ONEMAPPING);
@@ -1095,9 +1103,19 @@ vm_map_insert(vm_map_t map, int *countp,
         */
        if ((cow & (MAP_PREFAULT|MAP_PREFAULT_PARTIAL)) &&
            maptype != VM_MAPTYPE_VPAGETABLE) {
+               int dorelock = 0;
+               if (vm_map_relock_enable && (cow & MAP_PREFAULT_RELOCK)) {
+                       dorelock = 1;
+                       vm_object_lock_swap();
+                       vm_object_drop(object);
+               }
                pmap_object_init_pt(map->pmap, start, prot,
                                    object, OFF_TO_IDX(offset), end - start,
                                    cow & MAP_PREFAULT_PARTIAL);
+               if (dorelock) {
+                       vm_object_hold(object);
+                       vm_object_lock_swap();
+               }
        }
        if (must_drop)
                vm_object_drop(object);
@@ -2803,9 +2821,11 @@ again:
                } else if (object && object->type != OBJT_DEFAULT &&
                           object->type != OBJT_SWAP) {
                        /*
-                        * vnode object routines cannot be chain-locked
+                        * vnode object routines cannot be chain-locked,
+                        * but since we aren't removing pages from the
+                        * object here we can use a shared hold.
                         */
-                       vm_object_hold(object);
+                       vm_object_hold_shared(object);
                        pmap_remove(map->pmap, s, e);
                        vm_object_drop(object);
                } else if (object) {
index 6898e68..931ffbd 100644 (file)
@@ -505,6 +505,7 @@ vmspace_president_count(struct vmspace *vmspace)
 #define MAP_IS_KSTACK          0x0080
 #define MAP_DISABLE_COREDUMP   0x0100
 #define MAP_PREFAULT_MADVISE   0x0200  /* from (user) madvise request */
+#define MAP_PREFAULT_RELOCK    0x0200
 
 /*
  * vm_fault option flags
index 06ecc10..50ba558 100644 (file)
@@ -1,6 +1,4 @@
 /*
- * (MPSAFE)
- *
  * Copyright (c) 1991, 1993
  *     The Regents of the University of California.  All rights reserved.
  *
@@ -339,6 +337,28 @@ debugvm_object_hold_shared(vm_object_t obj, char *file, int line)
 }
 
 /*
+ * Obtain either a shared or exclusive lock on VM object
+ * based on whether this is a terminal vnode object or not.
+ */
+int
+#ifndef DEBUG_LOCKS
+vm_object_hold_maybe_shared(vm_object_t obj)
+#else
+debugvm_object_hold_maybe_shared(vm_object_t obj, char *file, int line)
+#endif
+{
+       if (vm_shared_fault &&
+           obj->type == OBJT_VNODE &&
+           obj->backing_object == NULL) {
+               vm_object_hold_shared(obj);
+               return(1);
+       } else {
+               vm_object_hold(obj);
+               return(0);
+       }
+}
+
+/*
  * Drop the token and hold_count on the object.
  */
 void
@@ -504,7 +524,8 @@ vm_object_allocate_hold(objtype_t type, vm_pindex_t size)
  * will call vm_object_chain_wait() prior to calling
  * vm_object_reference_locked() to avoid the case.
  *
- * The object must be held.
+ * The object must be held, but may be held shared if desired (hence why
+ * we use an atomic op).
  */
 void
 vm_object_reference_locked(vm_object_t object)
@@ -512,7 +533,7 @@ vm_object_reference_locked(vm_object_t object)
        KKASSERT(object != NULL);
        ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
        KKASSERT((object->flags & OBJ_CHAINLOCK) == 0);
-       object->ref_count++;
+       atomic_add_int(&object->ref_count, 1);
        if (object->type == OBJT_VNODE) {
                vref(object->handle);
                /* XXX what if the vnode is being destroyed? */
@@ -600,7 +621,8 @@ vm_object_chain_release_all(vm_object_t first_object, vm_object_t stopobj)
 /*
  * Dereference an object and its underlying vnode.
  *
- * The object must be held and will be held on return.
+ * The object must be held exclusively and will remain held on return.
+ * (We don't need an atomic op due to the exclusivity).
  */
 static void
 vm_object_vndeallocate(vm_object_t object)
@@ -631,6 +653,8 @@ vm_object_vndeallocate(vm_object_t object)
  * The caller does not have to hold the object locked but must have control
  * over the reference in question in order to guarantee that the object
  * does not get ripped out from under us.
+ *
+ * XXX Currently all deallocations require an exclusive lock.
  */
 void
 vm_object_deallocate(vm_object_t object)
@@ -657,6 +681,7 @@ vm_object_deallocate_locked(vm_object_t object)
         */
 again:
        while (object != NULL) {
+               ASSERT_LWKT_TOKEN_HELD_EXCL(&object->token);
 #if 0
                /*
                 * Don't rip a ref_count out from under an object undergoing
@@ -684,9 +709,14 @@ again:
                 *
                 * Nominal ref_count > 1 case if the second ref is not from
                 * a shadow.
+                *
+                * (ONEMAPPING only applies to DEFAULT AND SWAP objects)
                 */
                if (object->ref_count == 2 && object->shadow_count == 0) {
-                       vm_object_set_flag(object, OBJ_ONEMAPPING);
+                       if (object->type == OBJT_DEFAULT ||
+                           object->type == OBJT_SWAP) {
+                               vm_object_set_flag(object, OBJ_ONEMAPPING);
+                       }
                        object->ref_count--;
                        break;
                }
index 94d08da..93838b8 100644 (file)
@@ -196,6 +196,10 @@ struct vm_object {
 
 /*
  * Flags
+ *
+ * NOTE: OBJ_ONEMAPPING only applies to DEFAULT and SWAP objects.  It
+ *      may be gratuitously re-cleared in other cases but will already be
+ *      clear in those cases.
  */
 #define OBJ_CHAINLOCK  0x0001          /* backing_object/shadow changing */
 #define OBJ_CHAINWANT  0x0002
@@ -234,6 +238,7 @@ extern struct object_q vm_object_list;      /* list of allocated objects */
  /* lock for object list and count */
 
 extern struct vm_object kernel_object; /* the single kernel object */
+extern int vm_shared_fault;
 
 #endif                         /* _KERNEL */
 
@@ -309,7 +314,7 @@ void vm_object_shadow (vm_object_t *, vm_ooffset_t *, vm_size_t, int);
 void vm_object_madvise (vm_object_t, vm_pindex_t, int, int);
 void vm_object_init2 (void);
 vm_page_t vm_fault_object_page(vm_object_t, vm_ooffset_t,
-                               vm_prot_t, int, int *);
+                               vm_prot_t, int, int, int *);
 void vm_object_dead_sleep(vm_object_t, const char *);
 void vm_object_dead_wakeup(vm_object_t);
 void vm_object_lock_swap(void);
@@ -319,9 +324,12 @@ void vm_object_unlock(vm_object_t);
 
 #ifndef DEBUG_LOCKS
 void vm_object_hold(vm_object_t);
+int vm_object_hold_maybe_shared(vm_object_t);
 int vm_object_hold_try(vm_object_t);
 void vm_object_hold_shared(vm_object_t);
 #else
+#define vm_object_hold_maybe_shared(obj)               \
+       debugvm_object_hold_maybe_shared(obj, __FILE__, __LINE__)
 #define vm_object_hold(obj)            \
        debugvm_object_hold(obj, __FILE__, __LINE__)
 void debugvm_object_hold(vm_object_t, char *, int);
index ad6f177..b2a4e78 100644 (file)
@@ -913,7 +913,7 @@ vm_page_unhold(vm_page_t m)
 int
 vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
 {
-       ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
+       ASSERT_LWKT_TOKEN_HELD_EXCL(vm_object_token(object));
        if (m->object != NULL)
                panic("vm_page_insert: already inserted");
 
@@ -1146,9 +1146,9 @@ void
 vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex)
 {
        KKASSERT(m->flags & PG_BUSY);
-       ASSERT_LWKT_TOKEN_HELD(vm_object_token(new_object));
+       ASSERT_LWKT_TOKEN_HELD_EXCL(vm_object_token(new_object));
        if (m->object) {
-               ASSERT_LWKT_TOKEN_HELD(vm_object_token(m->object));
+               ASSERT_LWKT_TOKEN_HELD_EXCL(vm_object_token(m->object));
                vm_page_remove(m);
        }
        if (vm_page_insert(m, new_object, new_pindex) == FALSE) {
index 84a4498..b030a69 100644 (file)
@@ -865,7 +865,7 @@ vnode_pager_lock(vm_object_t object)
                tobject = lobject->backing_object;
                if (tobject == NULL)
                        break;
-               vm_object_hold(tobject);
+               vm_object_hold_shared(tobject);
                if (tobject == lobject->backing_object) {
                        if (lobject != object) {
                                vm_object_lock_swap();