kernel -- vm_object locking Part 1: Primitives

   * Create per-vm_object 'hold count'; the hold count ensures that a
     vm_object is not deallocated or collapsed till it transitions 1->0.
     Hold count is waited for in vm_object_terminate and _collapse.

   * Convert vm_object_lock/_unlock to use pool tokens; pool tokens do not
     share storage with the vm_object, simplifying lifetime.

Cleanups:

   * Remove first attempts at vm_object locking; it just wasn't clear
     what each token was protecting earlier.

   * Eliminate vestigial fields from vm_object structure.

   * Clean vm_object_pip_wakeup (just call _wakeupn)

   * Remove vm_page_(un)lock for now

Discussed-with: dillon@

diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c
index 4bf0a14..f8726e1 100644 (file)
--- a/sys/vm/vm_kern.c
+++ b/sys/vm/vm_kern.c
@@ -216,7 +216,6 @@ kmem_alloc3(vm_map_t map, vm_size_t size, int kmflags)
         * race with page-out.  vm_map_wire will wire the pages.
         */
        lwkt_gettoken(&vm_token);
-       vm_object_lock(&kernel_object);
        for (i = gstart; i < size; i += PAGE_SIZE) {
                vm_page_t mem;
 
@@ -228,7 +227,6 @@ kmem_alloc3(vm_map_t map, vm_size_t size, int kmflags)
                vm_page_flag_clear(mem, PG_ZERO);
                vm_page_wakeup(mem);
        }
-       vm_object_unlock(&kernel_object);
        lwkt_reltoken(&vm_token);
 
        /*

diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c
index 7cf9343..966213d 100644 (file)
--- a/sys/vm/vm_map.c
+++ b/sys/vm/vm_map.c
@@ -933,11 +933,9 @@ vm_map_insert(vm_map_t map, int *countp,
                 * appropriately.
                 */
 
-               vm_object_lock(object);
                if ((object->ref_count > 1) || (object->shadow_count != 0)) {
                        vm_object_clear_flag(object, OBJ_ONEMAPPING);
                }
-               vm_object_unlock(object);
        }
        else if ((prev_entry != &map->header) &&
                 (prev_entry->eflags == protoeflags) &&

diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index 9eeea98..e9d4503 100644 (file)
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -64,7 +64,6 @@
  * rights to redistribute these changes.
  *
  * $FreeBSD: src/sys/vm/vm_object.c,v 1.171.2.8 2003/05/26 19:17:56 alc Exp $
- * $DragonFly: src/sys/vm/vm_object.c,v 1.33 2008/05/09 07:24:48 dillon Exp $
  */
 
 /*
@@ -81,6 +80,7 @@
 #include <sys/mount.h>
 #include <sys/kernel.h>
 #include <sys/sysctl.h>
+#include <sys/refcount.h>
 
 #include <vm/vm.h>
 #include <vm/vm_param.h>
@@ -100,6 +100,10 @@
 static void    vm_object_qcollapse(vm_object_t object);
 static int     vm_object_page_collect_flush(vm_object_t object, vm_page_t p,
                                             int pagerflags);
+static void    vm_object_lock_init(vm_object_t);
+static void    vm_object_hold_wake(vm_object_t);
+static void    vm_object_hold_wait(vm_object_t);
+
 
 /*
  *     Virtual memory objects maintain the actual data
@@ -138,7 +142,6 @@ static long object_bypasses;
 static int next_index;
 static vm_zone_t obj_zone;
 static struct vm_zone obj_zone_store;
-static int object_hash_rand;
 #define VM_OBJECTS_INIT 256
 static struct vm_object vm_objects_init[VM_OBJECTS_INIT];
 
@@ -160,6 +163,7 @@ _vm_object_allocate(objtype_t type, vm_pindex_t size, vm_object_t object)
        object->type = type;
        object->size = size;
        object->ref_count = 1;
+       object->hold_count = 0;
        object->flags = 0;
        if ((object->type == OBJT_DEFAULT) || (object->type == OBJT_SWAP))
                vm_object_set_flag(object, OBJ_ONEMAPPING);
@@ -176,23 +180,15 @@ _vm_object_allocate(objtype_t type, vm_pindex_t size, vm_object_t object)
        object->handle = NULL;
        object->backing_object = NULL;
        object->backing_object_offset = (vm_ooffset_t) 0;
-       /*
-        * Try to generate a number that will spread objects out in the
-        * hash table.  We 'wipe' new objects across the hash in 128 page
-        * increments plus 1 more to offset it a little more by the time
-        * it wraps around.
-        */
-       object->hash_rand = object_hash_rand - 129;
 
        object->generation++;
        object->swblock_count = 0;
        RB_INIT(&object->swblock_root);
-       lwkt_token_init(&object->tok, "vmobjtk");
+       vm_object_lock_init(object);
 
        lwkt_gettoken(&vmobj_token);
        TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);
        vm_object_count++;
-       object_hash_rand = object->hash_rand;
        lwkt_reltoken(&vmobj_token);
 }
 
@@ -256,13 +252,11 @@ vm_object_reference_locked(vm_object_t object)
 {
        if (object) {
                ASSERT_LWKT_TOKEN_HELD(&vmobj_token);
-               vm_object_lock(object);
                object->ref_count++;
                if (object->type == OBJT_VNODE) {
                        vref(object->handle);
                        /* XXX what if the vnode is being destroyed? */
                }
-               vm_object_unlock(object);
        }
 }
 
@@ -451,7 +445,7 @@ vm_object_terminate(vm_object_t object)
        /*
         * Wait for the pageout daemon to be done with the object
         */
-       vm_object_pip_wait(object, "objtrm");
+       vm_object_pip_wait(object, "objtrm1");
 
        KASSERT(!object->paging_in_progress,
                ("vm_object_terminate: pageout in progress"));
@@ -476,7 +470,7 @@ vm_object_terminate(vm_object_t object)
         * Wait for any I/O to complete, after which there had better not
         * be any references left on the object.
         */
-       vm_object_pip_wait(object, "objtrm");
+       vm_object_pip_wait(object, "objtrm2");
 
        if (object->ref_count != 0) {
                panic("vm_object_terminate: object with references, "
@@ -494,6 +488,11 @@ vm_object_terminate(vm_object_t object)
        lwkt_reltoken(&vm_token);
 
        /*
+        * Wait for the object hold count to hit zero
+        */
+       vm_object_hold_wait(object);
+
+       /*
         * Let the pager know object is dead.
         */
        vm_pager_deallocate(object);
@@ -1514,6 +1513,11 @@ vm_object_collapse(vm_object_t object)
                                 "over pages during collapse!",
                                 backing_object));
 
+                       /*
+                        * Wait for hold count to hit zero
+                        */
+                       vm_object_hold_wait(backing_object);
+
                        /* (we are holding vmobj_token) */
                        TAILQ_REMOVE(&vm_object_list, backing_object,
                                     object_list);
@@ -1740,10 +1744,8 @@ vm_object_coalesce(vm_object_t prev_object, vm_pindex_t prev_pindex,
                return (TRUE);
        }
 
-       vm_object_lock(prev_object);
        if (prev_object->type != OBJT_DEFAULT &&
            prev_object->type != OBJT_SWAP) {
-               vm_object_unlock(prev_object);
                return (FALSE);
        }
 
@@ -1759,7 +1761,6 @@ vm_object_coalesce(vm_object_t prev_object, vm_pindex_t prev_pindex,
         */
 
        if (prev_object->backing_object != NULL) {
-               vm_object_unlock(prev_object);
                return (FALSE);
        }
 
@@ -1769,7 +1770,6 @@ vm_object_coalesce(vm_object_t prev_object, vm_pindex_t prev_pindex,
 
        if ((prev_object->ref_count > 1) &&
            (prev_object->size != next_pindex)) {
-               vm_object_unlock(prev_object);
                return (FALSE);
        }
 
@@ -1792,7 +1792,6 @@ vm_object_coalesce(vm_object_t prev_object, vm_pindex_t prev_pindex,
        if (next_pindex + next_size > prev_object->size)
                prev_object->size = next_pindex + next_size;
 
-       vm_object_unlock(prev_object);
        return (TRUE);
 }
 
@@ -1817,16 +1816,58 @@ vm_object_set_writeable_dirty(vm_object_t object)
        lwkt_reltoken(&vm_token);
 }
 
+static void
+vm_object_lock_init(vm_object_t obj)
+{
+}
+
+void
+vm_object_lock(vm_object_t obj)
+{
+       lwkt_getpooltoken(obj);
+}
+
 void
-vm_object_lock(vm_object_t object)
+vm_object_unlock(vm_object_t obj)
 {
-       lwkt_gettoken(&object->tok);
+       lwkt_relpooltoken(obj);
 }
 
 void
-vm_object_unlock(vm_object_t object)
+vm_object_hold(vm_object_t obj)
 {
-       lwkt_reltoken(&object->tok);
+       vm_object_lock(obj);
+
+       refcount_acquire(&obj->hold_count);
+}
+
+void
+vm_object_drop(vm_object_t obj)
+{
+       int rc;
+
+       rc = refcount_release(&obj->hold_count);
+       vm_object_unlock(obj);
+
+       if (rc) 
+               vm_object_hold_wake(obj);
+}
+
+static void
+vm_object_hold_wake(vm_object_t obj)
+{
+       wakeup(obj);
+}
+
+static void
+vm_object_hold_wait(vm_object_t obj)
+{
+       vm_object_lock(obj);
+
+       while (obj->hold_count)
+               tsleep(obj, 0, "vmobjhld", 0);
+
+       vm_object_unlock(obj);
 }
 
 #include "opt_ddb.h"

diff --git a/sys/vm/vm_object.h b/sys/vm/vm_object.h
index ec40039..cb29529 100644 (file)
--- a/sys/vm/vm_object.h
+++ b/sys/vm/vm_object.h
@@ -13,10 +13,6 @@
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *     This product includes software developed by the University of
- *     California, Berkeley and its contributors.
  * 4. Neither the name of the University nor the names of its contributors
  *    may be used to endorse or promote products derived from this software
  *    without specific prior written permission.
@@ -140,19 +136,16 @@ struct vm_object {
        vm_pindex_t size;               /* Object size */
        int ref_count;                  /* vmobj_token */
        int shadow_count;               /* how many objects that this is a shadow for */
-       int hash_rand;                  /* vm hash table randomizer     */
        objtype_t type;                 /* type of pager */
        u_short flags;                  /* see below */
        u_short pg_color;               /* color of first page in obj */
-       u_short unused01;
        int paging_in_progress;         /* Paging (in or out) so don't collapse or destroy */
        int resident_page_count;        /* number of resident pages */
         u_int agg_pv_list_count;        /* aggregate pv list count */
        struct vm_object *backing_object; /* object that I'm a shadow of */
        vm_ooffset_t backing_object_offset;/* Offset in backing object */
-       TAILQ_ENTRY(vm_object) pager_object_list; /* list of all objects of this pager type */
        void *handle;
-       struct lwkt_token tok;
+       int hold_count;                 /* refcount for object liveness */
        
        union {
                /*
@@ -234,9 +227,10 @@ vm_object_pip_subtract(vm_object_t object, int i)
 }
 
 static __inline void
-vm_object_pip_wakeup(vm_object_t object)
+vm_object_pip_wakeupn(vm_object_t object, int i)
 {
-       atomic_subtract_int(&object->paging_in_progress, 1);
+       if (i)
+               atomic_subtract_int(&object->paging_in_progress, i);
        if ((object->flags & OBJ_PIPWNT) && object->paging_in_progress == 0) {
                vm_object_clear_flag(object, OBJ_PIPWNT);
                wakeup(object);
@@ -244,14 +238,9 @@ vm_object_pip_wakeup(vm_object_t object)
 }
 
 static __inline void
-vm_object_pip_wakeupn(vm_object_t object, int i)
+vm_object_pip_wakeup(vm_object_t object)
 {
-       if (i)
-               atomic_subtract_int(&object->paging_in_progress, i);
-       if ((object->flags & OBJ_PIPWNT) && object->paging_in_progress == 0) {
-               vm_object_clear_flag(object, OBJ_PIPWNT);
-               wakeup(object);
-       }
+       vm_object_pip_wakeupn(object, 1);
 }
 
 static __inline void
@@ -298,6 +287,8 @@ void vm_object_dead_sleep(vm_object_t, const char *);
 void vm_object_dead_wakeup(vm_object_t);
 void vm_object_lock(vm_object_t);
 void vm_object_unlock(vm_object_t);
+void vm_object_hold(vm_object_t);
+void vm_object_drop(vm_object_t);
 
 #endif                         /* _KERNEL */
 

diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 7bd94b9..7a4d42e 100644 (file)
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -1989,19 +1989,6 @@ vm_page_event_internal(vm_page_t m, vm_page_event_t event)
        lwkt_reltoken(&vm_token);
 }
 
-
-void
-vm_page_lock(vm_page_t m)
-{
-       lwkt_getpooltoken(m);
-}
-
-void
-vm_page_unlock(vm_page_t m)
-{
-       lwkt_relpooltoken(m);
-}
-
 #include "opt_ddb.h"
 #ifdef DDB
 #include <sys/kernel.h>

diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index 6dc4eb8..a6f7db1 100644 (file)
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -534,8 +534,6 @@ void vm_page_event_internal(vm_page_t, vm_page_event_t);
 void vm_page_dirty(vm_page_t m);
 void vm_page_register_action(vm_page_action_t action, vm_page_event_t event);
 void vm_page_unregister_action(vm_page_action_t action);
-void vm_page_lock(vm_page_t m);
-void vm_page_unlock(vm_page_t m);
 
 /*
  * Reduce the protection of a page.  This routine never raises the