kernel - VM rework part 9 - Precursor work for terminal pv_entry removal
authorMatthew Dillon <dillon@apollo.backplane.com>
Fri, 17 May 2019 17:03:35 +0000 (10:03 -0700)
committerMatthew Dillon <dillon@apollo.backplane.com>
Mon, 20 May 2019 19:39:25 +0000 (12:39 -0700)
* Cleanup the API a bit

* Get rid of pmap_enter_quick()

* Remove unused procedures.

* Document that vm_page_protect() (and thus the related
  pmap_page_protect()) must be called with a hard-busied page.  This
  ensures that the operation does not race a new pmap_enter() of the page.

12 files changed:
sys/dev/drm/drm_vm.c
sys/net/netmap/netmap_freebsd.c
sys/platform/pc64/x86_64/pmap.c
sys/platform/vkernel64/platform/pmap.c
sys/vfs/tmpfs/tmpfs_vnops.c
sys/vm/pmap.h
sys/vm/vm_map.c
sys/vm/vm_object.c
sys/vm/vm_object.h
sys/vm/vm_page.c
sys/vm/vm_page.h
sys/vm/vm_page2.h

index 3b3f45e..5f4d8c4 100644 (file)
@@ -164,7 +164,7 @@ vm_phys_fictitious_to_vm_page(vm_paddr_t pa)
 
 int
 vm_phys_fictitious_reg_range(vm_paddr_t start, vm_paddr_t end,
-    vm_memattr_t memattr)
+                            vm_memattr_t memattr)
 {
         struct vm_phys_fictitious_seg *seg;
         vm_page_t fp;
index 38868ba..ad6f006 100644 (file)
@@ -233,8 +233,7 @@ vm_page_getfake(vm_paddr_t paddr, vm_memattr_t memattr)
 {
        vm_page_t m;
 
-       m = kmalloc(sizeof(struct vm_page), M_FICT_PAGES,
-           M_WAITOK | M_ZERO);
+       m = kmalloc(sizeof(struct vm_page), M_FICT_PAGES, M_WAITOK | M_ZERO);
        vm_page_initfake(m, paddr, memattr);
        return (m);
 }
index 0c149df..9d850d4 100644 (file)
@@ -5589,19 +5589,6 @@ done:
                pv_put(pt_pv);
 }
 
-/*
- * This code works like pmap_enter() but assumes VM_PROT_READ and not-wired.
- * This code also assumes that the pmap has no pre-existing entry for this
- * VA.
- *
- * This code currently may only be used on user pmaps, not kernel_pmap.
- */
-void
-pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m)
-{
-       pmap_enter(pmap, va, m, VM_PROT_READ, FALSE, NULL);
-}
-
 /*
  * Make a temporary mapping for a physical address.  This is only intended
  * to be used for panic dumps.
@@ -5625,10 +5612,12 @@ pmap_kenter_temporary(vm_paddr_t pa, long i)
 static int pmap_object_init_pt_callback(vm_page_t p, void *data);
 
 void
-pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_prot_t prot,
-                   vm_object_t object, vm_pindex_t pindex,
-                   vm_size_t size, int limit)
+pmap_object_init_pt(pmap_t pmap, vm_map_entry_t entry,
+                   vm_offset_t addr, vm_size_t size, int limit)
 {
+       vm_prot_t prot = entry->protection;
+       vm_object_t object = entry->ba.object;
+       vm_pindex_t pindex = atop(entry->ba.offset + (addr - entry->ba.start));
        struct rb_vm_page_scan_info info;
        struct lwp *lp;
        vm_size_t psize;
@@ -5693,6 +5682,7 @@ pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_prot_t prot,
        info.addr = addr;
        info.pmap = pmap;
        info.object = object;
+       info.entry = entry;
 
        /*
         * By using the NOLK scan, the callback function must be sure
@@ -5747,8 +5737,8 @@ again:
                        vm_page_deactivate(p);
                }
                rel_index = p->pindex - info->start_pindex;
-               pmap_enter_quick(info->pmap,
-                                info->addr + x86_64_ptob(rel_index), p);
+               pmap_enter(info->pmap, info->addr + x86_64_ptob(rel_index), p,
+                          VM_PROT_READ, FALSE, info->entry);
        }
        if (hard_busy)
                vm_page_wakeup(p);
@@ -5940,35 +5930,6 @@ pmap_copy_page_frag(vm_paddr_t src, vm_paddr_t dst, size_t bytes)
              bytes);
 }
 
-/*
- * Returns true if the pmap's pv is one of the first 16 pvs linked to from
- * this page.  This count may be changed upwards or downwards in the future;
- * it is only necessary that true be returned for a small subset of pmaps
- * for proper page aging.
- */
-boolean_t
-pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
-{
-       pv_entry_t pv;
-       int loops = 0;
-
-       if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
-               return FALSE;
-
-       vm_page_spin_lock(m);
-       TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
-               if (pv->pv_pmap == pmap) {
-                       vm_page_spin_unlock(m);
-                       return TRUE;
-               }
-               loops++;
-               if (loops >= 16)
-                       break;
-       }
-       vm_page_spin_unlock(m);
-       return (FALSE);
-}
-
 /*
  * Remove all pages from specified address space this aids process exit
  * speeds.  Also, this code may be special cased for the current process
index bbb370d..ea13f00 100644 (file)
@@ -2520,19 +2520,6 @@ validate:
        vm_object_drop(pmap->pm_pteobj);
 }
 
-/*
- * This code works like pmap_enter() but assumes VM_PROT_READ and not-wired.
- *
- * Currently this routine may only be used on user pmaps, not kernel_pmap.
- *
- * No requirements.
- */
-void
-pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m)
-{
-       pmap_enter(pmap, va, m, VM_PROT_READ, 0, NULL);
-}
-
 /*
  * Make a temporary mapping for a physical address.  This is only intended
  * to be used for panic dumps.
@@ -2558,10 +2545,12 @@ pmap_kenter_temporary(vm_paddr_t pa, long i)
 static int pmap_object_init_pt_callback(vm_page_t p, void *data);
 
 void
-pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_prot_t prot,
-                   vm_object_t object, vm_pindex_t pindex,
-                   vm_size_t size, int limit)
+pmap_object_init_pt(pmap_t pmap, vm_map_entry_t entry,
+                   vm_offset_t addr, vm_size_t size, int limit)
 {
+       vm_prot_t prot = entry->protection;
+       vm_object_t object = entry->ba.object;
+       vm_pindex_t pindex = atop(entry->ba.offset + (addr - entry->ba.start));
        struct rb_vm_page_scan_info info;
        struct lwp *lp;
        vm_size_t psize;
@@ -2613,6 +2602,7 @@ pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_prot_t prot,
        info.mpte = NULL;
        info.addr = addr;
        info.pmap = pmap;
+       info.entry = entry;
 
        vm_object_hold_shared(object);
        vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp,
@@ -2648,8 +2638,8 @@ pmap_object_init_pt_callback(vm_page_t p, void *data)
                if ((p->queue - p->pc) == PQ_CACHE)
                        vm_page_deactivate(p);
                rel_index = p->pindex - info->start_pindex;
-               pmap_enter_quick(info->pmap,
-                                info->addr + x86_64_ptob(rel_index), p);
+               pmap_enter(info->pmap, info->addr + x86_64_ptob(rel_index), p,
+                          VM_PROT_READ, FALSE, info->entry);
        }
        vm_page_wakeup(p);
        return(0);
@@ -2817,38 +2807,6 @@ pmap_copy_page_frag(vm_paddr_t src, vm_paddr_t dst, size_t bytes)
              bytes);
 }
 
-/*
- * Returns true if the pmap's pv is one of the first 16 pvs linked to
- * from this page.  This count may be changed upwards or downwards
- * in the future; it is only necessary that true be returned for a small
- * subset of pmaps for proper page aging.
- *
- * No other requirements.
- */
-boolean_t
-pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
-{
-       pv_entry_t pv;
-       int loops = 0;
-
-       if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
-               return FALSE;
-
-       vm_page_spin_lock(m);
-       TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
-               if (pv->pv_pmap == pmap) {
-                       vm_page_spin_unlock(m);
-                       return TRUE;
-               }
-               loops++;
-               if (loops >= 16)
-                       break;
-       }
-       vm_page_spin_unlock(m);
-
-       return (FALSE);
-}
-
 /*
  * Remove all pages from specified address space this aids process
  * exit speeds.  Also, this code is special cased for current
index a2dd4d2..ee636db 100644 (file)
@@ -1969,7 +1969,7 @@ tmpfs_move_pages_callback(vm_page_t p, void *data)
                info->error = -1;
                return -1;
        }
-       vm_page_rename(p, info->backing_object, pindex);
+       vm_page_rename(p, info->dest_object, pindex);
        vm_page_clear_commit(p);
        vm_page_wakeup(p);
        /* page automaticaly made dirty */
@@ -1986,7 +1986,7 @@ tmpfs_move_pages(vm_object_t src, vm_object_t dst)
        vm_object_hold(src);
        vm_object_hold(dst);
        info.object = src;
-       info.backing_object = dst;
+       info.dest_object = dst;
        do {
                info.error = 1;
                vm_page_rb_tree_RB_SCAN(&src->rb_memq, NULL,
index 6310781..6277b2c 100644 (file)
@@ -176,7 +176,6 @@ void                 pmap_copy_page (vm_paddr_t, vm_paddr_t);
 void            pmap_copy_page_frag (vm_paddr_t, vm_paddr_t, size_t bytes);
 void            pmap_enter (pmap_t, vm_offset_t, struct vm_page *,
                        vm_prot_t, boolean_t, struct vm_map_entry *);
-void            pmap_enter_quick (pmap_t, vm_offset_t, struct vm_page *);
 vm_page_t       pmap_fault_page_quick(pmap_t, vm_offset_t, vm_prot_t, int *);
 vm_paddr_t      pmap_extract (pmap_t pmap, vm_offset_t va, void **handlep);
 void            pmap_extract_done (void *handle);
@@ -185,10 +184,8 @@ void                pmap_init (void);
 boolean_t       pmap_is_modified (struct vm_page *m);
 int             pmap_ts_referenced (struct vm_page *m);
 vm_offset_t     pmap_map (vm_offset_t *, vm_paddr_t, vm_paddr_t, int);
-void            pmap_object_init_pt (pmap_t pmap, vm_offset_t addr,
-                   vm_prot_t prot, vm_object_t object, vm_pindex_t pindex,
-                   vm_offset_t size, int pagelimit);
-boolean_t       pmap_page_exists_quick (pmap_t pmap, struct vm_page *m);
+void            pmap_object_init_pt (pmap_t pmap, struct vm_map_entry *entry,
+                       vm_offset_t addr, vm_offset_t size, int pagelimit);
 void            pmap_page_protect (struct vm_page *m, vm_prot_t prot);
 void            pmap_page_init (struct vm_page *m);
 vm_paddr_t      uservtophys(vm_offset_t va);
index 26a19ce..5808818 100644 (file)
@@ -1423,8 +1423,9 @@ vm_map_insert(vm_map_t map, int *countp, void *map_object, void *map_aux,
                        vm_object_lock_swap();
                        vm_object_drop(object);
                }
-               pmap_object_init_pt(map->pmap, start, prot,
-                                   object, OFF_TO_IDX(offset), end - start,
+               pmap_object_init_pt(map->pmap, new_entry,
+                                   new_entry->ba.start,
+                                   new_entry->ba.end - new_entry->ba.start,
                                    cow & MAP_PREFAULT_PARTIAL);
                if (dorelock) {
                        vm_object_hold(object);
@@ -2432,12 +2433,9 @@ vm_map_madvise(vm_map_t map, vm_offset_t start, vm_offset_t end,
                        if (behav == MADV_WILLNEED &&
                            current->maptype != VM_MAPTYPE_VPAGETABLE) {
                                pmap_object_init_pt(
-                                   map->pmap, 
+                                   map->pmap, current,
                                    useStart,
-                                   current->protection,
-                                   current->ba.object,
-                                   pindex, 
-                                   (count << PAGE_SHIFT),
+                                   (delta << PAGE_SHIFT),
                                    MAP_PREFAULT_MADVISE
                                );
                        }
index e98ccd2..876e4c5 100644 (file)
@@ -1211,97 +1211,6 @@ vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags)
                vm_page_unhold(ma[i]);
 }
 
-/*
- * Same as vm_object_pmap_copy, except range checking really
- * works, and is meant for small sections of an object.
- *
- * This code protects resident pages by making them read-only
- * and is typically called on a fork or split when a page
- * is converted to copy-on-write.  
- *
- * NOTE: If the page is already at VM_PROT_NONE, calling
- * vm_page_protect will have no effect.
- */
-void
-vm_object_pmap_copy_1(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
-{
-       vm_pindex_t idx;
-       vm_page_t p;
-
-       if (object == NULL || (object->flags & OBJ_WRITEABLE) == 0)
-               return;
-
-       vm_object_hold(object);
-       for (idx = start; idx < end; idx++) {
-               p = vm_page_lookup(object, idx);
-               if (p == NULL)
-                       continue;
-               vm_page_protect(p, VM_PROT_READ);
-       }
-       vm_object_drop(object);
-}
-
-/*
- * Removes all physical pages in the specified object range from all
- * physical maps.
- *
- * The object must *not* be locked.
- */
-
-static int vm_object_pmap_remove_callback(vm_page_t p, void *data);
-
-void
-vm_object_pmap_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
-{
-       struct rb_vm_page_scan_info info;
-
-       if (object == NULL)
-               return;
-       if (start == end)
-               return;
-       info.start_pindex = start;
-       info.end_pindex = end - 1;
-       info.count = 0;
-       info.object = object;
-
-       vm_object_hold(object);
-       do {
-               info.error = 0;
-               vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp,
-                                       vm_object_pmap_remove_callback, &info);
-       } while (info.error);
-       if (start == 0 && end == object->size)
-               vm_object_clear_flag(object, OBJ_WRITEABLE);
-       vm_object_drop(object);
-}
-
-/*
- * The caller must hold the object
- */
-static int
-vm_object_pmap_remove_callback(vm_page_t p, void *data)
-{
-       struct rb_vm_page_scan_info *info = data;
-
-       if (info->object != p->object ||
-           p->pindex < info->start_pindex ||
-           p->pindex > info->end_pindex) {
-               kprintf("vm_object_pmap_remove_callback: obj/pg race %p/%p\n",
-                       info->object, p);
-               info->error = 1;
-               return(0);
-       }
-
-       vm_page_protect(p, VM_PROT_NONE);
-
-       /*
-        * Must be at end to avoid SMP races, caller holds object token
-        */
-       if ((++info->count & 63) == 0)
-               lwkt_user_yield();
-       return(0);
-}
-
 /*
  * Implements the madvise function at the object/page level.
  *
index db6a09b..98edacd 100644 (file)
@@ -317,8 +317,6 @@ void vm_object_init1 (void);
 void vm_object_page_clean (vm_object_t, vm_pindex_t, vm_pindex_t, int);
 void vm_object_page_remove (vm_object_t, vm_pindex_t, vm_pindex_t, boolean_t);
 void vm_object_pmap_copy (vm_object_t, vm_pindex_t, vm_pindex_t);
-void vm_object_pmap_copy_1 (vm_object_t, vm_pindex_t, vm_pindex_t);
-void vm_object_pmap_remove (vm_object_t, vm_pindex_t, vm_pindex_t);
 void vm_object_madvise (vm_object_t, vm_pindex_t, vm_pindex_t, int);
 void vm_object_init2 (void);
 vm_page_t vm_fault_object_page(vm_object_t, vm_ooffset_t,
index 90a9362..f56bfaa 100644 (file)
@@ -1343,7 +1343,6 @@ vm_page_unhold(vm_page_t m)
 void
 vm_page_initfake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr)
 {
-
        if ((m->flags & PG_FICTITIOUS) != 0) {
                /*
                 * The page's memattr might have changed since the
@@ -1390,8 +1389,7 @@ vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
        atomic_add_int(&object->generation, 1);
 
        /*
-        * Record the object/offset pair in this page and add the
-        * pv_list_count of the page to the object.
+        * Associate the VM page with an (object, offset).
         *
         * The vm_page spin lock is required for interactions with the pmap.
         */
index a82cc96..ca7cbd5 100644 (file)
@@ -216,9 +216,9 @@ struct rb_vm_page_scan_info {
        int             count;
        int             unused01;
        vm_offset_t     addr;
-       vm_pindex_t     backing_offset_index;
+       struct vm_map_entry *entry;
        struct vm_object *object;
-       struct vm_object *backing_object;
+       struct vm_object *dest_object;
        struct vm_page  *mpte;
        struct pmap     *pmap;
        struct vm_map   *map;
index c1d057f..e59572f 100644 (file)
@@ -323,9 +323,11 @@ vm_page_sbusy_drop(vm_page_t m)
  * Since 'prot' is usually a constant, this inline usually winds up optimizing
  * out the primary conditional.
  *
+ * Must be called with (m) hard-busied.
+ *
  * WARNING: VM_PROT_NONE can block, but will loop until all mappings have
- * been cleared.  Callers should be aware that other page related elements
- * might have changed, however.
+ *         been cleared.  Callers should be aware that other page related
+ *         elements might have changed, however.
  */
 static __inline void
 vm_page_protect(vm_page_t m, int prot)