From 44293a8055ecb27ff83ef3482f7e2fd33b35ebe8 Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Wed, 8 May 2019 23:31:25 -0700 Subject: [PATCH] kernel - VM rework part 3 - Cleanup pass * Cleanup various structures and code --- sys/ddb/db_break.h | 2 +- sys/ddb/db_watch.h | 2 +- sys/kern/imgact_elf.c | 2 +- sys/kern/sys_process.c | 10 +- sys/vfs/procfs/procfs_map.c | 2 +- sys/vm/vm.h | 8 +- sys/vm/vm_extern.h | 98 ++++++++++--------- sys/vm/vm_fault.c | 34 ++++--- sys/vm/vm_kern.h | 4 +- sys/vm/vm_map.c | 187 +++++++++++++++++++----------------- sys/vm/vm_map.h | 14 +-- sys/vm/vm_object.c | 2 +- sys/vm/vm_pageout.h | 5 +- sys/vm/vnode_pager.c | 4 +- usr.bin/fstat/fstat.c | 2 +- 15 files changed, 202 insertions(+), 174 deletions(-) diff --git a/sys/ddb/db_break.h b/sys/ddb/db_break.h index c2959673c2..bc7715174d 100644 --- a/sys/ddb/db_break.h +++ b/sys/ddb/db_break.h @@ -47,7 +47,7 @@ */ struct db_breakpoint { - vm_map_t map; /* in this map */ + struct vm_map *map; /* in this map */ db_addr_t address; /* set here */ int init_count; /* number of times to skip bkpt */ int count; /* current count */ diff --git a/sys/ddb/db_watch.h b/sys/ddb/db_watch.h index 92b88073e2..2db5425654 100644 --- a/sys/ddb/db_watch.h +++ b/sys/ddb/db_watch.h @@ -47,7 +47,7 @@ */ typedef struct db_watchpoint { - vm_map_t map; /* in this map */ + struct vm_map *map; /* in this map */ db_addr_t loaddr; /* from this address */ db_addr_t hiaddr; /* to this address */ struct db_watchpoint *link; /* link in in-use or free chain */ diff --git a/sys/kern/imgact_elf.c b/sys/kern/imgact_elf.c index 949c4f9f9f..042ea2cd09 100644 --- a/sys/kern/imgact_elf.c +++ b/sys/kern/imgact_elf.c @@ -1192,7 +1192,7 @@ each_segment(struct proc *p, segment_callback func, void *closure, int writable) vm_map_entry_t entry; RB_FOREACH(entry, vm_map_rb_tree, &map->rb_root) { - vm_map_backing_t *ba; + vm_map_backing_t ba; vm_object_t obj; /* diff --git a/sys/kern/sys_process.c b/sys/kern/sys_process.c index 4580956059..807fb7adb9 100644 --- a/sys/kern/sys_process.c +++ b/sys/kern/sys_process.c @@ -56,11 +56,12 @@ /* use the equivalent procfs code */ #if 0 static int -pread (struct proc *procp, unsigned int addr, unsigned int *retval) { +pread (struct proc *procp, unsigned int addr, unsigned int *retval) +{ int rv; vm_map_t map, tmap; vm_object_t object; - vm_map_backing_t *ba; + vm_map_backing_t ba; vm_offset_t kva = 0; int page_offset; /* offset into page */ vm_offset_t pageno; /* page number */ @@ -113,11 +114,12 @@ pread (struct proc *procp, unsigned int addr, unsigned int *retval) { } static int -pwrite (struct proc *procp, unsigned int addr, unsigned int datum) { +pwrite (struct proc *procp, unsigned int addr, unsigned int datum) +{ int rv; vm_map_t map, tmap; vm_object_t object; - vm_map_backing_t *ba; + vm_map_backing_t ba; vm_offset_t kva = 0; int page_offset; /* offset into page */ vm_offset_t pageno; /* page number */ diff --git a/sys/vfs/procfs/procfs_map.c b/sys/vfs/procfs/procfs_map.c index f0307f1816..c87fbbfa03 100644 --- a/sys/vfs/procfs/procfs_map.c +++ b/sys/vfs/procfs/procfs_map.c @@ -86,7 +86,7 @@ procfs_domap(struct proc *curp, struct lwp *lp, struct pfsnode *pfs, lwkt_reltoken(&p->p_token); RB_FOREACH(entry, vm_map_rb_tree, &map->rb_root) { - vm_map_backing_t *ba; + vm_map_backing_t ba; vm_object_t obj; int ref_count, flags; vm_offset_t e_start, e_end; diff --git a/sys/vm/vm.h b/sys/vm/vm.h index a502e08219..4ec6ae7590 100644 --- a/sys/vm/vm.h +++ b/sys/vm/vm.h @@ -1,6 +1,7 @@ /* * Copyright (c) 1991, 1993 * The Regents of the University of California. All rights reserved. + * Copyright (c) 2003-2019 The DragonFly Project. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -100,15 +101,8 @@ typedef u_char vm_maptype_t; /* type of vm_map_entry */ #define VM_MAPTYPE_SUBMAP 3 #define VM_MAPTYPE_UKSMAP 4 /* user-kernel shared memory */ -union vm_map_object; -typedef union vm_map_object vm_map_object_t; - struct vm_map_entry; -typedef struct vm_map_entry *vm_map_entry_t; - struct vm_map; -typedef struct vm_map *vm_map_t; - struct vm_object; typedef struct vm_object *vm_object_t; diff --git a/sys/vm/vm_extern.h b/sys/vm/vm_extern.h index b11726a58b..74157db7c9 100644 --- a/sys/vm/vm_extern.h +++ b/sys/vm/vm_extern.h @@ -58,68 +58,78 @@ struct vnode; extern int vkernel_enable; -int grow (struct proc *, size_t); +int grow(struct proc *, size_t); int kernacc(c_caddr_t, int, int); -vm_offset_t kmem_alloc3 (vm_map_t, vm_size_t, vm_subsys_t id, int flags); -vm_offset_t kmem_alloc_nofault (vm_map_t, vm_size_t, vm_subsys_t id, vm_size_t); -vm_offset_t kmem_alloc_pageable (vm_map_t, vm_size_t, vm_subsys_t id); -vm_offset_t kmem_alloc_wait (vm_map_t, vm_size_t, vm_subsys_t id); -vm_offset_t kmem_alloc_attr(vm_map_t map, vm_size_t size, vm_subsys_t id, - int flags, vm_paddr_t low, vm_paddr_t high, vm_memattr_t memattr); -void kmem_free (vm_map_t, vm_offset_t, vm_size_t); -void kmem_free_wakeup (vm_map_t, vm_offset_t, vm_size_t); -void kmem_init (void); -void kmem_suballoc (vm_map_t, vm_map_t, vm_offset_t *, vm_offset_t *, vm_size_t); -void munmapfd (struct proc *, int); -int swaponvp (struct thread *, struct vnode *, u_quad_t); -void swapout_procs (int); +vm_offset_t kmem_alloc3(struct vm_map *, vm_size_t, + vm_subsys_t id, int flags); +vm_offset_t kmem_alloc_nofault(struct vm_map *, vm_size_t, + vm_subsys_t id, vm_size_t); +vm_offset_t kmem_alloc_pageable(struct vm_map *, vm_size_t, + vm_subsys_t id); +vm_offset_t kmem_alloc_wait(struct vm_map *, vm_size_t, + vm_subsys_t id); +vm_offset_t kmem_alloc_attr(struct vm_map *map, vm_size_t size, + vm_subsys_t id, int flags, + vm_paddr_t low, vm_paddr_t high, vm_memattr_t memattr); +void kmem_free(struct vm_map *, vm_offset_t, vm_size_t); +void kmem_free_wakeup(struct vm_map *, vm_offset_t, vm_size_t); +void kmem_init(void); +void kmem_suballoc(struct vm_map *, struct vm_map *, + vm_offset_t *, vm_offset_t *, vm_size_t); +void munmapfd(struct proc *, int); +int swaponvp(struct thread *, struct vnode *, u_quad_t); +void swapout_procs(int); int useracc(c_caddr_t, int, int); -int vm_fault (vm_map_t, vm_offset_t, vm_prot_t, int); -vm_page_t vm_fault_page (vm_map_t, vm_offset_t, vm_prot_t, int, int *, int *); -vm_page_t vm_fault_page_quick (vm_offset_t, vm_prot_t, int *, int *); -void vm_fault_copy_entry (vm_map_t, vm_map_t, vm_map_entry_t, vm_map_entry_t); -void vm_fault_unwire (vm_map_t, vm_map_entry_t); -int vm_fault_wire (vm_map_t, vm_map_entry_t, boolean_t, int); -void vm_fork (struct proc *, struct proc *, int); -int vm_test_nominal (void); -void vm_wait_nominal (void); +int vm_fault(struct vm_map *, vm_offset_t, vm_prot_t, int); +vm_page_t vm_fault_page(struct vm_map *, vm_offset_t, + vm_prot_t, int, int *, int *); +vm_page_t vm_fault_page_quick(vm_offset_t, vm_prot_t, int *, int *); +void vm_fault_copy_entry(struct vm_map *, struct vm_map *, + struct vm_map_entry *, struct vm_map_entry *); +void vm_fault_unwire(struct vm_map *, struct vm_map_entry *); +int vm_fault_wire(struct vm_map *, struct vm_map_entry *, boolean_t, int); +void vm_fork(struct proc *, struct proc *, int); +int vm_test_nominal(void); +void vm_wait_nominal(void); void vm_init_limits(struct proc *); -int vm_mmap (vm_map_t, vm_offset_t *, vm_size_t, vm_prot_t, vm_prot_t, int, void *, vm_ooffset_t); +int vm_mmap(struct vm_map *, vm_offset_t *, vm_size_t, + vm_prot_t, vm_prot_t, int, void *, vm_ooffset_t); int vm_mmap_to_errno(int rv); -vm_offset_t kmem_alloc_contig (vm_offset_t, vm_paddr_t, vm_paddr_t, vm_offset_t); -void vm_set_page_size (void); -struct vmspace *vmspace_alloc (vm_offset_t, vm_offset_t); -void vmspace_initrefs (struct vmspace *); -int vmspace_getrefs (struct vmspace *); -void vmspace_hold (struct vmspace *); -void vmspace_drop (struct vmspace *); -void vmspace_ref (struct vmspace *); -void vmspace_rel (struct vmspace *); -void vmspace_relexit (struct vmspace *); -void vmspace_exitfree (struct proc *); +vm_offset_t kmem_alloc_contig(vm_offset_t, vm_paddr_t, + vm_paddr_t, vm_offset_t); +void vm_set_page_size(void); +struct vmspace *vmspace_alloc(vm_offset_t, vm_offset_t); +void vmspace_initrefs(struct vmspace *); +int vmspace_getrefs(struct vmspace *); +void vmspace_hold(struct vmspace *); +void vmspace_drop(struct vmspace *); +void vmspace_ref(struct vmspace *); +void vmspace_rel(struct vmspace *); +void vmspace_relexit(struct vmspace *); +void vmspace_exitfree(struct proc *); void *kmem_alloc_swapbacked(kmem_anon_desc_t *kp, vm_size_t size, vm_subsys_t id); void kmem_free_swapbacked(kmem_anon_desc_t *kp); -struct vmspace *vmspace_fork (struct vmspace *); -void vmspace_exec (struct proc *, struct vmspace *); -void vmspace_unshare (struct proc *); -void vslock (caddr_t, u_int); -void vsunlock (caddr_t, u_int); -void vm_object_print (/* db_expr_t */ long, boolean_t, /* db_expr_t */ long, - char *); +struct vmspace *vmspace_fork(struct vmspace *); +void vmspace_exec(struct proc *, struct vmspace *); +void vmspace_unshare(struct proc *); +void vslock(caddr_t, u_int); +void vsunlock(caddr_t, u_int); +void vm_object_print(/* db_expr_t */ long, boolean_t, + /* db_expr_t */ long, char *); static __inline vm_offset_t -kmem_alloc (vm_map_t map, vm_size_t size, vm_subsys_t id) +kmem_alloc(struct vm_map *map, vm_size_t size, vm_subsys_t id) { return(kmem_alloc3(map, size, id, 0)); } static __inline vm_offset_t -kmem_alloc_stack (vm_map_t map, vm_size_t size, int kmflags) +kmem_alloc_stack(struct vm_map *map, vm_size_t size, int kmflags) { return(kmem_alloc3(map, size, VM_SUBSYS_STACK, kmflags|KM_STACK)); } diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c index 59b9e441d5..051d5aaecb 100644 --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -130,18 +130,16 @@ struct faultstate { vm_page_t m; - vm_map_backing_t *ba; - vm_pindex_t pindex; + vm_map_backing_t ba; vm_prot_t prot; vm_page_t first_m; - vm_map_backing_t *first_ba; + vm_map_backing_t first_ba; vm_prot_t first_prot; vm_map_t map; vm_map_entry_t entry; int lookup_still_valid; int hardfault; int fault_flags; - int map_generation; int shared; int msoftonly; int first_shared; @@ -211,8 +209,7 @@ release_page(struct faultstate *fs) * NOTE: Once unlocked any cached fs->entry becomes invalid, any reuse * requires relocking and then checking the timestamp. * - * NOTE: vm_map_lock_read() does not bump fs->map->timestamp so we do - * not have to update fs->map_generation here. + * NOTE: vm_map_lock_read() does not bump fs->map->timestamp. * * NOTE: This function can fail due to a deadlock against the caller's * holding of a vm_page BUSY. @@ -529,7 +526,6 @@ RetryFault: * * Misc checks. Save the map generation number to detect races. */ - fs.map_generation = fs.map->timestamp; fs.lookup_still_valid = TRUE; fs.first_m = NULL; fs.ba = fs.first_ba; /* so unlock_things() works */ @@ -890,6 +886,14 @@ vm_fault_quick(struct faultstate *fs, vm_pindex_t first_pindex, /* * Don't waste time if the object is only being used by one vm_map. + * + * WARNING! We can't rely on obj->ref_count here because it might + * be part of a shared ba chain, and we can't rely on + * ba->refs for the same reason. The combination of it + * being the ba embedded in the entry (aka first_ba) AND + * ref_count == 1 would work, but OBJ_ONEMAPPING is better + * because it will remain flagged even when ref_count > 1 + * for situations where entries are clipped. */ obj = fs->first_ba->object; if (obj->flags & OBJ_ONEMAPPING) @@ -1162,7 +1166,6 @@ RetryFault: * * Misc checks. Save the map generation number to detect races. */ - fs.map_generation = fs.map->timestamp; fs.lookup_still_valid = TRUE; fs.first_m = NULL; fs.ba = fs.first_ba; @@ -1450,7 +1453,6 @@ RetryFault: fs.entry = &entry; fs.first_prot = fault_type; fs.wflags = 0; - /*fs.map_generation = 0; unused */ /* * Make a reference to this object to prevent its disposal while we @@ -1734,7 +1736,7 @@ int vm_fault_object(struct faultstate *fs, vm_pindex_t first_pindex, vm_prot_t fault_type, int allow_nofault) { - vm_map_backing_t *next_ba; + vm_map_backing_t next_ba; vm_pindex_t pindex; int error; @@ -2628,12 +2630,12 @@ vm_fault_unwire(vm_map_t map, vm_map_entry_t entry) } /* - * Copy all of the pages from a wired-down map entry to another. + * Copy all of the pages from one map entry to another. If the source + * is wired down we just use vm_page_lookup(). If not we use + * vm_fault_object(). * * The source and destination maps must be locked for write. * The source and destination maps token must be held - * The source map entry must be wired down (or be a sharing map - * entry corresponding to a main map entry that is wired down). * * No other requirements. * @@ -2925,7 +2927,7 @@ static void vm_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry, int prot, int fault_flags) { - vm_map_backing_t *ba; /* first ba */ + vm_map_backing_t ba; /* first ba */ struct lwp *lp; vm_page_t m; vm_offset_t addr; @@ -2977,8 +2979,8 @@ vm_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry, int prot, for (i = 0; i < maxpages; ++i) { vm_object_t lobject; vm_object_t nobject; - vm_map_backing_t *last_ba; /* last ba */ - vm_map_backing_t *next_ba; /* last ba */ + vm_map_backing_t last_ba; /* last ba */ + vm_map_backing_t next_ba; /* last ba */ int allocated = 0; int error; diff --git a/sys/vm/vm_kern.h b/sys/vm/vm_kern.h index ce54255240..11a3c973aa 100644 --- a/sys/vm/vm_kern.h +++ b/sys/vm/vm_kern.h @@ -77,7 +77,7 @@ struct vm_object; * Anonymous swap-backed kernel memory descriptor */ struct kmem_anon_desc { - vm_map_t map; + struct vm_map *map; vm_size_t size; vm_offset_t data; struct vm_object *object; @@ -108,6 +108,6 @@ extern vm_offset_t kernel_vm_end; /* XXX - elsewhere? */ struct malloc_type; extern void *contigmalloc_map(u_long, struct malloc_type *, int, - vm_paddr_t, vm_paddr_t, u_long, u_long, vm_map_t); + vm_paddr_t, vm_paddr_t, u_long, u_long, struct vm_map *); #endif /* _VM_VM_KERN_H_ */ diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c index b996a519b2..1cd8df1cee 100644 --- a/sys/vm/vm_map.c +++ b/sys/vm/vm_map.c @@ -161,11 +161,15 @@ SYSCTL_INT(_vm, OID_AUTO, map_relock_enable, CTLFLAG_RW, static int vm_map_partition_enable = 1; SYSCTL_INT(_vm, OID_AUTO, map_partition_enable, CTLFLAG_RW, &vm_map_partition_enable, 0, "Break up larger vm_map_entry's"); +static int vm_map_backing_limit = 5; +SYSCTL_INT(_vm, OID_AUTO, map_backing_limit, CTLFLAG_RW, + &vm_map_backing_limit, 0, "ba.backing_ba link depth"); static void vmspace_drop_notoken(struct vmspace *vm); static void vm_map_entry_shadow(vm_map_entry_t entry, int addref); static vm_map_entry_t vm_map_entry_create(vm_map_t map, int *); static void vm_map_entry_dispose (vm_map_t map, vm_map_entry_t entry, int *); +static void vm_map_entry_dispose_ba (vm_map_backing_t ba); static void _vm_map_clip_end (vm_map_t, vm_map_entry_t, vm_offset_t, int *); static void _vm_map_clip_start (vm_map_t, vm_map_entry_t, vm_offset_t, int *); static void vm_map_entry_delete (vm_map_t, vm_map_entry_t, int *); @@ -721,7 +725,7 @@ static void vm_map_entry_shadow(vm_map_entry_t entry, int addref) { - vm_map_backing_t *ba; + vm_map_backing_t ba; vm_size_t length; vm_object_t source; vm_object_t result; @@ -740,18 +744,16 @@ vm_map_entry_shadow(vm_map_entry_t entry, int addref) /* * Don't create the new object if the old object isn't shared. - * We have to chain wait before adding the reference to avoid - * racing a collapse or deallocation. * - * Clear OBJ_ONEMAPPING flag when shadowing. + * If addref is non-zero additional ref(s) are being added (probably + * for map entry fork purposes), so clear OBJ_ONEMAPPING. * - * The caller owns a ref on source via *objectp which we are going - * to replace. This ref is inherited by the backing_object assignment. - * from nobject and does not need to be incremented here. - * - * However, we add a temporary extra reference to the original source - * prior to holding nobject in case we block, to avoid races where - * someone else might believe that the source can be collapsed. + * WARNING! Checking ref_count == 1 only works because we are testing + * the object embedded in the entry (entry->ba.object). + * This test DOES NOT WORK if checking an object hanging off + * the backing chain (entry->ba.backing_ba list) because the + * vm_map_backing might be shared, or part of a chain that + * is shared. Checking ba->refs is worthless. */ drop_source = 0; if (source) { @@ -817,6 +819,7 @@ vm_map_entry_shadow(vm_map_entry_t entry, int addref) ba->refs = 1; /* initialize ref count */ entry->ba.object = result; /* new ba (at head of entry) */ entry->ba.backing_ba = ba; + entry->ba.backing_count = ba->backing_count + 1; entry->ba.offset = 0; entry->ba.refs = 0; @@ -1070,33 +1073,59 @@ vm_map_entry_create(vm_map_t map, int *countp) } /* - * Dispose of a vm_map_entry that is no longer being referenced. + * Dispose of the dynamically allocated backing_ba chain associated + * with a vm_map_entry. * - * No requirements. + * We decrement the (possibly shared) element and kfree() on the + * 1->0 transition. We only iterate to the next backing_ba when + * the previous one went through a 1->0 transition. */ static void -vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry, int *countp) +vm_map_entry_dispose_ba(vm_map_backing_t ba) { - struct globaldata *gd = mycpu; - vm_map_backing_t *ba; + vm_map_backing_t next; long refs; - /* - * We must also dispose of the vm_map_backing, kfree()ing the - * possibly shared element on the 1->0 transition. We only iterate - * to the next backing_ba when the previous one went through a - * 1->0 transition. - */ - while ((ba = entry->ba.backing_ba) != NULL) { + while (ba) { refs = atomic_fetchadd_long(&ba->refs, -1); if (refs > 1) break; KKASSERT(refs == 1); /* transitioned 1->0 */ if (ba->object) vm_object_deallocate(ba->object); - entry->ba.backing_ba = ba->backing_ba; + next = ba->backing_ba; kfree(ba, M_MAP_BACKING); + ba = next; } +} + +/* + * Dispose of a vm_map_entry that is no longer being referenced. + * + * No requirements. + */ +static void +vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry, int *countp) +{ + struct globaldata *gd = mycpu; + + /* + * Dispose of the base object and the backing link. + */ + switch(entry->maptype) { + case VM_MAPTYPE_NORMAL: + case VM_MAPTYPE_VPAGETABLE: + case VM_MAPTYPE_SUBMAP: + if (entry->ba.object) + vm_object_deallocate(entry->ba.object); + break; + case VM_MAPTYPE_UKSMAP: + /* XXX TODO */ + break; + default: + break; + } + vm_map_entry_dispose_ba(entry->ba.backing_ba); /* * Cleanup for safety. @@ -1349,6 +1378,7 @@ vm_map_insert(vm_map_t map, int *countp, void *map_object, void *map_aux, new_entry->aux.map_aux = map_aux; new_entry->ba.map_object = map_object; new_entry->ba.backing_ba = NULL; + new_entry->ba.backing_count = 0; new_entry->ba.offset = offset; new_entry->ba.refs = 0; new_entry->ba.flags = 0; @@ -1645,8 +1675,6 @@ vm_map_simplify_entry(vm_map_t map, vm_map_entry_t entry, int *countp) vm_map_entry_unlink(map, prev); entry->start = prev->start; entry->ba.offset = prev->ba.offset; - if (prev->ba.object) - vm_object_deallocate(prev->ba.object); vm_map_entry_dispose(map, prev, countp); } } @@ -1668,8 +1696,6 @@ vm_map_simplify_entry(vm_map_t map, vm_map_entry_t entry, int *countp) (next->wired_count == entry->wired_count)) { vm_map_entry_unlink(map, next); entry->end = next->end; - if (next->ba.object) - vm_object_deallocate(next->ba.object); vm_map_entry_dispose(map, next, countp); } } @@ -2900,7 +2926,7 @@ vm_map_clean(vm_map_t map, vm_offset_t start, vm_offset_t end, vm_map_entry_t current; vm_map_entry_t next; vm_map_entry_t entry; - vm_map_backing_t *ba; + vm_map_backing_t ba; vm_size_t size; vm_object_t object; vm_ooffset_t offset; @@ -3119,20 +3145,6 @@ vm_map_entry_delete(vm_map_t map, vm_map_entry_t entry, int *countp) { vm_map_entry_unlink(map, entry); map->size -= entry->end - entry->start; - - switch(entry->maptype) { - case VM_MAPTYPE_NORMAL: - case VM_MAPTYPE_VPAGETABLE: - case VM_MAPTYPE_SUBMAP: - vm_object_deallocate(entry->ba.object); - break; - case VM_MAPTYPE_UKSMAP: - /* XXX TODO */ - break; - default: - break; - } - vm_map_entry_dispose(map, entry, countp); } @@ -3411,43 +3423,53 @@ vm_map_copy_entry(vm_map_t src_map, vm_map_t dst_map, vm_map_entry_t src_entry, vm_map_entry_t dst_entry) { vm_object_t src_object; - vm_object_t oobject; /* * Nothing to do for special map types */ if (dst_entry->maptype == VM_MAPTYPE_SUBMAP || - dst_entry->maptype == VM_MAPTYPE_UKSMAP) + dst_entry->maptype == VM_MAPTYPE_UKSMAP) { return; + } if (src_entry->maptype == VM_MAPTYPE_SUBMAP || - src_entry->maptype == VM_MAPTYPE_UKSMAP) + src_entry->maptype == VM_MAPTYPE_UKSMAP) { return; + } - if (src_entry->wired_count == 0) { + if (src_entry->wired_count) { /* - * If the source entry is marked needs_copy, it is already - * write-protected. + * Of course, wired down pages can't be set copy-on-write. + * Cause wired pages to be copied into the new map by + * simulating faults (the new pages are pageable) * - * To avoid interacting with a vm_fault that might have - * released its vm_map, we must acquire the fronting - * object. + * Scrap ba.object (its ref-count has not yet been adjusted + * so we can just NULL out the field). Remove the backing + * store. + * + * Then call vm_fault_copy_entry() to create a new object + * in dst_entry and copy the wired pages from src to dst. */ - oobject = src_entry->ba.object; - if (oobject) - vm_object_hold(oobject); - + dst_entry->ba.object = NULL; + vm_map_entry_dispose_ba(dst_entry->ba.backing_ba); + dst_entry->ba.backing_ba = NULL; + dst_entry->ba.backing_count = 0; + vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry); + } else { if ((src_entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) { + /* + * If the source entry is not already marked NEEDS_COPY + * we need to write-protect the PTEs. + */ pmap_protect(src_map->pmap, - src_entry->start, - src_entry->end, - src_entry->protection & ~VM_PROT_WRITE); + src_entry->start, + src_entry->end, + src_entry->protection & ~VM_PROT_WRITE); } /* - * Make a copy of the object. - * - * The object has been copied but we need to bump the - * ref-count and then mark both entries NEEDS_COPY. + * dst_entry.ba_object might be stale. Update it (its + * ref-count has not yet been updated so just overwrite + * the field). * * If there is no object then we are golden. Also, in * this situation if there are no backing_ba linkages then @@ -3456,39 +3478,32 @@ vm_map_copy_entry(vm_map_t src_map, vm_map_t dst_map, * ba.offset cannot otherwise be modified because it effects * the offsets for the entire backing_ba chain. */ - KKASSERT(dst_entry->ba.object == oobject); - if (oobject) { - vm_object_reference_locked(oobject); - vm_object_clear_flag(oobject, OBJ_ONEMAPPING); + src_object = src_entry->ba.object; + + if (src_object) { + vm_object_hold(src_object); /* for ref & flag clr */ + vm_object_reference_locked(src_object); + vm_object_clear_flag(src_object, OBJ_ONEMAPPING); - src_object = src_entry->ba.object; - dst_entry->ba.object = src_object; src_entry->eflags |= (MAP_ENTRY_COW | MAP_ENTRY_NEEDS_COPY); dst_entry->eflags |= (MAP_ENTRY_COW | MAP_ENTRY_NEEDS_COPY); - dst_entry->ba.offset = src_entry->ba.offset; + KKASSERT(dst_entry->ba.offset == src_entry->ba.offset); + vm_object_drop(src_object); } else { if (dst_entry->ba.backing_ba == NULL) dst_entry->ba.offset = 0; } - pmap_copy(dst_map->pmap, src_map->pmap, dst_entry->start, - dst_entry->end - dst_entry->start, - src_entry->start); - if (oobject) - vm_object_drop(oobject); - } else { + /* - * Of course, wired down pages can't be set copy-on-write. - * Cause wired pages to be copied into the new map by - * simulating faults (the new pages are pageable) - * - * Scrap ba.object (its ref-count has not yet been adjusted - * so we can just NULL out the field). vm_fault_copy_entry() - * will create a new object. + * Normal, allow the backing_ba link depth to + * increase. */ - dst_entry->ba.object = NULL; - vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry); + pmap_copy(dst_map->pmap, src_map->pmap, + dst_entry->start, + dst_entry->end - dst_entry->start, + src_entry->start); } } @@ -4145,7 +4160,7 @@ vm_map_lookup(vm_map_t *var_map, /* IN/OUT */ vm_offset_t vaddr, vm_prot_t fault_typea, vm_map_entry_t *out_entry, /* OUT */ - vm_map_backing_t **bap, /* OUT */ + struct vm_map_backing **bap, /* OUT */ vm_pindex_t *pindex, /* OUT */ vm_prot_t *out_prot, /* OUT */ int *wflags) /* OUT */ diff --git a/sys/vm/vm_map.h b/sys/vm/vm_map.h index da28508b19..898dbb26db 100644 --- a/sys/vm/vm_map.h +++ b/sys/vm/vm_map.h @@ -166,8 +166,6 @@ typedef enum { * The offset is cumulatively added from its parent, allowing easy splits * and merges. */ -union vm_map_object; - struct vm_map_backing { struct vm_map_backing *backing_ba; /* backing store */ @@ -185,10 +183,10 @@ struct vm_map_backing { vm_ooffset_t offset; /* cumulative offset */ long refs; /* shared refs */ uint32_t flags; - uint32_t unused01; + uint32_t backing_count; /* #entries backing us */ }; -typedef struct vm_map_backing vm_map_backing_t; +typedef struct vm_map_backing *vm_map_backing_t; #define VM_MAP_BACK_EXCL_HEUR 0x00000001U @@ -213,7 +211,7 @@ struct vm_map_entry { vm_offset_t start; /* start address */ vm_offset_t end; /* end address */ union vm_map_aux aux; /* auxillary data */ - vm_map_backing_t ba; /* backing object chain */ + struct vm_map_backing ba; /* backing object chain */ vm_eflags_t eflags; /* map entry flags */ vm_maptype_t maptype; /* type of VM mapping */ vm_prot_t protection; /* protection code */ @@ -223,6 +221,8 @@ struct vm_map_entry { vm_subsys_t id; /* subsystem id */ }; +typedef struct vm_map_entry *vm_map_entry_t; + #define MAPENT_FREELIST(ent) (ent)->rb_entry.rbe_left #define MAP_ENTRY_NOSYNC 0x0001 @@ -350,6 +350,8 @@ struct vm_map { vm_offset_t pgout_offset; /* for RLIMIT_RSS scans */ }; +typedef struct vm_map *vm_map_t; + /* * vm_flags_t values */ @@ -664,7 +666,7 @@ int vm_map_insert (vm_map_t, int *, void *, void *, vm_maptype_t, vm_subsys_t id, vm_prot_t, vm_prot_t, int); int vm_map_lookup (vm_map_t *, vm_offset_t, vm_prot_t, - vm_map_entry_t *, vm_map_backing_t **, + vm_map_entry_t *, struct vm_map_backing **, vm_pindex_t *, vm_prot_t *, int *); void vm_map_lookup_done (vm_map_t, vm_map_entry_t, int); boolean_t vm_map_lookup_entry (vm_map_t, vm_offset_t, vm_map_entry_t *); diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c index 6bc471066b..c923bbcb1a 100644 --- a/sys/vm/vm_object.c +++ b/sys/vm/vm_object.c @@ -1725,7 +1725,7 @@ static int vm_object_in_map (vm_object_t object); static int _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry) { - vm_map_backing_t *ba; + vm_map_backing_t ba; vm_map_t tmpm; vm_map_entry_t tmpe; int entcount; diff --git a/sys/vm/vm_pageout.h b/sys/vm/vm_pageout.h index 736b05203a..e3c23a3543 100644 --- a/sys/vm/vm_pageout.h +++ b/sys/vm/vm_pageout.h @@ -106,10 +106,13 @@ extern void vm_wait (int timo); extern void vm_wait_pfault (void); #ifdef _KERNEL + +struct vm_map; + void vm_pageout_cluster (vm_page_t, vm_object_t); int vm_pageout_flush (vm_page_t *, int, int); int vm_pageout_clean (vm_page_t); -void vm_pageout_map_deactivate_pages(vm_map_t map, vm_pindex_t limit); +void vm_pageout_map_deactivate_pages(struct vm_map *map, vm_pindex_t limit); #endif diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c index 75d5662eca..f479fe6bcb 100644 --- a/sys/vm/vnode_pager.c +++ b/sys/vm/vnode_pager.c @@ -787,9 +787,9 @@ vnode_pager_generic_putpages(struct vnode *vp, vm_page_t *m, int bytecount, * Caller must hold the first object. */ struct vnode * -vnode_pager_lock(vm_map_backing_t *ba) +vnode_pager_lock(vm_map_backing_t ba) { - vm_map_backing_t *lba; + vm_map_backing_t lba; struct vnode *vp; vm_object_t lobject; int error; diff --git a/usr.bin/fstat/fstat.c b/usr.bin/fstat/fstat.c index a1737439f9..c13bde450f 100644 --- a/usr.bin/fstat/fstat.c +++ b/usr.bin/fstat/fstat.c @@ -395,7 +395,7 @@ dommap(struct proc *p) struct vm_map_entry entry; vm_map_entry_t ken; struct vm_object object; - vm_map_backing_t ba; + struct vm_map_backing ba; vm_object_t objp; int prot, fflags; -- 2.41.0