0b9d6097f7582681280b325ca04a65ade8d55c07
[dragonfly.git] / sys / platform / pc64 / x86_64 / pmap.c
1 /*
2  * Copyright (c) 1991 Regents of the University of California.
3  * Copyright (c) 1994 John S. Dyson
4  * Copyright (c) 1994 David Greenman
5  * Copyright (c) 2003 Peter Wemm
6  * Copyright (c) 2005-2008 Alan L. Cox <alc@cs.rice.edu>
7  * Copyright (c) 2008, 2009 The DragonFly Project.
8  * Copyright (c) 2008, 2009 Jordan Gordeev.
9  * Copyright (c) 2011-2012 Matthew Dillon
10  * All rights reserved.
11  *
12  * This code is derived from software contributed to Berkeley by
13  * the Systems Programming Group of the University of Utah Computer
14  * Science Department and William Jolitz of UUNET Technologies Inc.
15  *
16  * Redistribution and use in source and binary forms, with or without
17  * modification, are permitted provided that the following conditions
18  * are met:
19  * 1. Redistributions of source code must retain the above copyright
20  *    notice, this list of conditions and the following disclaimer.
21  * 2. Redistributions in binary form must reproduce the above copyright
22  *    notice, this list of conditions and the following disclaimer in the
23  *    documentation and/or other materials provided with the distribution.
24  * 3. All advertising materials mentioning features or use of this software
25  *    must display the following acknowledgement:
26  *      This product includes software developed by the University of
27  *      California, Berkeley and its contributors.
28  * 4. Neither the name of the University nor the names of its contributors
29  *    may be used to endorse or promote products derived from this software
30  *    without specific prior written permission.
31  *
32  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
33  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
34  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
35  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
36  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
37  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
38  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
39  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
40  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
41  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
42  * SUCH DAMAGE.
43  */
44 /*
45  * Manage physical address maps for x86-64 systems.
46  */
47
48 #if 0 /* JG */
49 #include "opt_disable_pse.h"
50 #include "opt_pmap.h"
51 #endif
52 #include "opt_msgbuf.h"
53
54 #include <sys/param.h>
55 #include <sys/kernel.h>
56 #include <sys/proc.h>
57 #include <sys/msgbuf.h>
58 #include <sys/vmmeter.h>
59 #include <sys/mman.h>
60 #include <sys/systm.h>
61
62 #include <vm/vm.h>
63 #include <vm/vm_param.h>
64 #include <sys/sysctl.h>
65 #include <sys/lock.h>
66 #include <vm/vm_kern.h>
67 #include <vm/vm_page.h>
68 #include <vm/vm_map.h>
69 #include <vm/vm_object.h>
70 #include <vm/vm_extern.h>
71 #include <vm/vm_pageout.h>
72 #include <vm/vm_pager.h>
73 #include <vm/vm_zone.h>
74
75 #include <sys/user.h>
76 #include <sys/thread2.h>
77 #include <sys/sysref2.h>
78 #include <sys/spinlock2.h>
79 #include <vm/vm_page2.h>
80
81 #include <machine/cputypes.h>
82 #include <machine/md_var.h>
83 #include <machine/specialreg.h>
84 #include <machine/smp.h>
85 #include <machine_base/apic/apicreg.h>
86 #include <machine/globaldata.h>
87 #include <machine/pmap.h>
88 #include <machine/pmap_inval.h>
89 #include <machine/inttypes.h>
90
91 #include <ddb/ddb.h>
92
93 #define PMAP_KEEP_PDIRS
94 #ifndef PMAP_SHPGPERPROC
95 #define PMAP_SHPGPERPROC 2000
96 #endif
97
98 #if defined(DIAGNOSTIC)
99 #define PMAP_DIAGNOSTIC
100 #endif
101
102 #define MINPV 2048
103
104 /*
105  * pmap debugging will report who owns a pv lock when blocking.
106  */
107 #ifdef PMAP_DEBUG
108
109 #define PMAP_DEBUG_DECL         ,const char *func, int lineno
110 #define PMAP_DEBUG_ARGS         , __func__, __LINE__
111 #define PMAP_DEBUG_COPY         , func, lineno
112
113 #define pv_get(pmap, pindex, pmarkp)    _pv_get(pmap, pindex, pmarkp    \
114                                                         PMAP_DEBUG_ARGS)
115 #define pv_lock(pv)                     _pv_lock(pv                     \
116                                                         PMAP_DEBUG_ARGS)
117 #define pv_hold_try(pv)                 _pv_hold_try(pv                 \
118                                                         PMAP_DEBUG_ARGS)
119 #define pv_alloc(pmap, pindex, isnewp)  _pv_alloc(pmap, pindex, isnewp  \
120                                                         PMAP_DEBUG_ARGS)
121
122 #else
123
124 #define PMAP_DEBUG_DECL
125 #define PMAP_DEBUG_ARGS
126 #define PMAP_DEBUG_COPY
127
128 #define pv_get(pmap, pindex, pmarkp)            _pv_get(pmap, pindex, pmarkp)
129 #define pv_lock(pv)                     _pv_lock(pv)
130 #define pv_hold_try(pv)                 _pv_hold_try(pv)
131 #define pv_alloc(pmap, pindex, isnewp)  _pv_alloc(pmap, pindex, isnewp)
132
133 #endif
134
135 /*
136  * Get PDEs and PTEs for user/kernel address space
137  */
138 #define pdir_pde(m, v) (m[(vm_offset_t)(v) >> PDRSHIFT])
139
140 #define pmap_pde_v(pmap, pte)           ((*(pd_entry_t *)pte & pmap->pmap_bits[PG_V_IDX]) != 0)
141 #define pmap_pte_w(pmap, pte)           ((*(pt_entry_t *)pte & pmap->pmap_bits[PG_W_IDX]) != 0)
142 #define pmap_pte_m(pmap, pte)           ((*(pt_entry_t *)pte & pmap->pmap_bits[PG_M_IDX]) != 0)
143 #define pmap_pte_u(pmap, pte)           ((*(pt_entry_t *)pte & pmap->pmap_bits[PG_U_IDX]) != 0)
144 #define pmap_pte_v(pmap, pte)           ((*(pt_entry_t *)pte & pmap->pmap_bits[PG_V_IDX]) != 0)
145
146 /*
147  * Given a map and a machine independent protection code,
148  * convert to a vax protection code.
149  */
150 #define pte_prot(m, p)          \
151         (m->protection_codes[p & (VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE)])
152 static int protection_codes[PROTECTION_CODES_SIZE];
153
154 struct pmap kernel_pmap;
155
156 MALLOC_DEFINE(M_OBJPMAP, "objpmap", "pmaps associated with VM objects");
157
158 vm_paddr_t avail_start;         /* PA of first available physical page */
159 vm_paddr_t avail_end;           /* PA of last available physical page */
160 vm_offset_t virtual2_start;     /* cutout free area prior to kernel start */
161 vm_offset_t virtual2_end;
162 vm_offset_t virtual_start;      /* VA of first avail page (after kernel bss) */
163 vm_offset_t virtual_end;        /* VA of last avail page (end of kernel AS) */
164 vm_offset_t KvaStart;           /* VA start of KVA space */
165 vm_offset_t KvaEnd;             /* VA end of KVA space (non-inclusive) */
166 vm_offset_t KvaSize;            /* max size of kernel virtual address space */
167 static boolean_t pmap_initialized = FALSE;      /* Has pmap_init completed? */
168 //static int pgeflag;           /* PG_G or-in */
169 //static int pseflag;           /* PG_PS or-in */
170 uint64_t PatMsr;
171
172 static int ndmpdp;
173 static vm_paddr_t dmaplimit;
174 static int nkpt;
175 vm_offset_t kernel_vm_end = VM_MIN_KERNEL_ADDRESS;
176
177 static pt_entry_t pat_pte_index[PAT_INDEX_SIZE];        /* PAT -> PG_ bits */
178 /*static pt_entry_t pat_pde_index[PAT_INDEX_SIZE];*/    /* PAT -> PG_ bits */
179
180 static uint64_t KPTbase;
181 static uint64_t KPTphys;
182 static uint64_t KPDphys;        /* phys addr of kernel level 2 */
183 static uint64_t KPDbase;        /* phys addr of kernel level 2 @ KERNBASE */
184 uint64_t KPDPphys;      /* phys addr of kernel level 3 */
185 uint64_t KPML4phys;     /* phys addr of kernel level 4 */
186
187 static uint64_t DMPDphys;       /* phys addr of direct mapped level 2 */
188 static uint64_t DMPDPphys;      /* phys addr of direct mapped level 3 */
189
190 /*
191  * Data for the pv entry allocation mechanism
192  */
193 static vm_zone_t pvzone;
194 static struct vm_zone pvzone_store;
195 static struct vm_object pvzone_obj;
196 static int pv_entry_max=0, pv_entry_high_water=0;
197 static int pmap_pagedaemon_waken = 0;
198 static struct pv_entry *pvinit;
199
200 /*
201  * All those kernel PT submaps that BSD is so fond of
202  */
203 pt_entry_t *CMAP1 = NULL, *ptmmap;
204 caddr_t CADDR1 = NULL, ptvmmap = NULL;
205 static pt_entry_t *msgbufmap;
206 struct msgbuf *msgbufp=NULL;
207
208 /*
209  * PMAP default PG_* bits. Needed to be able to add
210  * EPT/NPT pagetable pmap_bits for the VMM module
211  */
212 uint64_t pmap_bits_default[] = {
213                 REGULAR_PMAP,                                   /* TYPE_IDX             0 */
214                 X86_PG_V,                                       /* PG_V_IDX             1 */
215                 X86_PG_RW,                                      /* PG_RW_IDX            2 */
216                 X86_PG_U,                                       /* PG_U_IDX             3 */
217                 X86_PG_A,                                       /* PG_A_IDX             4 */
218                 X86_PG_M,                                       /* PG_M_IDX             5 */
219                 X86_PG_PS,                                      /* PG_PS_IDX3           6 */
220                 X86_PG_G,                                       /* PG_G_IDX             7 */
221                 X86_PG_AVAIL1,                                  /* PG_AVAIL1_IDX        8 */
222                 X86_PG_AVAIL2,                                  /* PG_AVAIL2_IDX        9 */
223                 X86_PG_AVAIL3,                                  /* PG_AVAIL3_IDX        10 */
224                 X86_PG_NC_PWT | X86_PG_NC_PCD,                  /* PG_N_IDX     11 */
225 };
226 /*
227  * Crashdump maps.
228  */
229 static pt_entry_t *pt_crashdumpmap;
230 static caddr_t crashdumpmap;
231
232 static int pmap_debug = 0;
233 SYSCTL_INT(_machdep, OID_AUTO, pmap_debug, CTLFLAG_RW,
234     &pmap_debug, 0, "Debug pmap's");
235 #ifdef PMAP_DEBUG2
236 static int pmap_enter_debug = 0;
237 SYSCTL_INT(_machdep, OID_AUTO, pmap_enter_debug, CTLFLAG_RW,
238     &pmap_enter_debug, 0, "Debug pmap_enter's");
239 #endif
240 static int pmap_yield_count = 64;
241 SYSCTL_INT(_machdep, OID_AUTO, pmap_yield_count, CTLFLAG_RW,
242     &pmap_yield_count, 0, "Yield during init_pt/release");
243 static int pmap_mmu_optimize = 0;
244 SYSCTL_INT(_machdep, OID_AUTO, pmap_mmu_optimize, CTLFLAG_RW,
245     &pmap_mmu_optimize, 0, "Share page table pages when possible");
246 int pmap_fast_kernel_cpusync = 0;
247 SYSCTL_INT(_machdep, OID_AUTO, pmap_fast_kernel_cpusync, CTLFLAG_RW,
248     &pmap_fast_kernel_cpusync, 0, "Share page table pages when possible");
249 int pmap_dynamic_delete = -1;
250 SYSCTL_INT(_machdep, OID_AUTO, pmap_dynamic_delete, CTLFLAG_RW,
251     &pmap_dynamic_delete, 0, "Dynamically delete PT/PD/PDPs");
252
253 #define DISABLE_PSE
254
255 /* Standard user access funtions */
256 extern int std_copyinstr (const void *udaddr, void *kaddr, size_t len,
257     size_t *lencopied);
258 extern int std_copyin (const void *udaddr, void *kaddr, size_t len);
259 extern int std_copyout (const void *kaddr, void *udaddr, size_t len);
260 extern int std_fubyte (const void *base);
261 extern int std_subyte (void *base, int byte);
262 extern long std_fuword (const void *base);
263 extern int std_suword (void *base, long word);
264 extern int std_suword32 (void *base, int word);
265
266 static void pv_hold(pv_entry_t pv);
267 static int _pv_hold_try(pv_entry_t pv
268                                 PMAP_DEBUG_DECL);
269 static void pv_drop(pv_entry_t pv);
270 static void _pv_lock(pv_entry_t pv
271                                 PMAP_DEBUG_DECL);
272 static void pv_unlock(pv_entry_t pv);
273 static pv_entry_t _pv_alloc(pmap_t pmap, vm_pindex_t pindex, int *isnew
274                                 PMAP_DEBUG_DECL);
275 static pv_entry_t _pv_get(pmap_t pmap, vm_pindex_t pindex, vm_pindex_t **pmarkp
276                                 PMAP_DEBUG_DECL);
277 static pv_entry_t pv_get_try(pmap_t pmap, vm_pindex_t pindex,
278                                 vm_pindex_t **pmarkp, int *errorp);
279 static void pv_put(pv_entry_t pv);
280 static void pv_free(pv_entry_t pv, pv_entry_t pvp);
281 static void *pv_pte_lookup(pv_entry_t pv, vm_pindex_t pindex);
282 static pv_entry_t pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex,
283                       pv_entry_t *pvpp);
284 static pv_entry_t pmap_allocpte_seg(pmap_t pmap, vm_pindex_t ptepindex,
285                       pv_entry_t *pvpp, vm_map_entry_t entry, vm_offset_t va);
286 static void pmap_remove_pv_pte(pv_entry_t pv, pv_entry_t pvp,
287                         pmap_inval_bulk_t *bulk, int destroy);
288 static vm_page_t pmap_remove_pv_page(pv_entry_t pv);
289 static int pmap_release_pv(pv_entry_t pv, pv_entry_t pvp,
290                         pmap_inval_bulk_t *bulk);
291
292 struct pmap_scan_info;
293 static void pmap_remove_callback(pmap_t pmap, struct pmap_scan_info *info,
294                       pv_entry_t pte_pv, vm_pindex_t *pte_placemark,
295                       pv_entry_t pt_pv, int sharept,
296                       vm_offset_t va, pt_entry_t *ptep, void *arg __unused);
297 static void pmap_protect_callback(pmap_t pmap, struct pmap_scan_info *info,
298                       pv_entry_t pte_pv, vm_pindex_t *pte_placemark,
299                       pv_entry_t pt_pv, int sharept,
300                       vm_offset_t va, pt_entry_t *ptep, void *arg __unused);
301
302 static void i386_protection_init (void);
303 static void create_pagetables(vm_paddr_t *firstaddr);
304 static void pmap_remove_all (vm_page_t m);
305 static boolean_t pmap_testbit (vm_page_t m, int bit);
306
307 static pt_entry_t * pmap_pte_quick (pmap_t pmap, vm_offset_t va);
308 static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
309
310 static void pmap_pinit_defaults(struct pmap *pmap);
311 static void pv_placemarker_wait(pmap_t pmap, vm_pindex_t *pmark);
312 static void pv_placemarker_wakeup(pmap_t pmap, vm_pindex_t *pmark);
313
314 static unsigned pdir4mb;
315
316 static int
317 pv_entry_compare(pv_entry_t pv1, pv_entry_t pv2)
318 {
319         if (pv1->pv_pindex < pv2->pv_pindex)
320                 return(-1);
321         if (pv1->pv_pindex > pv2->pv_pindex)
322                 return(1);
323         return(0);
324 }
325
326 RB_GENERATE2(pv_entry_rb_tree, pv_entry, pv_entry,
327              pv_entry_compare, vm_pindex_t, pv_pindex);
328
329 static __inline
330 void
331 pmap_page_stats_adding(vm_page_t m)
332 {
333         globaldata_t gd = mycpu;
334
335         if (TAILQ_EMPTY(&m->md.pv_list)) {
336                 ++gd->gd_vmtotal.t_arm;
337         } else if (TAILQ_FIRST(&m->md.pv_list) ==
338                    TAILQ_LAST(&m->md.pv_list, md_page_pv_list)) {
339                 ++gd->gd_vmtotal.t_armshr;
340                 ++gd->gd_vmtotal.t_avmshr;
341         } else {
342                 ++gd->gd_vmtotal.t_avmshr;
343         }
344 }
345
346 static __inline
347 void
348 pmap_page_stats_deleting(vm_page_t m)
349 {
350         globaldata_t gd = mycpu;
351
352         if (TAILQ_EMPTY(&m->md.pv_list)) {
353                 --gd->gd_vmtotal.t_arm;
354         } else if (TAILQ_FIRST(&m->md.pv_list) ==
355                    TAILQ_LAST(&m->md.pv_list, md_page_pv_list)) {
356                 --gd->gd_vmtotal.t_armshr;
357                 --gd->gd_vmtotal.t_avmshr;
358         } else {
359                 --gd->gd_vmtotal.t_avmshr;
360         }
361 }
362
363 /*
364  * Move the kernel virtual free pointer to the next
365  * 2MB.  This is used to help improve performance
366  * by using a large (2MB) page for much of the kernel
367  * (.text, .data, .bss)
368  */
369 static
370 vm_offset_t
371 pmap_kmem_choose(vm_offset_t addr)
372 {
373         vm_offset_t newaddr = addr;
374
375         newaddr = roundup2(addr, NBPDR);
376         return newaddr;
377 }
378
379 /*
380  * pmap_pte_quick:
381  *
382  *      Super fast pmap_pte routine best used when scanning the pv lists.
383  *      This eliminates many course-grained invltlb calls.  Note that many of
384  *      the pv list scans are across different pmaps and it is very wasteful
385  *      to do an entire invltlb when checking a single mapping.
386  */
387 static __inline pt_entry_t *pmap_pte(pmap_t pmap, vm_offset_t va);
388
389 static
390 pt_entry_t *
391 pmap_pte_quick(pmap_t pmap, vm_offset_t va)
392 {
393         return pmap_pte(pmap, va);
394 }
395
396 /*
397  * Returns the pindex of a page table entry (representing a terminal page).
398  * There are NUPTE_TOTAL page table entries possible (a huge number)
399  *
400  * x86-64 has a 48-bit address space, where bit 47 is sign-extended out.
401  * We want to properly translate negative KVAs.
402  */
403 static __inline
404 vm_pindex_t
405 pmap_pte_pindex(vm_offset_t va)
406 {
407         return ((va >> PAGE_SHIFT) & (NUPTE_TOTAL - 1));
408 }
409
410 /*
411  * Returns the pindex of a page table.
412  */
413 static __inline
414 vm_pindex_t
415 pmap_pt_pindex(vm_offset_t va)
416 {
417         return (NUPTE_TOTAL + ((va >> PDRSHIFT) & (NUPT_TOTAL - 1)));
418 }
419
420 /*
421  * Returns the pindex of a page directory.
422  */
423 static __inline
424 vm_pindex_t
425 pmap_pd_pindex(vm_offset_t va)
426 {
427         return (NUPTE_TOTAL + NUPT_TOTAL +
428                 ((va >> PDPSHIFT) & (NUPD_TOTAL - 1)));
429 }
430
431 static __inline
432 vm_pindex_t
433 pmap_pdp_pindex(vm_offset_t va)
434 {
435         return (NUPTE_TOTAL + NUPT_TOTAL + NUPD_TOTAL +
436                 ((va >> PML4SHIFT) & (NUPDP_TOTAL - 1)));
437 }
438
439 static __inline
440 vm_pindex_t
441 pmap_pml4_pindex(void)
442 {
443         return (NUPTE_TOTAL + NUPT_TOTAL + NUPD_TOTAL + NUPDP_TOTAL);
444 }
445
446 /*
447  * Return various clipped indexes for a given VA
448  *
449  * Returns the index of a pt in a page directory, representing a page
450  * table.
451  */
452 static __inline
453 vm_pindex_t
454 pmap_pt_index(vm_offset_t va)
455 {
456         return ((va >> PDRSHIFT) & ((1ul << NPDEPGSHIFT) - 1));
457 }
458
459 /*
460  * Returns the index of a pd in a page directory page, representing a page
461  * directory.
462  */
463 static __inline
464 vm_pindex_t
465 pmap_pd_index(vm_offset_t va)
466 {
467         return ((va >> PDPSHIFT) & ((1ul << NPDPEPGSHIFT) - 1));
468 }
469
470 /*
471  * Returns the index of a pdp in the pml4 table, representing a page
472  * directory page.
473  */
474 static __inline
475 vm_pindex_t
476 pmap_pdp_index(vm_offset_t va)
477 {
478         return ((va >> PML4SHIFT) & ((1ul << NPML4EPGSHIFT) - 1));
479 }
480
481 /*
482  * The placemarker hash must be broken up into four zones so lock
483  * ordering semantics continue to work (e.g. pte, pt, pd, then pdp).
484  *
485  * Placemarkers are used to 'lock' page table indices that do not have
486  * a pv_entry.  This allows the pmap to support managed and unmanaged
487  * pages and shared page tables.
488  */
489 #define PM_PLACE_BASE   (PM_PLACEMARKS >> 2)
490
491 static __inline
492 vm_pindex_t *
493 pmap_placemarker_hash(pmap_t pmap, vm_pindex_t pindex)
494 {
495         int hi;
496
497         if (pindex < pmap_pt_pindex(0))         /* zone 0 - PTE */
498                 hi = 0;
499         else if (pindex < pmap_pd_pindex(0))    /* zone 1 - PT */
500                 hi = PM_PLACE_BASE;
501         else if (pindex < pmap_pdp_pindex(0))   /* zone 2 - PD */
502                 hi = PM_PLACE_BASE << 1;
503         else                                    /* zone 3 - PDP (and PML4E) */
504                 hi = PM_PLACE_BASE | (PM_PLACE_BASE << 1);
505         hi += pindex & (PM_PLACE_BASE - 1);
506
507         return (&pmap->pm_placemarks[hi]);
508 }
509
510
511 /*
512  * Generic procedure to index a pte from a pt, pd, or pdp.
513  *
514  * NOTE: Normally passed pindex as pmap_xx_index().  pmap_xx_pindex() is NOT
515  *       a page table page index but is instead of PV lookup index.
516  */
517 static
518 void *
519 pv_pte_lookup(pv_entry_t pv, vm_pindex_t pindex)
520 {
521         pt_entry_t *pte;
522
523         pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pv->pv_m));
524         return(&pte[pindex]);
525 }
526
527 /*
528  * Return pointer to PDP slot in the PML4
529  */
530 static __inline
531 pml4_entry_t *
532 pmap_pdp(pmap_t pmap, vm_offset_t va)
533 {
534         return (&pmap->pm_pml4[pmap_pdp_index(va)]);
535 }
536
537 /*
538  * Return pointer to PD slot in the PDP given a pointer to the PDP
539  */
540 static __inline
541 pdp_entry_t *
542 pmap_pdp_to_pd(pml4_entry_t pdp_pte, vm_offset_t va)
543 {
544         pdp_entry_t *pd;
545
546         pd = (pdp_entry_t *)PHYS_TO_DMAP(pdp_pte & PG_FRAME);
547         return (&pd[pmap_pd_index(va)]);
548 }
549
550 /*
551  * Return pointer to PD slot in the PDP.
552  */
553 static __inline
554 pdp_entry_t *
555 pmap_pd(pmap_t pmap, vm_offset_t va)
556 {
557         pml4_entry_t *pdp;
558
559         pdp = pmap_pdp(pmap, va);
560         if ((*pdp & pmap->pmap_bits[PG_V_IDX]) == 0)
561                 return NULL;
562         return (pmap_pdp_to_pd(*pdp, va));
563 }
564
565 /*
566  * Return pointer to PT slot in the PD given a pointer to the PD
567  */
568 static __inline
569 pd_entry_t *
570 pmap_pd_to_pt(pdp_entry_t pd_pte, vm_offset_t va)
571 {
572         pd_entry_t *pt;
573
574         pt = (pd_entry_t *)PHYS_TO_DMAP(pd_pte & PG_FRAME);
575         return (&pt[pmap_pt_index(va)]);
576 }
577
578 /*
579  * Return pointer to PT slot in the PD
580  *
581  * SIMPLE PMAP NOTE: Simple pmaps (embedded in objects) do not have PDPs,
582  *                   so we cannot lookup the PD via the PDP.  Instead we
583  *                   must look it up via the pmap.
584  */
585 static __inline
586 pd_entry_t *
587 pmap_pt(pmap_t pmap, vm_offset_t va)
588 {
589         pdp_entry_t *pd;
590         pv_entry_t pv;
591         vm_pindex_t pd_pindex;
592
593         if (pmap->pm_flags & PMAP_FLAG_SIMPLE) {
594                 pd_pindex = pmap_pd_pindex(va);
595                 spin_lock(&pmap->pm_spin);
596                 pv = pv_entry_rb_tree_RB_LOOKUP(&pmap->pm_pvroot, pd_pindex);
597                 spin_unlock(&pmap->pm_spin);
598                 if (pv == NULL || pv->pv_m == NULL)
599                         return NULL;
600                 return (pmap_pd_to_pt(VM_PAGE_TO_PHYS(pv->pv_m), va));
601         } else {
602                 pd = pmap_pd(pmap, va);
603                 if (pd == NULL || (*pd & pmap->pmap_bits[PG_V_IDX]) == 0)
604                          return NULL;
605                 return (pmap_pd_to_pt(*pd, va));
606         }
607 }
608
609 /*
610  * Return pointer to PTE slot in the PT given a pointer to the PT
611  */
612 static __inline
613 pt_entry_t *
614 pmap_pt_to_pte(pd_entry_t pt_pte, vm_offset_t va)
615 {
616         pt_entry_t *pte;
617
618         pte = (pt_entry_t *)PHYS_TO_DMAP(pt_pte & PG_FRAME);
619         return (&pte[pmap_pte_index(va)]);
620 }
621
622 /*
623  * Return pointer to PTE slot in the PT
624  */
625 static __inline
626 pt_entry_t *
627 pmap_pte(pmap_t pmap, vm_offset_t va)
628 {
629         pd_entry_t *pt;
630
631         pt = pmap_pt(pmap, va);
632         if (pt == NULL || (*pt & pmap->pmap_bits[PG_V_IDX]) == 0)
633                  return NULL;
634         if ((*pt & pmap->pmap_bits[PG_PS_IDX]) != 0)
635                 return ((pt_entry_t *)pt);
636         return (pmap_pt_to_pte(*pt, va));
637 }
638
639 /*
640  * Of all the layers (PTE, PT, PD, PDP, PML4) the best one to cache is
641  * the PT layer.  This will speed up core pmap operations considerably.
642  *
643  * NOTE: The pmap spinlock does not need to be held but the passed-in pv
644  *       must be in a known associated state (typically by being locked when
645  *       the pmap spinlock isn't held).  We allow the race for that case.
646  *
647  * NOTE: pm_pvhint is only accessed (read) with the spin-lock held, using
648  *       cpu_ccfence() to prevent compiler optimizations from reloading the
649  *       field.
650  */
651 static __inline
652 void
653 pv_cache(pv_entry_t pv, vm_pindex_t pindex)
654 {
655         if (pindex >= pmap_pt_pindex(0) && pindex <= pmap_pd_pindex(0)) {
656                 if (pv->pv_pmap)
657                         pv->pv_pmap->pm_pvhint = pv;
658         }
659 }
660
661
662 /*
663  * Return address of PT slot in PD (KVM only)
664  *
665  * Cannot be used for user page tables because it might interfere with
666  * the shared page-table-page optimization (pmap_mmu_optimize).
667  */
668 static __inline
669 pd_entry_t *
670 vtopt(vm_offset_t va)
671 {
672         uint64_t mask = ((1ul << (NPDEPGSHIFT + NPDPEPGSHIFT +
673                                   NPML4EPGSHIFT)) - 1);
674
675         return (PDmap + ((va >> PDRSHIFT) & mask));
676 }
677
678 /*
679  * KVM - return address of PTE slot in PT
680  */
681 static __inline
682 pt_entry_t *
683 vtopte(vm_offset_t va)
684 {
685         uint64_t mask = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT +
686                                   NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
687
688         return (PTmap + ((va >> PAGE_SHIFT) & mask));
689 }
690
691 static uint64_t
692 allocpages(vm_paddr_t *firstaddr, long n)
693 {
694         uint64_t ret;
695
696         ret = *firstaddr;
697         bzero((void *)ret, n * PAGE_SIZE);
698         *firstaddr += n * PAGE_SIZE;
699         return (ret);
700 }
701
702 static
703 void
704 create_pagetables(vm_paddr_t *firstaddr)
705 {
706         long i;         /* must be 64 bits */
707         long nkpt_base;
708         long nkpt_phys;
709         int j;
710
711         /*
712          * We are running (mostly) V=P at this point
713          *
714          * Calculate NKPT - number of kernel page tables.  We have to
715          * accomodoate prealloction of the vm_page_array, dump bitmap,
716          * MSGBUF_SIZE, and other stuff.  Be generous.
717          *
718          * Maxmem is in pages.
719          *
720          * ndmpdp is the number of 1GB pages we wish to map.
721          */
722         ndmpdp = (ptoa(Maxmem) + NBPDP - 1) >> PDPSHIFT;
723         if (ndmpdp < 4)         /* Minimum 4GB of dirmap */
724                 ndmpdp = 4;
725         KKASSERT(ndmpdp <= NKPDPE * NPDEPG);
726
727         /*
728          * Starting at the beginning of kvm (not KERNBASE).
729          */
730         nkpt_phys = (Maxmem * sizeof(struct vm_page) + NBPDR - 1) / NBPDR;
731         nkpt_phys += (Maxmem * sizeof(struct pv_entry) + NBPDR - 1) / NBPDR;
732         nkpt_phys += ((nkpt + nkpt + 1 + NKPML4E + NKPDPE + NDMPML4E +
733                        ndmpdp) + 511) / 512;
734         nkpt_phys += 128;
735
736         /*
737          * Starting at KERNBASE - map 2G worth of page table pages.
738          * KERNBASE is offset -2G from the end of kvm.
739          */
740         nkpt_base = (NPDPEPG - KPDPI) * NPTEPG; /* typically 2 x 512 */
741
742         /*
743          * Allocate pages
744          */
745         KPTbase = allocpages(firstaddr, nkpt_base);
746         KPTphys = allocpages(firstaddr, nkpt_phys);
747         KPML4phys = allocpages(firstaddr, 1);
748         KPDPphys = allocpages(firstaddr, NKPML4E);
749         KPDphys = allocpages(firstaddr, NKPDPE);
750
751         /*
752          * Calculate the page directory base for KERNBASE,
753          * that is where we start populating the page table pages.
754          * Basically this is the end - 2.
755          */
756         KPDbase = KPDphys + ((NKPDPE - (NPDPEPG - KPDPI)) << PAGE_SHIFT);
757
758         DMPDPphys = allocpages(firstaddr, NDMPML4E);
759         if ((amd_feature & AMDID_PAGE1GB) == 0)
760                 DMPDphys = allocpages(firstaddr, ndmpdp);
761         dmaplimit = (vm_paddr_t)ndmpdp << PDPSHIFT;
762
763         /*
764          * Fill in the underlying page table pages for the area around
765          * KERNBASE.  This remaps low physical memory to KERNBASE.
766          *
767          * Read-only from zero to physfree
768          * XXX not fully used, underneath 2M pages
769          */
770         for (i = 0; (i << PAGE_SHIFT) < *firstaddr; i++) {
771                 ((pt_entry_t *)KPTbase)[i] = i << PAGE_SHIFT;
772                 ((pt_entry_t *)KPTbase)[i] |=
773                     pmap_bits_default[PG_RW_IDX] |
774                     pmap_bits_default[PG_V_IDX] |
775                     pmap_bits_default[PG_G_IDX];
776         }
777
778         /*
779          * Now map the initial kernel page tables.  One block of page
780          * tables is placed at the beginning of kernel virtual memory,
781          * and another block is placed at KERNBASE to map the kernel binary,
782          * data, bss, and initial pre-allocations.
783          */
784         for (i = 0; i < nkpt_base; i++) {
785                 ((pd_entry_t *)KPDbase)[i] = KPTbase + (i << PAGE_SHIFT);
786                 ((pd_entry_t *)KPDbase)[i] |=
787                     pmap_bits_default[PG_RW_IDX] |
788                     pmap_bits_default[PG_V_IDX];
789         }
790         for (i = 0; i < nkpt_phys; i++) {
791                 ((pd_entry_t *)KPDphys)[i] = KPTphys + (i << PAGE_SHIFT);
792                 ((pd_entry_t *)KPDphys)[i] |=
793                     pmap_bits_default[PG_RW_IDX] |
794                     pmap_bits_default[PG_V_IDX];
795         }
796
797         /*
798          * Map from zero to end of allocations using 2M pages as an
799          * optimization.  This will bypass some of the KPTBase pages
800          * above in the KERNBASE area.
801          */
802         for (i = 0; (i << PDRSHIFT) < *firstaddr; i++) {
803                 ((pd_entry_t *)KPDbase)[i] = i << PDRSHIFT;
804                 ((pd_entry_t *)KPDbase)[i] |=
805                     pmap_bits_default[PG_RW_IDX] |
806                     pmap_bits_default[PG_V_IDX] |
807                     pmap_bits_default[PG_PS_IDX] |
808                     pmap_bits_default[PG_G_IDX];
809         }
810
811         /*
812          * And connect up the PD to the PDP.  The kernel pmap is expected
813          * to pre-populate all of its PDs.  See NKPDPE in vmparam.h.
814          */
815         for (i = 0; i < NKPDPE; i++) {
816                 ((pdp_entry_t *)KPDPphys)[NPDPEPG - NKPDPE + i] =
817                                 KPDphys + (i << PAGE_SHIFT);
818                 ((pdp_entry_t *)KPDPphys)[NPDPEPG - NKPDPE + i] |=
819                     pmap_bits_default[PG_RW_IDX] |
820                     pmap_bits_default[PG_V_IDX] |
821                     pmap_bits_default[PG_U_IDX];
822         }
823
824         /*
825          * Now set up the direct map space using either 2MB or 1GB pages
826          * Preset PG_M and PG_A because demotion expects it.
827          *
828          * When filling in entries in the PD pages make sure any excess
829          * entries are set to zero as we allocated enough PD pages
830          */
831         if ((amd_feature & AMDID_PAGE1GB) == 0) {
832                 for (i = 0; i < NPDEPG * ndmpdp; i++) {
833                         ((pd_entry_t *)DMPDphys)[i] = i << PDRSHIFT;
834                         ((pd_entry_t *)DMPDphys)[i] |=
835                             pmap_bits_default[PG_RW_IDX] |
836                             pmap_bits_default[PG_V_IDX] |
837                             pmap_bits_default[PG_PS_IDX] |
838                             pmap_bits_default[PG_G_IDX] |
839                             pmap_bits_default[PG_M_IDX] |
840                             pmap_bits_default[PG_A_IDX];
841                 }
842
843                 /*
844                  * And the direct map space's PDP
845                  */
846                 for (i = 0; i < ndmpdp; i++) {
847                         ((pdp_entry_t *)DMPDPphys)[i] = DMPDphys +
848                                                         (i << PAGE_SHIFT);
849                         ((pdp_entry_t *)DMPDPphys)[i] |=
850                             pmap_bits_default[PG_RW_IDX] |
851                             pmap_bits_default[PG_V_IDX] |
852                             pmap_bits_default[PG_U_IDX];
853                 }
854         } else {
855                 for (i = 0; i < ndmpdp; i++) {
856                         ((pdp_entry_t *)DMPDPphys)[i] =
857                                                 (vm_paddr_t)i << PDPSHIFT;
858                         ((pdp_entry_t *)DMPDPphys)[i] |=
859                             pmap_bits_default[PG_RW_IDX] |
860                             pmap_bits_default[PG_V_IDX] |
861                             pmap_bits_default[PG_PS_IDX] |
862                             pmap_bits_default[PG_G_IDX] |
863                             pmap_bits_default[PG_M_IDX] |
864                             pmap_bits_default[PG_A_IDX];
865                 }
866         }
867
868         /* And recursively map PML4 to itself in order to get PTmap */
869         ((pdp_entry_t *)KPML4phys)[PML4PML4I] = KPML4phys;
870         ((pdp_entry_t *)KPML4phys)[PML4PML4I] |=
871             pmap_bits_default[PG_RW_IDX] |
872             pmap_bits_default[PG_V_IDX] |
873             pmap_bits_default[PG_U_IDX];
874
875         /*
876          * Connect the Direct Map slots up to the PML4
877          */
878         for (j = 0; j < NDMPML4E; ++j) {
879                 ((pdp_entry_t *)KPML4phys)[DMPML4I + j] =
880                     (DMPDPphys + ((vm_paddr_t)j << PML4SHIFT)) |
881                     pmap_bits_default[PG_RW_IDX] |
882                     pmap_bits_default[PG_V_IDX] |
883                     pmap_bits_default[PG_U_IDX];
884         }
885
886         /*
887          * Connect the KVA slot up to the PML4
888          */
889         ((pdp_entry_t *)KPML4phys)[KPML4I] = KPDPphys;
890         ((pdp_entry_t *)KPML4phys)[KPML4I] |=
891             pmap_bits_default[PG_RW_IDX] |
892             pmap_bits_default[PG_V_IDX] |
893             pmap_bits_default[PG_U_IDX];
894 }
895
896 /*
897  *      Bootstrap the system enough to run with virtual memory.
898  *
899  *      On the i386 this is called after mapping has already been enabled
900  *      and just syncs the pmap module with what has already been done.
901  *      [We can't call it easily with mapping off since the kernel is not
902  *      mapped with PA == VA, hence we would have to relocate every address
903  *      from the linked base (virtual) address "KERNBASE" to the actual
904  *      (physical) address starting relative to 0]
905  */
906 void
907 pmap_bootstrap(vm_paddr_t *firstaddr)
908 {
909         vm_offset_t va;
910         pt_entry_t *pte;
911         int i;
912
913         KvaStart = VM_MIN_KERNEL_ADDRESS;
914         KvaEnd = VM_MAX_KERNEL_ADDRESS;
915         KvaSize = KvaEnd - KvaStart;
916
917         avail_start = *firstaddr;
918
919         /*
920          * Create an initial set of page tables to run the kernel in.
921          */
922         create_pagetables(firstaddr);
923
924         virtual2_start = KvaStart;
925         virtual2_end = PTOV_OFFSET;
926
927         virtual_start = (vm_offset_t) PTOV_OFFSET + *firstaddr;
928         virtual_start = pmap_kmem_choose(virtual_start);
929
930         virtual_end = VM_MAX_KERNEL_ADDRESS;
931
932         /* XXX do %cr0 as well */
933         load_cr4(rcr4() | CR4_PGE | CR4_PSE);
934         load_cr3(KPML4phys);
935
936         /*
937          * Initialize protection array.
938          */
939         i386_protection_init();
940
941         /*
942          * The kernel's pmap is statically allocated so we don't have to use
943          * pmap_create, which is unlikely to work correctly at this part of
944          * the boot sequence (XXX and which no longer exists).
945          */
946         kernel_pmap.pm_pml4 = (pdp_entry_t *) (PTOV_OFFSET + KPML4phys);
947         kernel_pmap.pm_count = 1;
948         CPUMASK_ASSALLONES(kernel_pmap.pm_active);
949         RB_INIT(&kernel_pmap.pm_pvroot);
950         spin_init(&kernel_pmap.pm_spin, "pmapbootstrap");
951         for (i = 0; i < PM_PLACEMARKS; ++i)
952                 kernel_pmap.pm_placemarks[i] = PM_NOPLACEMARK;
953
954         /*
955          * Reserve some special page table entries/VA space for temporary
956          * mapping of pages.
957          */
958 #define SYSMAP(c, p, v, n)      \
959         v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n);
960
961         va = virtual_start;
962         pte = vtopte(va);
963
964         /*
965          * CMAP1/CMAP2 are used for zeroing and copying pages.
966          */
967         SYSMAP(caddr_t, CMAP1, CADDR1, 1)
968
969         /*
970          * Crashdump maps.
971          */
972         SYSMAP(caddr_t, pt_crashdumpmap, crashdumpmap, MAXDUMPPGS);
973
974         /*
975          * ptvmmap is used for reading arbitrary physical pages via
976          * /dev/mem.
977          */
978         SYSMAP(caddr_t, ptmmap, ptvmmap, 1)
979
980         /*
981          * msgbufp is used to map the system message buffer.
982          * XXX msgbufmap is not used.
983          */
984         SYSMAP(struct msgbuf *, msgbufmap, msgbufp,
985                atop(round_page(MSGBUF_SIZE)))
986
987         virtual_start = va;
988         virtual_start = pmap_kmem_choose(virtual_start);
989
990         *CMAP1 = 0;
991
992         /*
993          * PG_G is terribly broken on SMP because we IPI invltlb's in some
994          * cases rather then invl1pg.  Actually, I don't even know why it
995          * works under UP because self-referential page table mappings
996          */
997 //      pgeflag = 0;
998
999 /*
1000  * Initialize the 4MB page size flag
1001  */
1002 //      pseflag = 0;
1003 /*
1004  * The 4MB page version of the initial
1005  * kernel page mapping.
1006  */
1007         pdir4mb = 0;
1008
1009 #if !defined(DISABLE_PSE)
1010         if (cpu_feature & CPUID_PSE) {
1011                 pt_entry_t ptditmp;
1012                 /*
1013                  * Note that we have enabled PSE mode
1014                  */
1015 //              pseflag = kernel_pmap.pmap_bits[PG_PS_IDX];
1016                 ptditmp = *(PTmap + x86_64_btop(KERNBASE));
1017                 ptditmp &= ~(NBPDR - 1);
1018                 ptditmp |= pmap_bits_default[PG_V_IDX] |
1019                     pmap_bits_default[PG_RW_IDX] |
1020                     pmap_bits_default[PG_PS_IDX] |
1021                     pmap_bits_default[PG_U_IDX];
1022 //                  pgeflag;
1023                 pdir4mb = ptditmp;
1024         }
1025 #endif
1026         cpu_invltlb();
1027
1028         /* Initialize the PAT MSR */
1029         pmap_init_pat();
1030         pmap_pinit_defaults(&kernel_pmap);
1031
1032         TUNABLE_INT_FETCH("machdep.pmap_fast_kernel_cpusync",
1033                           &pmap_fast_kernel_cpusync);
1034
1035 }
1036
1037 /*
1038  * Setup the PAT MSR.
1039  */
1040 void
1041 pmap_init_pat(void)
1042 {
1043         uint64_t pat_msr;
1044         u_long cr0, cr4;
1045
1046         /*
1047          * Default values mapping PATi,PCD,PWT bits at system reset.
1048          * The default values effectively ignore the PATi bit by
1049          * repeating the encodings for 0-3 in 4-7, and map the PCD
1050          * and PWT bit combinations to the expected PAT types.
1051          */
1052         pat_msr = PAT_VALUE(0, PAT_WRITE_BACK) |        /* 000 */
1053                   PAT_VALUE(1, PAT_WRITE_THROUGH) |     /* 001 */
1054                   PAT_VALUE(2, PAT_UNCACHED) |          /* 010 */
1055                   PAT_VALUE(3, PAT_UNCACHEABLE) |       /* 011 */
1056                   PAT_VALUE(4, PAT_WRITE_BACK) |        /* 100 */
1057                   PAT_VALUE(5, PAT_WRITE_THROUGH) |     /* 101 */
1058                   PAT_VALUE(6, PAT_UNCACHED) |          /* 110 */
1059                   PAT_VALUE(7, PAT_UNCACHEABLE);        /* 111 */
1060         pat_pte_index[PAT_WRITE_BACK]   = 0;
1061         pat_pte_index[PAT_WRITE_THROUGH]= 0         | X86_PG_NC_PWT;
1062         pat_pte_index[PAT_UNCACHED]     = X86_PG_NC_PCD;
1063         pat_pte_index[PAT_UNCACHEABLE]  = X86_PG_NC_PCD | X86_PG_NC_PWT;
1064         pat_pte_index[PAT_WRITE_PROTECTED] = pat_pte_index[PAT_UNCACHEABLE];
1065         pat_pte_index[PAT_WRITE_COMBINING] = pat_pte_index[PAT_UNCACHEABLE];
1066
1067         if (cpu_feature & CPUID_PAT) {
1068                 /*
1069                  * If we support the PAT then set-up entries for
1070                  * WRITE_PROTECTED and WRITE_COMBINING using bit patterns
1071                  * 4 and 5.
1072                  */
1073                 pat_msr = (pat_msr & ~PAT_MASK(4)) |
1074                           PAT_VALUE(4, PAT_WRITE_PROTECTED);
1075                 pat_msr = (pat_msr & ~PAT_MASK(5)) |
1076                           PAT_VALUE(5, PAT_WRITE_COMBINING);
1077                 pat_pte_index[PAT_WRITE_PROTECTED] = X86_PG_PTE_PAT | 0;
1078                 pat_pte_index[PAT_WRITE_COMBINING] = X86_PG_PTE_PAT | X86_PG_NC_PWT;
1079
1080                 /*
1081                  * Then enable the PAT
1082                  */
1083
1084                 /* Disable PGE. */
1085                 cr4 = rcr4();
1086                 load_cr4(cr4 & ~CR4_PGE);
1087
1088                 /* Disable caches (CD = 1, NW = 0). */
1089                 cr0 = rcr0();
1090                 load_cr0((cr0 & ~CR0_NW) | CR0_CD);
1091
1092                 /* Flushes caches and TLBs. */
1093                 wbinvd();
1094                 cpu_invltlb();
1095
1096                 /* Update PAT and index table. */
1097                 wrmsr(MSR_PAT, pat_msr);
1098
1099                 /* Flush caches and TLBs again. */
1100                 wbinvd();
1101                 cpu_invltlb();
1102
1103                 /* Restore caches and PGE. */
1104                 load_cr0(cr0);
1105                 load_cr4(cr4);
1106                 PatMsr = pat_msr;
1107         }
1108 }
1109
1110 /*
1111  * Set 4mb pdir for mp startup
1112  */
1113 void
1114 pmap_set_opt(void)
1115 {
1116         if (cpu_feature & CPUID_PSE) {
1117                 load_cr4(rcr4() | CR4_PSE);
1118                 if (pdir4mb && mycpu->gd_cpuid == 0) {  /* only on BSP */
1119                         cpu_invltlb();
1120                 }
1121         }
1122 }
1123
1124 /*
1125  *      Initialize the pmap module.
1126  *      Called by vm_init, to initialize any structures that the pmap
1127  *      system needs to map virtual memory.
1128  *      pmap_init has been enhanced to support in a fairly consistant
1129  *      way, discontiguous physical memory.
1130  */
1131 void
1132 pmap_init(void)
1133 {
1134         int i;
1135         int initial_pvs;
1136
1137         /*
1138          * Allocate memory for random pmap data structures.  Includes the
1139          * pv_head_table.
1140          */
1141
1142         for (i = 0; i < vm_page_array_size; i++) {
1143                 vm_page_t m;
1144
1145                 m = &vm_page_array[i];
1146                 TAILQ_INIT(&m->md.pv_list);
1147         }
1148
1149         /*
1150          * init the pv free list
1151          */
1152         initial_pvs = vm_page_array_size;
1153         if (initial_pvs < MINPV)
1154                 initial_pvs = MINPV;
1155         pvzone = &pvzone_store;
1156         pvinit = (void *)kmem_alloc(&kernel_map,
1157                                     initial_pvs * sizeof (struct pv_entry),
1158                                     VM_SUBSYS_PVENTRY);
1159         zbootinit(pvzone, "PV ENTRY", sizeof (struct pv_entry),
1160                   pvinit, initial_pvs);
1161
1162         /*
1163          * Now it is safe to enable pv_table recording.
1164          */
1165         pmap_initialized = TRUE;
1166 }
1167
1168 /*
1169  * Initialize the address space (zone) for the pv_entries.  Set a
1170  * high water mark so that the system can recover from excessive
1171  * numbers of pv entries.
1172  */
1173 void
1174 pmap_init2(void)
1175 {
1176         int shpgperproc = PMAP_SHPGPERPROC;
1177         int entry_max;
1178
1179         TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
1180         pv_entry_max = shpgperproc * maxproc + vm_page_array_size;
1181         TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
1182         pv_entry_high_water = 9 * (pv_entry_max / 10);
1183
1184         /*
1185          * Subtract out pages already installed in the zone (hack)
1186          */
1187         entry_max = pv_entry_max - vm_page_array_size;
1188         if (entry_max <= 0)
1189                 entry_max = 1;
1190
1191         zinitna(pvzone, &pvzone_obj, NULL, 0, entry_max, ZONE_INTERRUPT);
1192
1193         /*
1194          * Enable dynamic deletion of empty higher-level page table pages
1195          * by default only if system memory is < 8GB (use 7GB for slop).
1196          * This can save a little memory, but imposes significant
1197          * performance overhead for things like bulk builds, and for programs
1198          * which do a lot of memory mapping and memory unmapping.
1199          */
1200         if (pmap_dynamic_delete < 0) {
1201                 if (vmstats.v_page_count < 7LL * 1024 * 1024 * 1024 / PAGE_SIZE)
1202                         pmap_dynamic_delete = 1;
1203                 else
1204                         pmap_dynamic_delete = 0;
1205         }
1206 }
1207
1208 /*
1209  * Typically used to initialize a fictitious page by vm/device_pager.c
1210  */
1211 void
1212 pmap_page_init(struct vm_page *m)
1213 {
1214         vm_page_init(m);
1215         TAILQ_INIT(&m->md.pv_list);
1216 }
1217
1218 /***************************************************
1219  * Low level helper routines.....
1220  ***************************************************/
1221
1222 /*
1223  * this routine defines the region(s) of memory that should
1224  * not be tested for the modified bit.
1225  */
1226 static __inline
1227 int
1228 pmap_track_modified(vm_pindex_t pindex)
1229 {
1230         vm_offset_t va = (vm_offset_t)pindex << PAGE_SHIFT;
1231         if ((va < clean_sva) || (va >= clean_eva)) 
1232                 return 1;
1233         else
1234                 return 0;
1235 }
1236
1237 /*
1238  * Extract the physical page address associated with the map/VA pair.
1239  * The page must be wired for this to work reliably.
1240  */
1241 vm_paddr_t 
1242 pmap_extract(pmap_t pmap, vm_offset_t va, void **handlep)
1243 {
1244         vm_paddr_t rtval;
1245         pv_entry_t pt_pv;
1246         pt_entry_t *ptep;
1247
1248         rtval = 0;
1249         if (va >= VM_MAX_USER_ADDRESS) {
1250                 /*
1251                  * Kernel page directories might be direct-mapped and
1252                  * there is typically no PV tracking of pte's
1253                  */
1254                 pd_entry_t *pt;
1255
1256                 pt = pmap_pt(pmap, va);
1257                 if (pt && (*pt & pmap->pmap_bits[PG_V_IDX])) {
1258                         if (*pt & pmap->pmap_bits[PG_PS_IDX]) {
1259                                 rtval = *pt & PG_PS_FRAME;
1260                                 rtval |= va & PDRMASK;
1261                         } else {
1262                                 ptep = pmap_pt_to_pte(*pt, va);
1263                                 if (*pt & pmap->pmap_bits[PG_V_IDX]) {
1264                                         rtval = *ptep & PG_FRAME;
1265                                         rtval |= va & PAGE_MASK;
1266                                 }
1267                         }
1268                 }
1269                 if (handlep)
1270                         *handlep = NULL;
1271         } else {
1272                 /*
1273                  * User pages currently do not direct-map the page directory
1274                  * and some pages might not used managed PVs.  But all PT's
1275                  * will have a PV.
1276                  */
1277                 pt_pv = pv_get(pmap, pmap_pt_pindex(va), NULL);
1278                 if (pt_pv) {
1279                         ptep = pv_pte_lookup(pt_pv, pmap_pte_index(va));
1280                         if (*ptep & pmap->pmap_bits[PG_V_IDX]) {
1281                                 rtval = *ptep & PG_FRAME;
1282                                 rtval |= va & PAGE_MASK;
1283                         }
1284                         if (handlep)
1285                                 *handlep = pt_pv;       /* locked until done */
1286                         else
1287                                 pv_put (pt_pv);
1288                 } else if (handlep) {
1289                         *handlep = NULL;
1290                 }
1291         }
1292         return rtval;
1293 }
1294
1295 void
1296 pmap_extract_done(void *handle)
1297 {
1298         if (handle)
1299                 pv_put((pv_entry_t)handle);
1300 }
1301
1302 /*
1303  * Similar to extract but checks protections, SMP-friendly short-cut for
1304  * vm_fault_page[_quick]().  Can return NULL to cause the caller to
1305  * fall-through to the real fault code.
1306  *
1307  * The returned page, if not NULL, is held (and not busied).
1308  */
1309 vm_page_t
1310 pmap_fault_page_quick(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
1311 {
1312         if (pmap && va < VM_MAX_USER_ADDRESS) {
1313                 pv_entry_t pt_pv;
1314                 pv_entry_t pte_pv;
1315                 pt_entry_t *ptep;
1316                 pt_entry_t req;
1317                 vm_page_t m;
1318                 int error;
1319
1320                 req = pmap->pmap_bits[PG_V_IDX] |
1321                       pmap->pmap_bits[PG_U_IDX];
1322                 if (prot & VM_PROT_WRITE)
1323                         req |= pmap->pmap_bits[PG_RW_IDX];
1324
1325                 pt_pv = pv_get(pmap, pmap_pt_pindex(va), NULL);
1326                 if (pt_pv == NULL)
1327                         return (NULL);
1328                 ptep = pv_pte_lookup(pt_pv, pmap_pte_index(va));
1329                 if ((*ptep & req) != req) {
1330                         pv_put(pt_pv);
1331                         return (NULL);
1332                 }
1333                 pte_pv = pv_get_try(pmap, pmap_pte_pindex(va), NULL, &error);
1334                 if (pte_pv && error == 0) {
1335                         m = pte_pv->pv_m;
1336                         vm_page_hold(m);
1337                         if (prot & VM_PROT_WRITE)
1338                                 vm_page_dirty(m);
1339                         pv_put(pte_pv);
1340                 } else if (pte_pv) {
1341                         pv_drop(pte_pv);
1342                         m = NULL;
1343                 } else {
1344                         /* error can be 0 or 1 */
1345                         m = NULL;
1346                 }
1347                 pv_put(pt_pv);
1348                 return(m);
1349         } else {
1350                 return(NULL);
1351         }
1352 }
1353
1354 /*
1355  * Extract the physical page address associated kernel virtual address.
1356  */
1357 vm_paddr_t
1358 pmap_kextract(vm_offset_t va)
1359 {
1360         pd_entry_t pt;          /* pt entry in pd */
1361         vm_paddr_t pa;
1362
1363         if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
1364                 pa = DMAP_TO_PHYS(va);
1365         } else {
1366                 pt = *vtopt(va);
1367                 if (pt & kernel_pmap.pmap_bits[PG_PS_IDX]) {
1368                         pa = (pt & PG_PS_FRAME) | (va & PDRMASK);
1369                 } else {
1370                         /*
1371                          * Beware of a concurrent promotion that changes the
1372                          * PDE at this point!  For example, vtopte() must not
1373                          * be used to access the PTE because it would use the
1374                          * new PDE.  It is, however, safe to use the old PDE
1375                          * because the page table page is preserved by the
1376                          * promotion.
1377                          */
1378                         pa = *pmap_pt_to_pte(pt, va);
1379                         pa = (pa & PG_FRAME) | (va & PAGE_MASK);
1380                 }
1381         }
1382         return pa;
1383 }
1384
1385 /***************************************************
1386  * Low level mapping routines.....
1387  ***************************************************/
1388
1389 /*
1390  * Routine: pmap_kenter
1391  * Function:
1392  *      Add a wired page to the KVA
1393  *      NOTE! note that in order for the mapping to take effect -- you
1394  *      should do an invltlb after doing the pmap_kenter().
1395  */
1396 void 
1397 pmap_kenter(vm_offset_t va, vm_paddr_t pa)
1398 {
1399         pt_entry_t *ptep;
1400         pt_entry_t npte;
1401
1402         npte = pa |
1403             kernel_pmap.pmap_bits[PG_RW_IDX] |
1404             kernel_pmap.pmap_bits[PG_V_IDX];
1405 //          pgeflag;
1406         ptep = vtopte(va);
1407 #if 1
1408         pmap_inval_smp(&kernel_pmap, va, 1, ptep, npte);
1409 #else
1410         /* FUTURE */
1411         if (*ptep)
1412                 pmap_inval_smp(&kernel_pmap, va, ptep, npte);
1413         else
1414                 *ptep = npte;
1415 #endif
1416 }
1417
1418 /*
1419  * Similar to pmap_kenter(), except we only invalidate the mapping on the
1420  * current CPU.  Returns 0 if the previous pte was 0, 1 if it wasn't
1421  * (caller can conditionalize calling smp_invltlb()).
1422  */
1423 int
1424 pmap_kenter_quick(vm_offset_t va, vm_paddr_t pa)
1425 {
1426         pt_entry_t *ptep;
1427         pt_entry_t npte;
1428         int res;
1429
1430         npte = pa | kernel_pmap.pmap_bits[PG_RW_IDX] |
1431                     kernel_pmap.pmap_bits[PG_V_IDX];
1432         // npte |= pgeflag;
1433         ptep = vtopte(va);
1434 #if 1
1435         res = 1;
1436 #else
1437         /* FUTURE */
1438         res = (*ptep != 0);
1439 #endif
1440         atomic_swap_long(ptep, npte);
1441         cpu_invlpg((void *)va);
1442
1443         return res;
1444 }
1445
1446 /*
1447  * Enter addresses into the kernel pmap but don't bother
1448  * doing any tlb invalidations.  Caller will do a rollup
1449  * invalidation via pmap_rollup_inval().
1450  */
1451 int
1452 pmap_kenter_noinval(vm_offset_t va, vm_paddr_t pa)
1453 {
1454         pt_entry_t *ptep;
1455         pt_entry_t npte;
1456         int res;
1457
1458         npte = pa |
1459             kernel_pmap.pmap_bits[PG_RW_IDX] |
1460             kernel_pmap.pmap_bits[PG_V_IDX];
1461 //          pgeflag;
1462         ptep = vtopte(va);
1463 #if 1
1464         res = 1;
1465 #else
1466         /* FUTURE */
1467         res = (*ptep != 0);
1468 #endif
1469         atomic_swap_long(ptep, npte);
1470         cpu_invlpg((void *)va);
1471
1472         return res;
1473 }
1474
1475 /*
1476  * remove a page from the kernel pagetables
1477  */
1478 void
1479 pmap_kremove(vm_offset_t va)
1480 {
1481         pt_entry_t *ptep;
1482
1483         ptep = vtopte(va);
1484         pmap_inval_smp(&kernel_pmap, va, 1, ptep, 0);
1485 }
1486
1487 void
1488 pmap_kremove_quick(vm_offset_t va)
1489 {
1490         pt_entry_t *ptep;
1491
1492         ptep = vtopte(va);
1493         (void)pte_load_clear(ptep);
1494         cpu_invlpg((void *)va);
1495 }
1496
1497 /*
1498  * Remove addresses from the kernel pmap but don't bother
1499  * doing any tlb invalidations.  Caller will do a rollup
1500  * invalidation via pmap_rollup_inval().
1501  */
1502 void
1503 pmap_kremove_noinval(vm_offset_t va)
1504 {
1505         pt_entry_t *ptep;
1506
1507         ptep = vtopte(va);
1508         (void)pte_load_clear(ptep);
1509 }
1510
1511 /*
1512  * XXX these need to be recoded.  They are not used in any critical path.
1513  */
1514 void
1515 pmap_kmodify_rw(vm_offset_t va)
1516 {
1517         atomic_set_long(vtopte(va), kernel_pmap.pmap_bits[PG_RW_IDX]);
1518         cpu_invlpg((void *)va);
1519 }
1520
1521 /* NOT USED
1522 void
1523 pmap_kmodify_nc(vm_offset_t va)
1524 {
1525         atomic_set_long(vtopte(va), PG_N);
1526         cpu_invlpg((void *)va);
1527 }
1528 */
1529
1530 /*
1531  * Used to map a range of physical addresses into kernel virtual
1532  * address space during the low level boot, typically to map the
1533  * dump bitmap, message buffer, and vm_page_array.
1534  *
1535  * These mappings are typically made at some pointer after the end of the
1536  * kernel text+data.
1537  *
1538  * We could return PHYS_TO_DMAP(start) here and not allocate any
1539  * via (*virtp), but then kmem from userland and kernel dumps won't
1540  * have access to the related pointers.
1541  */
1542 vm_offset_t
1543 pmap_map(vm_offset_t *virtp, vm_paddr_t start, vm_paddr_t end, int prot)
1544 {
1545         vm_offset_t va;
1546         vm_offset_t va_start;
1547
1548         /*return PHYS_TO_DMAP(start);*/
1549
1550         va_start = *virtp;
1551         va = va_start;
1552
1553         while (start < end) {
1554                 pmap_kenter_quick(va, start);
1555                 va += PAGE_SIZE;
1556                 start += PAGE_SIZE;
1557         }
1558         *virtp = va;
1559         return va_start;
1560 }
1561
1562 #define PMAP_CLFLUSH_THRESHOLD  (2 * 1024 * 1024)
1563
1564 /*
1565  * Remove the specified set of pages from the data and instruction caches.
1566  *
1567  * In contrast to pmap_invalidate_cache_range(), this function does not
1568  * rely on the CPU's self-snoop feature, because it is intended for use
1569  * when moving pages into a different cache domain.
1570  */
1571 void
1572 pmap_invalidate_cache_pages(vm_page_t *pages, int count)
1573 {
1574         vm_offset_t daddr, eva;
1575         int i;
1576
1577         if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE ||
1578             (cpu_feature & CPUID_CLFSH) == 0)
1579                 wbinvd();
1580         else {
1581                 cpu_mfence();
1582                 for (i = 0; i < count; i++) {
1583                         daddr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pages[i]));
1584                         eva = daddr + PAGE_SIZE;
1585                         for (; daddr < eva; daddr += cpu_clflush_line_size)
1586                                 clflush(daddr);
1587                 }
1588                 cpu_mfence();
1589         }
1590 }
1591
1592 void
1593 pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
1594 {
1595         KASSERT((sva & PAGE_MASK) == 0,
1596             ("pmap_invalidate_cache_range: sva not page-aligned"));
1597         KASSERT((eva & PAGE_MASK) == 0,
1598             ("pmap_invalidate_cache_range: eva not page-aligned"));
1599
1600         if (cpu_feature & CPUID_SS) {
1601                 ; /* If "Self Snoop" is supported, do nothing. */
1602         } else {
1603                 /* Globally invalidate caches */
1604                 cpu_wbinvd_on_all_cpus();
1605         }
1606 }
1607
1608 /*
1609  * Invalidate the specified range of virtual memory on all cpus associated
1610  * with the pmap.
1611  */
1612 void
1613 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
1614 {
1615         pmap_inval_smp(pmap, sva, (eva - sva) >> PAGE_SHIFT, NULL, 0);
1616 }
1617
1618 /*
1619  * Add a list of wired pages to the kva.  This routine is used for temporary
1620  * kernel mappings such as those found in buffer cache buffer.  Page
1621  * modifications and accesses are not tracked or recorded.
1622  *
1623  * NOTE! Old mappings are simply overwritten, and we cannot assume relaxed
1624  *       semantics as previous mappings may have been zerod without any
1625  *       invalidation.
1626  *
1627  * The page *must* be wired.
1628  */
1629 void
1630 pmap_qenter(vm_offset_t beg_va, vm_page_t *m, int count)
1631 {
1632         vm_offset_t end_va;
1633         vm_offset_t va;
1634
1635         end_va = beg_va + count * PAGE_SIZE;
1636
1637         for (va = beg_va; va < end_va; va += PAGE_SIZE) {
1638                 pt_entry_t pte;
1639                 pt_entry_t *ptep;
1640
1641                 ptep = vtopte(va);
1642                 pte = VM_PAGE_TO_PHYS(*m) |
1643                         kernel_pmap.pmap_bits[PG_RW_IDX] |
1644                         kernel_pmap.pmap_bits[PG_V_IDX] |
1645                         kernel_pmap.pmap_cache_bits[(*m)->pat_mode];
1646 //              pgeflag;
1647                 atomic_swap_long(ptep, pte);
1648                 m++;
1649         }
1650         pmap_invalidate_range(&kernel_pmap, beg_va, end_va);
1651 }
1652
1653 /*
1654  * This routine jerks page mappings from the kernel -- it is meant only
1655  * for temporary mappings such as those found in buffer cache buffers.
1656  * No recording modified or access status occurs.
1657  *
1658  * MPSAFE, INTERRUPT SAFE (cluster callback)
1659  */
1660 void
1661 pmap_qremove(vm_offset_t beg_va, int count)
1662 {
1663         vm_offset_t end_va;
1664         vm_offset_t va;
1665
1666         end_va = beg_va + count * PAGE_SIZE;
1667
1668         for (va = beg_va; va < end_va; va += PAGE_SIZE) {
1669                 pt_entry_t *pte;
1670
1671                 pte = vtopte(va);
1672                 (void)pte_load_clear(pte);
1673                 cpu_invlpg((void *)va);
1674         }
1675         pmap_invalidate_range(&kernel_pmap, beg_va, end_va);
1676 }
1677
1678 /*
1679  * This routine removes temporary kernel mappings, only invalidating them
1680  * on the current cpu.  It should only be used under carefully controlled
1681  * conditions.
1682  */
1683 void
1684 pmap_qremove_quick(vm_offset_t beg_va, int count)
1685 {
1686         vm_offset_t end_va;
1687         vm_offset_t va;
1688
1689         end_va = beg_va + count * PAGE_SIZE;
1690
1691         for (va = beg_va; va < end_va; va += PAGE_SIZE) {
1692                 pt_entry_t *pte;
1693
1694                 pte = vtopte(va);
1695                 (void)pte_load_clear(pte);
1696                 cpu_invlpg((void *)va);
1697         }
1698 }
1699
1700 /*
1701  * This routine removes temporary kernel mappings *without* invalidating
1702  * the TLB.  It can only be used on permanent kva reservations such as those
1703  * found in buffer cache buffers, under carefully controlled circumstances.
1704  *
1705  * NOTE: Repopulating these KVAs requires unconditional invalidation.
1706  *       (pmap_qenter() does unconditional invalidation).
1707  */
1708 void
1709 pmap_qremove_noinval(vm_offset_t beg_va, int count)
1710 {
1711         vm_offset_t end_va;
1712         vm_offset_t va;
1713
1714         end_va = beg_va + count * PAGE_SIZE;
1715
1716         for (va = beg_va; va < end_va; va += PAGE_SIZE) {
1717                 pt_entry_t *pte;
1718
1719                 pte = vtopte(va);
1720                 (void)pte_load_clear(pte);
1721         }
1722 }
1723
1724 /*
1725  * Create a new thread and optionally associate it with a (new) process.
1726  * NOTE! the new thread's cpu may not equal the current cpu.
1727  */
1728 void
1729 pmap_init_thread(thread_t td)
1730 {
1731         /* enforce pcb placement & alignment */
1732         td->td_pcb = (struct pcb *)(td->td_kstack + td->td_kstack_size) - 1;
1733         td->td_pcb = (struct pcb *)((intptr_t)td->td_pcb & ~(intptr_t)0xF);
1734         td->td_savefpu = &td->td_pcb->pcb_save;
1735         td->td_sp = (char *)td->td_pcb; /* no -16 */
1736 }
1737
1738 /*
1739  * This routine directly affects the fork perf for a process.
1740  */
1741 void
1742 pmap_init_proc(struct proc *p)
1743 {
1744 }
1745
1746 static void
1747 pmap_pinit_defaults(struct pmap *pmap)
1748 {
1749         bcopy(pmap_bits_default, pmap->pmap_bits,
1750               sizeof(pmap_bits_default));
1751         bcopy(protection_codes, pmap->protection_codes,
1752               sizeof(protection_codes));
1753         bcopy(pat_pte_index, pmap->pmap_cache_bits,
1754               sizeof(pat_pte_index));
1755         pmap->pmap_cache_mask = X86_PG_NC_PWT | X86_PG_NC_PCD | X86_PG_PTE_PAT;
1756         pmap->copyinstr = std_copyinstr;
1757         pmap->copyin = std_copyin;
1758         pmap->copyout = std_copyout;
1759         pmap->fubyte = std_fubyte;
1760         pmap->subyte = std_subyte;
1761         pmap->fuword = std_fuword;
1762         pmap->suword = std_suword;
1763         pmap->suword32 = std_suword32;
1764 }
1765 /*
1766  * Initialize pmap0/vmspace0.
1767  *
1768  * On architectures where the kernel pmap is not integrated into the user
1769  * process pmap, this pmap represents the process pmap, not the kernel pmap.
1770  * kernel_pmap should be used to directly access the kernel_pmap.
1771  */
1772 void
1773 pmap_pinit0(struct pmap *pmap)
1774 {
1775         int i;
1776
1777         pmap->pm_pml4 = (pml4_entry_t *)(PTOV_OFFSET + KPML4phys);
1778         pmap->pm_count = 1;
1779         CPUMASK_ASSZERO(pmap->pm_active);
1780         pmap->pm_pvhint = NULL;
1781         RB_INIT(&pmap->pm_pvroot);
1782         spin_init(&pmap->pm_spin, "pmapinit0");
1783         for (i = 0; i < PM_PLACEMARKS; ++i)
1784                 pmap->pm_placemarks[i] = PM_NOPLACEMARK;
1785         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
1786         pmap_pinit_defaults(pmap);
1787 }
1788
1789 /*
1790  * Initialize a preallocated and zeroed pmap structure,
1791  * such as one in a vmspace structure.
1792  */
1793 static void
1794 pmap_pinit_simple(struct pmap *pmap)
1795 {
1796         int i;
1797
1798         /*
1799          * Misc initialization
1800          */
1801         pmap->pm_count = 1;
1802         CPUMASK_ASSZERO(pmap->pm_active);
1803         pmap->pm_pvhint = NULL;
1804         pmap->pm_flags = PMAP_FLAG_SIMPLE;
1805
1806         pmap_pinit_defaults(pmap);
1807
1808         /*
1809          * Don't blow up locks/tokens on re-use (XXX fix/use drop code
1810          * for this).
1811          */
1812         if (pmap->pm_pmlpv == NULL) {
1813                 RB_INIT(&pmap->pm_pvroot);
1814                 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
1815                 spin_init(&pmap->pm_spin, "pmapinitsimple");
1816                 for (i = 0; i < PM_PLACEMARKS; ++i)
1817                         pmap->pm_placemarks[i] = PM_NOPLACEMARK;
1818         }
1819 }
1820
1821 void
1822 pmap_pinit(struct pmap *pmap)
1823 {
1824         pv_entry_t pv;
1825         int j;
1826
1827         if (pmap->pm_pmlpv) {
1828                 if (pmap->pmap_bits[TYPE_IDX] != REGULAR_PMAP) {
1829                         pmap_puninit(pmap);
1830                 }
1831         }
1832
1833         pmap_pinit_simple(pmap);
1834         pmap->pm_flags &= ~PMAP_FLAG_SIMPLE;
1835
1836         /*
1837          * No need to allocate page table space yet but we do need a valid
1838          * page directory table.
1839          */
1840         if (pmap->pm_pml4 == NULL) {
1841                 pmap->pm_pml4 =
1842                     (pml4_entry_t *)kmem_alloc_pageable(&kernel_map,
1843                                                         PAGE_SIZE,
1844                                                         VM_SUBSYS_PML4);
1845         }
1846
1847         /*
1848          * Allocate the page directory page, which wires it even though
1849          * it isn't being entered into some higher level page table (it
1850          * being the highest level).  If one is already cached we don't
1851          * have to do anything.
1852          */
1853         if ((pv = pmap->pm_pmlpv) == NULL) {
1854                 pv = pmap_allocpte(pmap, pmap_pml4_pindex(), NULL);
1855                 pmap->pm_pmlpv = pv;
1856                 pmap_kenter((vm_offset_t)pmap->pm_pml4,
1857                             VM_PAGE_TO_PHYS(pv->pv_m));
1858                 pv_put(pv);
1859
1860                 /*
1861                  * Install DMAP and KMAP.
1862                  */
1863                 for (j = 0; j < NDMPML4E; ++j) {
1864                         pmap->pm_pml4[DMPML4I + j] =
1865                             (DMPDPphys + ((vm_paddr_t)j << PML4SHIFT)) |
1866                             pmap->pmap_bits[PG_RW_IDX] |
1867                             pmap->pmap_bits[PG_V_IDX] |
1868                             pmap->pmap_bits[PG_U_IDX];
1869                 }
1870                 pmap->pm_pml4[KPML4I] = KPDPphys |
1871                     pmap->pmap_bits[PG_RW_IDX] |
1872                     pmap->pmap_bits[PG_V_IDX] |
1873                     pmap->pmap_bits[PG_U_IDX];
1874
1875                 /*
1876                  * install self-referential address mapping entry
1877                  */
1878                 pmap->pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pv->pv_m) |
1879                     pmap->pmap_bits[PG_V_IDX] |
1880                     pmap->pmap_bits[PG_RW_IDX] |
1881                     pmap->pmap_bits[PG_A_IDX] |
1882                     pmap->pmap_bits[PG_M_IDX];
1883         } else {
1884                 KKASSERT(pv->pv_m->flags & PG_MAPPED);
1885                 KKASSERT(pv->pv_m->flags & PG_WRITEABLE);
1886         }
1887         KKASSERT(pmap->pm_pml4[255] == 0);
1888         KKASSERT(RB_ROOT(&pmap->pm_pvroot) == pv);
1889         KKASSERT(pv->pv_entry.rbe_left == NULL);
1890         KKASSERT(pv->pv_entry.rbe_right == NULL);
1891 }
1892
1893 /*
1894  * Clean up a pmap structure so it can be physically freed.  This routine
1895  * is called by the vmspace dtor function.  A great deal of pmap data is
1896  * left passively mapped to improve vmspace management so we have a bit
1897  * of cleanup work to do here.
1898  */
1899 void
1900 pmap_puninit(pmap_t pmap)
1901 {
1902         pv_entry_t pv;
1903         vm_page_t p;
1904
1905         KKASSERT(CPUMASK_TESTZERO(pmap->pm_active));
1906         if ((pv = pmap->pm_pmlpv) != NULL) {
1907                 if (pv_hold_try(pv) == 0)
1908                         pv_lock(pv);
1909                 KKASSERT(pv == pmap->pm_pmlpv);
1910                 p = pmap_remove_pv_page(pv);
1911                 pv_free(pv, NULL);
1912                 pv = NULL;      /* safety */
1913                 pmap_kremove((vm_offset_t)pmap->pm_pml4);
1914                 vm_page_busy_wait(p, FALSE, "pgpun");
1915                 KKASSERT(p->flags & (PG_FICTITIOUS|PG_UNMANAGED));
1916                 vm_page_unwire(p, 0);
1917                 vm_page_flag_clear(p, PG_MAPPED | PG_WRITEABLE);
1918
1919                 /*
1920                  * XXX eventually clean out PML4 static entries and
1921                  * use vm_page_free_zero()
1922                  */
1923                 vm_page_free(p);
1924                 pmap->pm_pmlpv = NULL;
1925         }
1926         if (pmap->pm_pml4) {
1927                 KKASSERT(pmap->pm_pml4 != (void *)(PTOV_OFFSET + KPML4phys));
1928                 kmem_free(&kernel_map, (vm_offset_t)pmap->pm_pml4, PAGE_SIZE);
1929                 pmap->pm_pml4 = NULL;
1930         }
1931         KKASSERT(pmap->pm_stats.resident_count == 0);
1932         KKASSERT(pmap->pm_stats.wired_count == 0);
1933 }
1934
1935 /*
1936  * This function is now unused (used to add the pmap to the pmap_list)
1937  */
1938 void
1939 pmap_pinit2(struct pmap *pmap)
1940 {
1941 }
1942
1943 /*
1944  * This routine is called when various levels in the page table need to
1945  * be populated.  This routine cannot fail.
1946  *
1947  * This function returns two locked pv_entry's, one representing the
1948  * requested pv and one representing the requested pv's parent pv.  If
1949  * an intermediate page table does not exist it will be created, mapped,
1950  * wired, and the parent page table will be given an additional hold
1951  * count representing the presence of the child pv_entry.
1952  */
1953 static
1954 pv_entry_t
1955 pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, pv_entry_t *pvpp)
1956 {
1957         pt_entry_t *ptep;
1958         pv_entry_t pv;
1959         pv_entry_t pvp;
1960         pt_entry_t v;
1961         vm_pindex_t pt_pindex;
1962         vm_page_t m;
1963         int isnew;
1964         int ispt;
1965
1966         /*
1967          * If the pv already exists and we aren't being asked for the
1968          * parent page table page we can just return it.  A locked+held pv
1969          * is returned.  The pv will also have a second hold related to the
1970          * pmap association that we don't have to worry about.
1971          */
1972         ispt = 0;
1973         pv = pv_alloc(pmap, ptepindex, &isnew);
1974         if (isnew == 0 && pvpp == NULL)
1975                 return(pv);
1976
1977         /*
1978          * Special case terminal PVs.  These are not page table pages so
1979          * no vm_page is allocated (the caller supplied the vm_page).  If
1980          * pvpp is non-NULL we are being asked to also removed the pt_pv
1981          * for this pv.
1982          *
1983          * Note that pt_pv's are only returned for user VAs. We assert that
1984          * a pt_pv is not being requested for kernel VAs.  The kernel
1985          * pre-wires all higher-level page tables so don't overload managed
1986          * higher-level page tables on top of it!
1987          */
1988         if (ptepindex < pmap_pt_pindex(0)) {
1989                 if (ptepindex >= NUPTE_USER) {
1990                         /* kernel manages this manually for KVM */
1991                         KKASSERT(pvpp == NULL);
1992                 } else {
1993                         KKASSERT(pvpp != NULL);
1994                         pt_pindex = NUPTE_TOTAL + (ptepindex >> NPTEPGSHIFT);
1995                         pvp = pmap_allocpte(pmap, pt_pindex, NULL);
1996                         if (isnew)
1997                                 vm_page_wire_quick(pvp->pv_m);
1998                         *pvpp = pvp;
1999                 }
2000                 return(pv);
2001         }
2002
2003         /*
2004          * The kernel never uses managed PT/PD/PDP pages.
2005          */
2006         KKASSERT(pmap != &kernel_pmap);
2007
2008         /*
2009          * Non-terminal PVs allocate a VM page to represent the page table,
2010          * so we have to resolve pvp and calculate ptepindex for the pvp
2011          * and then for the page table entry index in the pvp for
2012          * fall-through.
2013          */
2014         if (ptepindex < pmap_pd_pindex(0)) {
2015                 /*
2016                  * pv is PT, pvp is PD
2017                  */
2018                 ptepindex = (ptepindex - pmap_pt_pindex(0)) >> NPDEPGSHIFT;
2019                 ptepindex += NUPTE_TOTAL + NUPT_TOTAL;
2020                 pvp = pmap_allocpte(pmap, ptepindex, NULL);
2021
2022                 /*
2023                  * PT index in PD
2024                  */
2025                 ptepindex = pv->pv_pindex - pmap_pt_pindex(0);
2026                 ptepindex &= ((1ul << NPDEPGSHIFT) - 1);
2027                 ispt = 1;
2028         } else if (ptepindex < pmap_pdp_pindex(0)) {
2029                 /*
2030                  * pv is PD, pvp is PDP
2031                  *
2032                  * SIMPLE PMAP NOTE: Simple pmaps do not allocate above
2033                  *                   the PD.
2034                  */
2035                 ptepindex = (ptepindex - pmap_pd_pindex(0)) >> NPDPEPGSHIFT;
2036                 ptepindex += NUPTE_TOTAL + NUPT_TOTAL + NUPD_TOTAL;
2037
2038                 if (pmap->pm_flags & PMAP_FLAG_SIMPLE) {
2039                         KKASSERT(pvpp == NULL);
2040                         pvp = NULL;
2041                 } else {
2042                         pvp = pmap_allocpte(pmap, ptepindex, NULL);
2043                 }
2044
2045                 /*
2046                  * PD index in PDP
2047                  */
2048                 ptepindex = pv->pv_pindex - pmap_pd_pindex(0);
2049                 ptepindex &= ((1ul << NPDPEPGSHIFT) - 1);
2050         } else if (ptepindex < pmap_pml4_pindex()) {
2051                 /*
2052                  * pv is PDP, pvp is the root pml4 table
2053                  */
2054                 pvp = pmap_allocpte(pmap, pmap_pml4_pindex(), NULL);
2055
2056                 /*
2057                  * PDP index in PML4
2058                  */
2059                 ptepindex = pv->pv_pindex - pmap_pdp_pindex(0);
2060                 ptepindex &= ((1ul << NPML4EPGSHIFT) - 1);
2061         } else {
2062                 /*
2063                  * pv represents the top-level PML4, there is no parent.
2064                  */
2065                 pvp = NULL;
2066         }
2067
2068         if (isnew == 0)
2069                 goto notnew;
2070
2071         /*
2072          * (isnew) is TRUE, pv is not terminal.
2073          *
2074          * (1) Add a wire count to the parent page table (pvp).
2075          * (2) Allocate a VM page for the page table.
2076          * (3) Enter the VM page into the parent page table.
2077          *
2078          * page table pages are marked PG_WRITEABLE and PG_MAPPED.
2079          */
2080         if (pvp)
2081                 vm_page_wire_quick(pvp->pv_m);
2082
2083         for (;;) {
2084                 m = vm_page_alloc(NULL, pv->pv_pindex,
2085                                   VM_ALLOC_NORMAL | VM_ALLOC_SYSTEM |
2086                                   VM_ALLOC_INTERRUPT);
2087                 if (m)
2088                         break;
2089                 vm_wait(0);
2090         }
2091         vm_page_wire(m);        /* wire for mapping in parent */
2092         vm_page_unmanage(m);    /* m must be spinunlocked */
2093         pmap_zero_page(VM_PAGE_TO_PHYS(m));
2094         m->valid = VM_PAGE_BITS_ALL;
2095
2096         vm_page_spin_lock(m);
2097         pmap_page_stats_adding(m);
2098         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
2099         pv->pv_m = m;
2100         vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE);
2101         vm_page_spin_unlock(m);
2102
2103         /*
2104          * (isnew) is TRUE, pv is not terminal.
2105          *
2106          * Wire the page into pvp.  Bump the resident_count for the pmap.
2107          * There is no pvp for the top level, address the pm_pml4[] array
2108          * directly.
2109          *
2110          * If the caller wants the parent we return it, otherwise
2111          * we just put it away.
2112          *
2113          * No interlock is needed for pte 0 -> non-zero.
2114          *
2115          * In the situation where *ptep is valid we might have an unmanaged
2116          * page table page shared from another page table which we need to
2117          * unshare before installing our private page table page.
2118          */
2119         if (pvp) {
2120                 v = VM_PAGE_TO_PHYS(m) |
2121                     (pmap->pmap_bits[PG_U_IDX] |
2122                      pmap->pmap_bits[PG_RW_IDX] |
2123                      pmap->pmap_bits[PG_V_IDX] |
2124                      pmap->pmap_bits[PG_A_IDX] |
2125                      pmap->pmap_bits[PG_M_IDX]);
2126                 ptep = pv_pte_lookup(pvp, ptepindex);
2127                 if (*ptep & pmap->pmap_bits[PG_V_IDX]) {
2128                         pt_entry_t pte;
2129
2130                         if (ispt == 0) {
2131                                 panic("pmap_allocpte: unexpected pte %p/%d",
2132                                       pvp, (int)ptepindex);
2133                         }
2134                         pte = pmap_inval_smp(pmap, (vm_offset_t)-1, 1, ptep, v);
2135                         if (vm_page_unwire_quick(
2136                                         PHYS_TO_VM_PAGE(pte & PG_FRAME))) {
2137                                 panic("pmap_allocpte: shared pgtable "
2138                                       "pg bad wirecount");
2139                         }
2140                 } else {
2141                         pt_entry_t pte;
2142
2143                         pte = atomic_swap_long(ptep, v);
2144                         if (pte != 0) {
2145                                 kprintf("install pgtbl mixup 0x%016jx "
2146                                         "old/new 0x%016jx/0x%016jx\n",
2147                                         (intmax_t)ptepindex, pte, v);
2148                         }
2149                 }
2150         }
2151         vm_page_wakeup(m);
2152
2153         /*
2154          * (isnew) may be TRUE or FALSE, pv may or may not be terminal.
2155          */
2156 notnew:
2157         if (pvp) {
2158                 KKASSERT(pvp->pv_m != NULL);
2159                 ptep = pv_pte_lookup(pvp, ptepindex);
2160                 v = VM_PAGE_TO_PHYS(pv->pv_m) |
2161                     (pmap->pmap_bits[PG_U_IDX] |
2162                      pmap->pmap_bits[PG_RW_IDX] |
2163                      pmap->pmap_bits[PG_V_IDX] |
2164                      pmap->pmap_bits[PG_A_IDX] |
2165                      pmap->pmap_bits[PG_M_IDX]);
2166                 if (*ptep != v) {
2167                         kprintf("mismatched upper level pt %016jx/%016jx\n",
2168                                 *ptep, v);
2169                 }
2170         }
2171         if (pvpp)
2172                 *pvpp = pvp;
2173         else if (pvp)
2174                 pv_put(pvp);
2175         return (pv);
2176 }
2177
2178 /*
2179  * This version of pmap_allocpte() checks for possible segment optimizations
2180  * that would allow page-table sharing.  It can be called for terminal
2181  * page or page table page ptepindex's.
2182  *
2183  * The function is called with page table page ptepindex's for fictitious
2184  * and unmanaged terminal pages.  That is, we don't want to allocate a
2185  * terminal pv, we just want the pt_pv.  pvpp is usually passed as NULL
2186  * for this case.
2187  *
2188  * This function can return a pv and *pvpp associated with the passed in pmap
2189  * OR a pv and *pvpp associated with the shared pmap.  In the latter case
2190  * an unmanaged page table page will be entered into the pass in pmap.
2191  */
2192 static
2193 pv_entry_t
2194 pmap_allocpte_seg(pmap_t pmap, vm_pindex_t ptepindex, pv_entry_t *pvpp,
2195                   vm_map_entry_t entry, vm_offset_t va)
2196 {
2197         vm_object_t object;
2198         pmap_t obpmap;
2199         pmap_t *obpmapp;
2200         vm_offset_t b;
2201         pv_entry_t pte_pv;      /* in original or shared pmap */
2202         pv_entry_t pt_pv;       /* in original or shared pmap */
2203         pv_entry_t proc_pd_pv;  /* in original pmap */
2204         pv_entry_t proc_pt_pv;  /* in original pmap */
2205         pv_entry_t xpv;         /* PT in shared pmap */
2206         pd_entry_t *pt;         /* PT entry in PD of original pmap */
2207         pd_entry_t opte;        /* contents of *pt */
2208         pd_entry_t npte;        /* contents of *pt */
2209         vm_page_t m;
2210
2211 retry:
2212         /*
2213          * Basic tests, require a non-NULL vm_map_entry, require proper
2214          * alignment and type for the vm_map_entry, require that the
2215          * underlying object already be allocated.
2216          *
2217          * We allow almost any type of object to use this optimization.
2218          * The object itself does NOT have to be sized to a multiple of the
2219          * segment size, but the memory mapping does.
2220          *
2221          * XXX don't handle devices currently, because VM_PAGE_TO_PHYS()
2222          *     won't work as expected.
2223          */
2224         if (entry == NULL ||
2225             pmap_mmu_optimize == 0 ||                   /* not enabled */
2226             (pmap->pm_flags & PMAP_HVM) ||              /* special pmap */
2227             ptepindex >= pmap_pd_pindex(0) ||           /* not terminal or pt */
2228             entry->inheritance != VM_INHERIT_SHARE ||   /* not shared */
2229             entry->maptype != VM_MAPTYPE_NORMAL ||      /* weird map type */
2230             entry->object.vm_object == NULL ||          /* needs VM object */
2231             entry->object.vm_object->type == OBJT_DEVICE ||     /* ick */
2232             entry->object.vm_object->type == OBJT_MGTDEVICE ||  /* ick */
2233             (entry->offset & SEG_MASK) ||               /* must be aligned */
2234             (entry->start & SEG_MASK)) {
2235                 return(pmap_allocpte(pmap, ptepindex, pvpp));
2236         }
2237
2238         /*
2239          * Make sure the full segment can be represented.
2240          */
2241         b = va & ~(vm_offset_t)SEG_MASK;
2242         if (b < entry->start || b + SEG_SIZE > entry->end)
2243                 return(pmap_allocpte(pmap, ptepindex, pvpp));
2244
2245         /*
2246          * If the full segment can be represented dive the VM object's
2247          * shared pmap, allocating as required.
2248          */
2249         object = entry->object.vm_object;
2250
2251         if (entry->protection & VM_PROT_WRITE)
2252                 obpmapp = &object->md.pmap_rw;
2253         else
2254                 obpmapp = &object->md.pmap_ro;
2255
2256 #ifdef PMAP_DEBUG2
2257         if (pmap_enter_debug > 0) {
2258                 --pmap_enter_debug;
2259                 kprintf("pmap_allocpte_seg: va=%jx prot %08x o=%p "
2260                         "obpmapp %p %p\n",
2261                         va, entry->protection, object,
2262                         obpmapp, *obpmapp);
2263                 kprintf("pmap_allocpte_seg: entry %p %jx-%jx\n",
2264                         entry, entry->start, entry->end);
2265         }
2266 #endif
2267
2268         /*
2269          * We allocate what appears to be a normal pmap but because portions
2270          * of this pmap are shared with other unrelated pmaps we have to
2271          * set pm_active to point to all cpus.
2272          *
2273          * XXX Currently using pmap_spin to interlock the update, can't use
2274          *     vm_object_hold/drop because the token might already be held
2275          *     shared OR exclusive and we don't know.
2276          */
2277         while ((obpmap = *obpmapp) == NULL) {
2278                 obpmap = kmalloc(sizeof(*obpmap), M_OBJPMAP, M_WAITOK|M_ZERO);
2279                 pmap_pinit_simple(obpmap);
2280                 pmap_pinit2(obpmap);
2281                 spin_lock(&pmap_spin);
2282                 if (*obpmapp != NULL) {
2283                         /*
2284                          * Handle race
2285                          */
2286                         spin_unlock(&pmap_spin);
2287                         pmap_release(obpmap);
2288                         pmap_puninit(obpmap);
2289                         kfree(obpmap, M_OBJPMAP);
2290                         obpmap = *obpmapp; /* safety */
2291                 } else {
2292                         obpmap->pm_active = smp_active_mask;
2293                         obpmap->pm_flags |= PMAP_SEGSHARED;
2294                         *obpmapp = obpmap;
2295                         spin_unlock(&pmap_spin);
2296                 }
2297         }
2298
2299         /*
2300          * Layering is: PTE, PT, PD, PDP, PML4.  We have to return the
2301          * pte/pt using the shared pmap from the object but also adjust
2302          * the process pmap's page table page as a side effect.
2303          */
2304
2305         /*
2306          * Resolve the terminal PTE and PT in the shared pmap.  This is what
2307          * we will return.  This is true if ptepindex represents a terminal
2308          * page, otherwise pte_pv is actually the PT and pt_pv is actually
2309          * the PD.
2310          */
2311         pt_pv = NULL;
2312         pte_pv = pmap_allocpte(obpmap, ptepindex, &pt_pv);
2313         if (ptepindex >= pmap_pt_pindex(0))
2314                 xpv = pte_pv;
2315         else
2316                 xpv = pt_pv;
2317
2318         /*
2319          * Resolve the PD in the process pmap so we can properly share the
2320          * page table page.  Lock order is bottom-up (leaf first)!
2321          *
2322          * NOTE: proc_pt_pv can be NULL.
2323          */
2324         proc_pt_pv = pv_get(pmap, pmap_pt_pindex(b), NULL);
2325         proc_pd_pv = pmap_allocpte(pmap, pmap_pd_pindex(b), NULL);
2326 #ifdef PMAP_DEBUG2
2327         if (pmap_enter_debug > 0) {
2328                 --pmap_enter_debug;
2329                 kprintf("proc_pt_pv %p (wc %d) pd_pv %p va=%jx\n",
2330                         proc_pt_pv,
2331                         (proc_pt_pv ? proc_pt_pv->pv_m->wire_count : -1),
2332                         proc_pd_pv,
2333                         va);
2334         }
2335 #endif
2336
2337         /*
2338          * xpv is the page table page pv from the shared object
2339          * (for convenience), from above.
2340          *
2341          * Calculate the pte value for the PT to load into the process PD.
2342          * If we have to change it we must properly dispose of the previous
2343          * entry.
2344          */
2345         pt = pv_pte_lookup(proc_pd_pv, pmap_pt_index(b));
2346         npte = VM_PAGE_TO_PHYS(xpv->pv_m) |
2347                (pmap->pmap_bits[PG_U_IDX] |
2348                 pmap->pmap_bits[PG_RW_IDX] |
2349                 pmap->pmap_bits[PG_V_IDX] |
2350                 pmap->pmap_bits[PG_A_IDX] |
2351                 pmap->pmap_bits[PG_M_IDX]);
2352
2353         /*
2354          * Dispose of previous page table page if it was local to the
2355          * process pmap.  If the old pt is not empty we cannot dispose of it
2356          * until we clean it out.  This case should not arise very often so
2357          * it is not optimized.
2358          */
2359         if (proc_pt_pv) {
2360                 pmap_inval_bulk_t bulk;
2361
2362                 if (proc_pt_pv->pv_m->wire_count != 1) {
2363                         pv_put(proc_pd_pv);
2364                         pv_put(proc_pt_pv);
2365                         pv_put(pt_pv);
2366                         pv_put(pte_pv);
2367                         pmap_remove(pmap,
2368                                     va & ~(vm_offset_t)SEG_MASK,
2369                                     (va + SEG_SIZE) & ~(vm_offset_t)SEG_MASK);
2370                         goto retry;
2371                 }
2372
2373                 /*
2374                  * The release call will indirectly clean out *pt
2375                  */
2376                 pmap_inval_bulk_init(&bulk, proc_pt_pv->pv_pmap);
2377                 pmap_release_pv(proc_pt_pv, proc_pd_pv, &bulk);
2378                 pmap_inval_bulk_flush(&bulk);
2379                 proc_pt_pv = NULL;
2380                 /* relookup */
2381                 pt = pv_pte_lookup(proc_pd_pv, pmap_pt_index(b));
2382         }
2383
2384         /*
2385          * Handle remaining cases.
2386          */
2387         if (*pt == 0) {
2388                 atomic_swap_long(pt, npte);
2389                 vm_page_wire_quick(xpv->pv_m);          /* shared pt -> proc */
2390                 vm_page_wire_quick(proc_pd_pv->pv_m);   /* proc pd for sh pt */
2391                 atomic_add_long(&pmap->pm_stats.resident_count, 1);
2392         } else if (*pt != npte) {
2393                 opte = pmap_inval_smp(pmap, (vm_offset_t)-1, 1, pt, npte);
2394
2395 #if 0
2396                 opte = pte_load_clear(pt);
2397                 KKASSERT(opte && opte != npte);
2398
2399                 *pt = npte;
2400 #endif
2401                 vm_page_wire_quick(xpv->pv_m);          /* shared pt -> proc */
2402
2403                 /*
2404                  * Clean up opte, bump the wire_count for the process
2405                  * PD page representing the new entry if it was
2406                  * previously empty.
2407                  *
2408                  * If the entry was not previously empty and we have
2409                  * a PT in the proc pmap then opte must match that
2410                  * pt.  The proc pt must be retired (this is done
2411                  * later on in this procedure).
2412                  *
2413                  * NOTE: replacing valid pte, wire_count on proc_pd_pv
2414                  * stays the same.
2415                  */
2416                 KKASSERT(opte & pmap->pmap_bits[PG_V_IDX]);
2417                 m = PHYS_TO_VM_PAGE(opte & PG_FRAME);
2418                 if (vm_page_unwire_quick(m)) {
2419                         panic("pmap_allocpte_seg: "
2420                               "bad wire count %p",
2421                               m);
2422                 }
2423         }
2424
2425         /*
2426          * The existing process page table was replaced and must be destroyed
2427          * here.
2428          */
2429         if (proc_pd_pv)
2430                 pv_put(proc_pd_pv);
2431         if (pvpp)
2432                 *pvpp = pt_pv;
2433         else
2434                 pv_put(pt_pv);
2435
2436         return (pte_pv);
2437 }
2438
2439 /*
2440  * Release any resources held by the given physical map.
2441  *
2442  * Called when a pmap initialized by pmap_pinit is being released.  Should
2443  * only be called if the map contains no valid mappings.
2444  */
2445 struct pmap_release_info {
2446         pmap_t  pmap;
2447         int     retry;
2448         pv_entry_t pvp;
2449 };
2450
2451 static int pmap_release_callback(pv_entry_t pv, void *data);
2452
2453 void
2454 pmap_release(struct pmap *pmap)
2455 {
2456         struct pmap_release_info info;
2457
2458         KASSERT(CPUMASK_TESTZERO(pmap->pm_active),
2459                 ("pmap still active! %016jx",
2460                 (uintmax_t)CPUMASK_LOWMASK(pmap->pm_active)));
2461
2462         /*
2463          * There is no longer a pmap_list, if there were we would remove the
2464          * pmap from it here.
2465          */
2466
2467         /*
2468          * Pull pv's off the RB tree in order from low to high and release
2469          * each page.
2470          */
2471         info.pmap = pmap;
2472         do {
2473                 info.retry = 0;
2474                 info.pvp = NULL;
2475
2476                 spin_lock(&pmap->pm_spin);
2477                 RB_SCAN(pv_entry_rb_tree, &pmap->pm_pvroot, NULL,
2478                         pmap_release_callback, &info);
2479                 spin_unlock(&pmap->pm_spin);
2480
2481                 if (info.pvp)
2482                         pv_put(info.pvp);
2483         } while (info.retry);
2484
2485
2486         /*
2487          * One resident page (the pml4 page) should remain.
2488          * No wired pages should remain.
2489          */
2490 #if 1
2491         if (pmap->pm_stats.resident_count !=
2492             ((pmap->pm_flags & PMAP_FLAG_SIMPLE) ? 0 : 1) ||
2493             pmap->pm_stats.wired_count != 0) {
2494                 kprintf("fatal pmap problem - pmap %p flags %08x "
2495                         "rescnt=%jd wirecnt=%jd\n",
2496                         pmap,
2497                         pmap->pm_flags,
2498                         pmap->pm_stats.resident_count,
2499                         pmap->pm_stats.wired_count);
2500                 tsleep(pmap, 0, "DEAD", 0);
2501         }
2502 #else
2503         KKASSERT(pmap->pm_stats.resident_count ==
2504                  ((pmap->pm_flags & PMAP_FLAG_SIMPLE) ? 0 : 1));
2505         KKASSERT(pmap->pm_stats.wired_count == 0);
2506 #endif
2507 }
2508
2509 /*
2510  * Called from low to high.  We must cache the proper parent pv so we
2511  * can adjust its wired count.
2512  */
2513 static int
2514 pmap_release_callback(pv_entry_t pv, void *data)
2515 {
2516         struct pmap_release_info *info = data;
2517         pmap_t pmap = info->pmap;
2518         vm_pindex_t pindex;
2519         int r;
2520
2521         /*
2522          * Acquire a held and locked pv, check for release race
2523          */
2524         pindex = pv->pv_pindex;
2525         if (info->pvp == pv) {
2526                 spin_unlock(&pmap->pm_spin);
2527                 info->pvp = NULL;
2528         } else if (pv_hold_try(pv)) {
2529                 spin_unlock(&pmap->pm_spin);
2530         } else {
2531                 spin_unlock(&pmap->pm_spin);
2532                 pv_lock(pv);
2533         }
2534         if (pv->pv_pmap != pmap || pindex != pv->pv_pindex) {
2535                 pv_put(pv);
2536                 spin_lock(&pmap->pm_spin);
2537                 info->retry = 1;
2538                 return(-1);
2539         }
2540
2541         if (pv->pv_pindex < pmap_pt_pindex(0)) {
2542                 /*
2543                  * I am PTE, parent is PT
2544                  */
2545                 pindex = pv->pv_pindex >> NPTEPGSHIFT;
2546                 pindex += NUPTE_TOTAL;
2547         } else if (pv->pv_pindex < pmap_pd_pindex(0)) {
2548                 /*
2549                  * I am PT, parent is PD
2550                  */
2551                 pindex = (pv->pv_pindex - NUPTE_TOTAL) >> NPDEPGSHIFT;
2552                 pindex += NUPTE_TOTAL + NUPT_TOTAL;
2553         } else if (pv->pv_pindex < pmap_pdp_pindex(0)) {
2554                 /*
2555                  * I am PD, parent is PDP
2556                  */
2557                 pindex = (pv->pv_pindex - NUPTE_TOTAL - NUPT_TOTAL) >>
2558                          NPDPEPGSHIFT;
2559                 pindex += NUPTE_TOTAL + NUPT_TOTAL + NUPD_TOTAL;
2560         } else if (pv->pv_pindex < pmap_pml4_pindex()) {
2561                 /*
2562                  * I am PDP, parent is PML4 (there's only one)
2563                  */
2564 #if 0
2565                 pindex = (pv->pv_pindex - NUPTE_TOTAL - NUPT_TOTAL -
2566                            NUPD_TOTAL) >> NPML4EPGSHIFT;
2567                 pindex += NUPTE_TOTAL + NUPT_TOTAL + NUPD_TOTAL + NUPDP_TOTAL;
2568 #endif
2569                 pindex = pmap_pml4_pindex();
2570         } else {
2571                 /*
2572                  * parent is NULL
2573                  */
2574                 if (info->pvp) {
2575                         pv_put(info->pvp);
2576                         info->pvp = NULL;
2577                 }
2578                 pindex = 0;
2579         }
2580         if (pindex) {
2581                 if (info->pvp && info->pvp->pv_pindex != pindex) {
2582                         pv_put(info->pvp);
2583                         info->pvp = NULL;
2584                 }
2585                 if (info->pvp == NULL)
2586                         info->pvp = pv_get(pmap, pindex, NULL);
2587         } else {
2588                 if (info->pvp) {
2589                         pv_put(info->pvp);
2590                         info->pvp = NULL;
2591                 }
2592         }
2593         r = pmap_release_pv(pv, info->pvp, NULL);
2594         spin_lock(&pmap->pm_spin);
2595
2596         return(r);
2597 }
2598
2599 /*
2600  * Called with held (i.e. also locked) pv.  This function will dispose of
2601  * the lock along with the pv.
2602  *
2603  * If the caller already holds the locked parent page table for pv it
2604  * must pass it as pvp, allowing us to avoid a deadlock, else it can
2605  * pass NULL for pvp.
2606  */
2607 static int
2608 pmap_release_pv(pv_entry_t pv, pv_entry_t pvp, pmap_inval_bulk_t *bulk)
2609 {
2610         vm_page_t p;
2611
2612         /*
2613          * The pmap is currently not spinlocked, pv is held+locked.
2614          * Remove the pv's page from its parent's page table.  The
2615          * parent's page table page's wire_count will be decremented.
2616          *
2617          * This will clean out the pte at any level of the page table.
2618          * If smp != 0 all cpus are affected.
2619          *
2620          * Do not tear-down recursively, its faster to just let the
2621          * release run its course.
2622          */
2623         pmap_remove_pv_pte(pv, pvp, bulk, 0);
2624
2625         /*
2626          * Terminal pvs are unhooked from their vm_pages.  Because
2627          * terminal pages aren't page table pages they aren't wired
2628          * by us, so we have to be sure not to unwire them either.
2629          */
2630         if (pv->pv_pindex < pmap_pt_pindex(0)) {
2631                 pmap_remove_pv_page(pv);
2632                 goto skip;
2633         }
2634
2635         /*
2636          * We leave the top-level page table page cached, wired, and
2637          * mapped in the pmap until the dtor function (pmap_puninit())
2638          * gets called.
2639          *
2640          * Since we are leaving the top-level pv intact we need
2641          * to break out of what would otherwise be an infinite loop.
2642          */
2643         if (pv->pv_pindex == pmap_pml4_pindex()) {
2644                 pv_put(pv);
2645                 return(-1);
2646         }
2647
2648         /*
2649          * For page table pages (other than the top-level page),
2650          * remove and free the vm_page.  The representitive mapping
2651          * removed above by pmap_remove_pv_pte() did not undo the
2652          * last wire_count so we have to do that as well.
2653          */
2654         p = pmap_remove_pv_page(pv);
2655         vm_page_busy_wait(p, FALSE, "pmaprl");
2656         if (p->wire_count != 1) {
2657                 kprintf("p->wire_count was %016lx %d\n",
2658                         pv->pv_pindex, p->wire_count);
2659         }
2660         KKASSERT(p->wire_count == 1);
2661         KKASSERT(p->flags & PG_UNMANAGED);
2662
2663         vm_page_unwire(p, 0);
2664         KKASSERT(p->wire_count == 0);
2665
2666         vm_page_free(p);
2667 skip:
2668         pv_free(pv, pvp);
2669
2670         return 0;
2671 }
2672
2673 /*
2674  * This function will remove the pte associated with a pv from its parent.
2675  * Terminal pv's are supported.  All cpus specified by (bulk) are properly
2676  * invalidated.
2677  *
2678  * The wire count will be dropped on the parent page table.  The wire
2679  * count on the page being removed (pv->pv_m) from the parent page table
2680  * is NOT touched.  Note that terminal pages will not have any additional
2681  * wire counts while page table pages will have at least one representing
2682  * the mapping, plus others representing sub-mappings.
2683  *
2684  * NOTE: Cannot be called on kernel page table pages, only KVM terminal
2685  *       pages and user page table and terminal pages.
2686  *
2687  * The pv must be locked.  The pvp, if supplied, must be locked.  All
2688  * supplied pv's will remain locked on return.
2689  *
2690  * XXX must lock parent pv's if they exist to remove pte XXX
2691  */
2692 static
2693 void
2694 pmap_remove_pv_pte(pv_entry_t pv, pv_entry_t pvp, pmap_inval_bulk_t *bulk,
2695                    int destroy)
2696 {
2697         vm_pindex_t ptepindex = pv->pv_pindex;
2698         pmap_t pmap = pv->pv_pmap;
2699         vm_page_t p;
2700         int gotpvp = 0;
2701
2702         KKASSERT(pmap);
2703
2704         if (ptepindex == pmap_pml4_pindex()) {
2705                 /*
2706                  * We are the top level PML4E table, there is no parent.
2707                  */
2708                 p = pmap->pm_pmlpv->pv_m;
2709         } else if (ptepindex >= pmap_pdp_pindex(0)) {
2710                 /*
2711                  * Remove a PDP page from the PML4E.  This can only occur
2712                  * with user page tables.  We do not have to lock the
2713                  * pml4 PV so just ignore pvp.
2714                  */
2715                 vm_pindex_t pml4_pindex;
2716                 vm_pindex_t pdp_index;
2717                 pml4_entry_t *pdp;
2718
2719                 pdp_index = ptepindex - pmap_pdp_pindex(0);
2720                 if (pvp == NULL) {
2721                         pml4_pindex = pmap_pml4_pindex();
2722                         pvp = pv_get(pv->pv_pmap, pml4_pindex, NULL);
2723                         KKASSERT(pvp);
2724                         gotpvp = 1;
2725                 }
2726
2727                 pdp = &pmap->pm_pml4[pdp_index & ((1ul << NPML4EPGSHIFT) - 1)];
2728                 KKASSERT((*pdp & pmap->pmap_bits[PG_V_IDX]) != 0);
2729                 p = PHYS_TO_VM_PAGE(*pdp & PG_FRAME);
2730                 pmap_inval_bulk(bulk, (vm_offset_t)-1, pdp, 0);
2731         } else if (ptepindex >= pmap_pd_pindex(0)) {
2732                 /*
2733                  * Remove a PD page from the PDP
2734                  *
2735                  * SIMPLE PMAP NOTE: Non-existant pvp's are ok in the case
2736                  *                   of a simple pmap because it stops at
2737                  *                   the PD page.
2738                  */
2739                 vm_pindex_t pdp_pindex;
2740                 vm_pindex_t pd_index;
2741                 pdp_entry_t *pd;
2742
2743                 pd_index = ptepindex - pmap_pd_pindex(0);
2744
2745                 if (pvp == NULL) {
2746                         pdp_pindex = NUPTE_TOTAL + NUPT_TOTAL + NUPD_TOTAL +
2747                                      (pd_index >> NPML4EPGSHIFT);
2748                         pvp = pv_get(pv->pv_pmap, pdp_pindex, NULL);
2749                         gotpvp = 1;
2750                 }
2751
2752                 if (pvp) {
2753                         pd = pv_pte_lookup(pvp, pd_index &
2754                                                 ((1ul << NPDPEPGSHIFT) - 1));
2755                         KKASSERT((*pd & pmap->pmap_bits[PG_V_IDX]) != 0);
2756                         p = PHYS_TO_VM_PAGE(*pd & PG_FRAME);
2757                         pmap_inval_bulk(bulk, (vm_offset_t)-1, pd, 0);
2758                 } else {
2759                         KKASSERT(pmap->pm_flags & PMAP_FLAG_SIMPLE);
2760                         p = pv->pv_m;           /* degenerate test later */
2761                 }
2762         } else if (ptepindex >= pmap_pt_pindex(0)) {
2763                 /*
2764                  *  Remove a PT page from the PD
2765                  */
2766                 vm_pindex_t pd_pindex;
2767                 vm_pindex_t pt_index;
2768                 pd_entry_t *pt;
2769
2770                 pt_index = ptepindex - pmap_pt_pindex(0);
2771
2772                 if (pvp == NULL) {
2773                         pd_pindex = NUPTE_TOTAL + NUPT_TOTAL +
2774                                     (pt_index >> NPDPEPGSHIFT);
2775                         pvp = pv_get(pv->pv_pmap, pd_pindex, NULL);
2776                         KKASSERT(pvp);
2777                         gotpvp = 1;
2778                 }
2779
2780                 pt = pv_pte_lookup(pvp, pt_index & ((1ul << NPDPEPGSHIFT) - 1));
2781 #if 0
2782                 KASSERT((*pt & pmap->pmap_bits[PG_V_IDX]) != 0,
2783                         ("*pt unexpectedly invalid %016jx "
2784                          "gotpvp=%d ptepindex=%ld ptindex=%ld pv=%p pvp=%p",
2785                         *pt, gotpvp, ptepindex, pt_index, pv, pvp));
2786                 p = PHYS_TO_VM_PAGE(*pt & PG_FRAME);
2787 #else
2788                 if ((*pt & pmap->pmap_bits[PG_V_IDX]) == 0) {
2789                         kprintf("*pt unexpectedly invalid %016jx "
2790                                 "gotpvp=%d ptepindex=%ld ptindex=%ld "
2791                                 "pv=%p pvp=%p\n",
2792                                 *pt, gotpvp, ptepindex, pt_index, pv, pvp);
2793                         tsleep(pt, 0, "DEAD", 0);
2794                         p = pv->pv_m;
2795                 } else {
2796                         p = PHYS_TO_VM_PAGE(*pt & PG_FRAME);
2797                 }
2798 #endif
2799                 pmap_inval_bulk(bulk, (vm_offset_t)-1, pt, 0);
2800         } else {
2801                 /*
2802                  * Remove a managed PTE from the PT page.  Userland pmaps
2803                  * manage PT/PD/PDP page tables pages but the kernel_pmap
2804                  * does not.
2805                  *
2806                  * NOTE: pv's must be locked bottom-up to avoid deadlocking.
2807                  *       pv is a pte_pv so we can safely lock pt_pv.
2808                  *
2809                  * NOTE: FICTITIOUS pages may have multiple physical mappings
2810                  *       so PHYS_TO_VM_PAGE() will not necessarily work for
2811                  *       terminal ptes.
2812                  */
2813                 vm_pindex_t pt_pindex;
2814                 pt_entry_t *ptep;
2815                 pt_entry_t pte;
2816                 vm_offset_t va;
2817
2818                 pt_pindex = ptepindex >> NPTEPGSHIFT;
2819                 va = (vm_offset_t)ptepindex << PAGE_SHIFT;
2820
2821                 if (ptepindex >= NUPTE_USER) {
2822                         ptep = vtopte(ptepindex << PAGE_SHIFT);
2823                         KKASSERT(pvp == NULL);
2824                         /* pvp remains NULL */
2825                 } else {
2826                         if (pvp == NULL) {
2827                                 pt_pindex = NUPTE_TOTAL +
2828                                             (ptepindex >> NPDPEPGSHIFT);
2829                                 pvp = pv_get(pv->pv_pmap, pt_pindex, NULL);
2830                                 KKASSERT(pvp);
2831                                 gotpvp = 1;
2832                         }
2833                         ptep = pv_pte_lookup(pvp, ptepindex &
2834                                                   ((1ul << NPDPEPGSHIFT) - 1));
2835                 }
2836                 pte = pmap_inval_bulk(bulk, va, ptep, 0);
2837                 if (bulk == NULL)               /* XXX */
2838                         cpu_invlpg((void *)va); /* XXX */
2839
2840                 /*
2841                  * Now update the vm_page_t
2842                  */
2843                 if ((pte & (pmap->pmap_bits[PG_MANAGED_IDX] |
2844                             pmap->pmap_bits[PG_V_IDX])) !=
2845                     (pmap->pmap_bits[PG_MANAGED_IDX] |
2846                      pmap->pmap_bits[PG_V_IDX])) {
2847                         kprintf("remove_pte badpte %016lx %016lx %d\n",
2848                                 pte, pv->pv_pindex,
2849                                 pv->pv_pindex < pmap_pt_pindex(0));
2850                 }
2851
2852                 /* PHYS_TO_VM_PAGE() will not work for FICTITIOUS pages */
2853                 /*KKASSERT((pte & (PG_MANAGED|PG_V)) == (PG_MANAGED|PG_V));*/
2854                 if (pte & pmap->pmap_bits[PG_DEVICE_IDX])
2855                         p = pv->pv_m;
2856                 else
2857                         p = PHYS_TO_VM_PAGE(pte & PG_FRAME);
2858                 /* p = pv->pv_m; */
2859
2860                 if (pte & pmap->pmap_bits[PG_M_IDX]) {
2861                         if (pmap_track_modified(ptepindex))
2862                                 vm_page_dirty(p);
2863                 }
2864                 if (pte & pmap->pmap_bits[PG_A_IDX]) {
2865                         vm_page_flag_set(p, PG_REFERENCED);
2866                 }
2867                 if (pte & pmap->pmap_bits[PG_W_IDX])
2868                         atomic_add_long(&pmap->pm_stats.wired_count, -1);
2869                 if (pte & pmap->pmap_bits[PG_G_IDX])
2870                         cpu_invlpg((void *)va);
2871         }
2872         KKASSERT(pv->pv_m == p);        /* XXX remove me later */
2873
2874         /*
2875          * If requested, scrap the underlying pv->pv_m and the underlying
2876          * pv.  If this is a page-table-page we must also free the page.
2877          *
2878          * pvp must be returned locked.
2879          */
2880         if (destroy == 1) {
2881                 /*
2882                  * page table page (PT, PD, PDP, PML4), caller was responsible
2883                  * for testing wired_count.
2884                  */
2885                 vm_page_t p;
2886
2887                 KKASSERT(pv->pv_m->wire_count == 1);
2888                 p = pmap_remove_pv_page(pv);
2889                 pv_free(pv, pvp);
2890                 pv = NULL;
2891
2892                 KKASSERT(p->flags & (PG_FICTITIOUS|PG_UNMANAGED));
2893                 vm_page_busy_wait(p, FALSE, "pgpun");
2894
2895 #if 0
2896                 if (ptepindex != pmap_pml4_pindex()) {
2897                         pt_entry_t *ptep;
2898                         int i;
2899
2900                         ptep = (void *)PHYS_TO_DMAP(p->phys_addr);
2901                         for (i = 0; i < 512; ++i) {
2902                                 if (ptep[i]) {
2903                                         kprintf("PGTBL %016jx@%d "
2904                                                 "not empty %016jx\n",
2905                                                 ptepindex, i, ptep[i]);
2906                                 }
2907                         }
2908                 }
2909 #endif
2910                 vm_page_unwire(p, 0);
2911                 vm_page_flag_clear(p, PG_MAPPED | PG_WRITEABLE);
2912                 vm_page_free(p);
2913         } else if (destroy == 2) {
2914                 /*
2915                  * Normal page, remove from pmap and leave the underlying
2916                  * page untouched.
2917                  */
2918                 pmap_remove_pv_page(pv);
2919                 pv_free(pv, pvp);
2920                 pv = NULL;              /* safety */
2921         }
2922
2923         /*
2924          * If we acquired pvp ourselves then we are responsible for
2925          * recursively deleting it.
2926          */
2927         if (pvp && gotpvp) {
2928                 /*
2929                  * Recursively destroy higher-level page tables.
2930                  *
2931                  * This is optional.  If we do not, they will still
2932                  * be destroyed when the process exits.
2933                  *
2934                  * NOTE: Do not destroy pv_entry's with extra hold refs,
2935                  *       a caller may have unlocked it and intends to
2936                  *       continue to use it.
2937                  */
2938                 if (pmap_dynamic_delete &&
2939                     pvp->pv_m &&
2940                     pvp->pv_m->wire_count == 1 &&
2941                     (pvp->pv_hold & PV_HOLD_MASK) == 2 &&
2942                     pvp->pv_pindex != pmap_pml4_pindex()) {
2943                         if (pmap_dynamic_delete == 2)
2944                                 kprintf("A %jd %08x\n", pvp->pv_pindex, pvp->pv_hold);
2945                         if (pmap != &kernel_pmap) {
2946                                 pmap_remove_pv_pte(pvp, NULL, bulk, 1);
2947                                 pvp = NULL;     /* safety */
2948                         } else {
2949                                 kprintf("Attempt to remove kernel_pmap pindex "
2950                                         "%jd\n", pvp->pv_pindex);
2951                                 pv_put(pvp);
2952                         }
2953                 } else {
2954                         pv_put(pvp);
2955                 }
2956         }
2957 }
2958
2959 /*
2960  * Remove the vm_page association to a pv.  The pv must be locked.
2961  */
2962 static
2963 vm_page_t
2964 pmap_remove_pv_page(pv_entry_t pv)
2965 {
2966         vm_page_t m;
2967
2968         m = pv->pv_m;
2969         vm_page_spin_lock(m);
2970         KKASSERT(m && m == pv->pv_m);
2971         pv->pv_m = NULL;
2972         TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
2973         pmap_page_stats_deleting(m);
2974         /*
2975         if (m->object)
2976                 atomic_add_int(&m->object->agg_pv_list_count, -1);
2977         */
2978         if (TAILQ_EMPTY(&m->md.pv_list))
2979                 vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE);
2980         vm_page_spin_unlock(m);
2981
2982         return(m);
2983 }
2984
2985 /*
2986  * Grow the number of kernel page table entries, if needed.
2987  *
2988  * This routine is always called to validate any address space
2989  * beyond KERNBASE (for kldloads).  kernel_vm_end only governs the address
2990  * space below KERNBASE.
2991  *
2992  * kernel_map must be locked exclusively by the caller.
2993  */
2994 void
2995 pmap_growkernel(vm_offset_t kstart, vm_offset_t kend)
2996 {
2997         vm_paddr_t paddr;
2998         vm_offset_t ptppaddr;
2999         vm_page_t nkpg;
3000         pd_entry_t *pt, newpt;
3001         pdp_entry_t newpd;
3002         int update_kernel_vm_end;
3003
3004         /*
3005          * bootstrap kernel_vm_end on first real VM use
3006          */
3007         if (kernel_vm_end == 0) {
3008                 kernel_vm_end = VM_MIN_KERNEL_ADDRESS;
3009                 nkpt = 0;
3010                 while ((*pmap_pt(&kernel_pmap, kernel_vm_end) & kernel_pmap.pmap_bits[PG_V_IDX]) != 0) {
3011                         kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) &
3012                                         ~(PAGE_SIZE * NPTEPG - 1);
3013                         nkpt++;
3014                         if (kernel_vm_end - 1 >= kernel_map.max_offset) {
3015                                 kernel_vm_end = kernel_map.max_offset;
3016                                 break;                       
3017                         }
3018                 }
3019         }
3020
3021         /*
3022          * Fill in the gaps.  kernel_vm_end is only adjusted for ranges
3023          * below KERNBASE.  Ranges above KERNBASE are kldloaded and we
3024          * do not want to force-fill 128G worth of page tables.
3025          */
3026         if (kstart < KERNBASE) {
3027                 if (kstart > kernel_vm_end)
3028                         kstart = kernel_vm_end;
3029                 KKASSERT(kend <= KERNBASE);
3030                 update_kernel_vm_end = 1;
3031         } else {
3032                 update_kernel_vm_end = 0;
3033         }
3034
3035         kstart = rounddown2(kstart, PAGE_SIZE * NPTEPG);
3036         kend = roundup2(kend, PAGE_SIZE * NPTEPG);
3037
3038         if (kend - 1 >= kernel_map.max_offset)
3039                 kend = kernel_map.max_offset;
3040
3041         while (kstart < kend) {
3042                 pt = pmap_pt(&kernel_pmap, kstart);
3043                 if (pt == NULL) {
3044                         /* We need a new PD entry */
3045                         nkpg = vm_page_alloc(NULL, mycpu->gd_rand_incr++,
3046                                              VM_ALLOC_NORMAL |
3047                                              VM_ALLOC_SYSTEM |
3048                                              VM_ALLOC_INTERRUPT);
3049                         if (nkpg == NULL) {
3050                                 panic("pmap_growkernel: no memory to grow "
3051                                       "kernel");
3052                         }
3053                         paddr = VM_PAGE_TO_PHYS(nkpg);
3054                         pmap_zero_page(paddr);
3055                         newpd = (pdp_entry_t)
3056                             (paddr |
3057                             kernel_pmap.pmap_bits[PG_V_IDX] |
3058                             kernel_pmap.pmap_bits[PG_RW_IDX] |
3059                             kernel_pmap.pmap_bits[PG_A_IDX] |
3060                             kernel_pmap.pmap_bits[PG_M_IDX]);
3061                         *pmap_pd(&kernel_pmap, kstart) = newpd;
3062                         continue; /* try again */
3063                 }
3064                 if ((*pt & kernel_pmap.pmap_bits[PG_V_IDX]) != 0) {
3065                         kstart = (kstart + PAGE_SIZE * NPTEPG) &
3066                                  ~(PAGE_SIZE * NPTEPG - 1);
3067                         if (kstart - 1 >= kernel_map.max_offset) {
3068                                 kstart = kernel_map.max_offset;
3069                                 break;                       
3070                         }
3071                         continue;
3072                 }
3073
3074                 /*
3075                  * We need a new PT
3076                  *
3077                  * This index is bogus, but out of the way
3078                  */
3079                 nkpg = vm_page_alloc(NULL, mycpu->gd_rand_incr++,
3080                                      VM_ALLOC_NORMAL |
3081                                      VM_ALLOC_SYSTEM |
3082                                      VM_ALLOC_INTERRUPT);
3083                 if (nkpg == NULL)
3084                         panic("pmap_growkernel: no memory to grow kernel");
3085
3086                 vm_page_wire(nkpg);
3087                 ptppaddr = VM_PAGE_TO_PHYS(nkpg);
3088                 pmap_zero_page(ptppaddr);
3089                 newpt = (pd_entry_t)(ptppaddr |
3090                                      kernel_pmap.pmap_bits[PG_V_IDX] |
3091                                      kernel_pmap.pmap_bits[PG_RW_IDX] |
3092                                      kernel_pmap.pmap_bits[PG_A_IDX] |
3093                                      kernel_pmap.pmap_bits[PG_M_IDX]);
3094                 atomic_swap_long(pmap_pt(&kernel_pmap, kstart), newpt);
3095
3096                 kstart = (kstart + PAGE_SIZE * NPTEPG) &
3097                           ~(PAGE_SIZE * NPTEPG - 1);
3098
3099                 if (kstart - 1 >= kernel_map.max_offset) {
3100                         kstart = kernel_map.max_offset;
3101                         break;                       
3102                 }
3103         }
3104
3105         /*
3106          * Only update kernel_vm_end for areas below KERNBASE.
3107          */
3108         if (update_kernel_vm_end && kernel_vm_end < kstart)
3109                 kernel_vm_end = kstart;
3110 }
3111
3112 /*
3113  *      Add a reference to the specified pmap.
3114  */
3115 void
3116 pmap_reference(pmap_t pmap)
3117 {
3118         if (pmap != NULL)
3119                 atomic_add_int(&pmap->pm_count, 1);
3120 }
3121
3122 /***************************************************
3123  * page management routines.
3124  ***************************************************/
3125
3126 /*
3127  * Hold a pv without locking it
3128  */
3129 static void
3130 pv_hold(pv_entry_t pv)
3131 {
3132         atomic_add_int(&pv->pv_hold, 1);
3133 }
3134
3135 /*
3136  * Hold a pv_entry, preventing its destruction.  TRUE is returned if the pv
3137  * was successfully locked, FALSE if it wasn't.  The caller must dispose of
3138  * the pv properly.
3139  *
3140  * Either the pmap->pm_spin or the related vm_page_spin (if traversing a
3141  * pv list via its page) must be held by the caller.
3142  */
3143 static int
3144 _pv_hold_try(pv_entry_t pv PMAP_DEBUG_DECL)
3145 {
3146         u_int count;
3147
3148         /*
3149          * Critical path shortcut expects pv to already have one ref
3150          * (for the pv->pv_pmap).
3151          */
3152         if (atomic_cmpset_int(&pv->pv_hold, 1, PV_HOLD_LOCKED | 2)) {
3153 #ifdef PMAP_DEBUG
3154                 pv->pv_func = func;
3155                 pv->pv_line = lineno;
3156 #endif
3157                 return TRUE;
3158         }
3159
3160         for (;;) {
3161                 count = pv->pv_hold;
3162                 cpu_ccfence();
3163                 if ((count & PV_HOLD_LOCKED) == 0) {
3164                         if (atomic_cmpset_int(&pv->pv_hold, count,
3165                                               (count + 1) | PV_HOLD_LOCKED)) {
3166 #ifdef PMAP_DEBUG
3167                                 pv->pv_func = func;
3168                                 pv->pv_line = lineno;
3169 #endif
3170                                 return TRUE;
3171                         }
3172                 } else {
3173                         if (atomic_cmpset_int(&pv->pv_hold, count, count + 1))
3174                                 return FALSE;
3175                 }
3176                 /* retry */
3177         }
3178 }
3179
3180 /*
3181  * Drop a previously held pv_entry which could not be locked, allowing its
3182  * destruction.
3183  *
3184  * Must not be called with a spinlock held as we might zfree() the pv if it
3185  * is no longer associated with a pmap and this was the last hold count.
3186  */
3187 static void
3188 pv_drop(pv_entry_t pv)
3189 {
3190         u_int count;
3191
3192         for (;;) {
3193                 count = pv->pv_hold;
3194                 cpu_ccfence();
3195                 KKASSERT((count & PV_HOLD_MASK) > 0);
3196                 KKASSERT((count & (PV_HOLD_LOCKED | PV_HOLD_MASK)) !=
3197                          (PV_HOLD_LOCKED | 1));
3198                 if (atomic_cmpset_int(&pv->pv_hold, count, count - 1)) {
3199                         if ((count & PV_HOLD_MASK) == 1) {
3200 #ifdef PMAP_DEBUG2
3201                                 if (pmap_enter_debug > 0) {
3202                                         --pmap_enter_debug;
3203                                         kprintf("pv_drop: free pv %p\n", pv);
3204                                 }
3205 #endif
3206                                 KKASSERT(count == 1);
3207                                 KKASSERT(pv->pv_pmap == NULL);
3208                                 zfree(pvzone, pv);
3209                         }
3210                         return;
3211                 }
3212                 /* retry */
3213         }
3214 }
3215
3216 /*
3217  * Find or allocate the requested PV entry, returning a locked, held pv.
3218  *
3219  * If (*isnew) is non-zero, the returned pv will have two hold counts, one
3220  * for the caller and one representing the pmap and vm_page association.
3221  *
3222  * If (*isnew) is zero, the returned pv will have only one hold count.
3223  *
3224  * Since both associations can only be adjusted while the pv is locked,
3225  * together they represent just one additional hold.
3226  */
3227 static
3228 pv_entry_t
3229 _pv_alloc(pmap_t pmap, vm_pindex_t pindex, int *isnew PMAP_DEBUG_DECL)
3230 {
3231         pv_entry_t pv;
3232         pv_entry_t pnew = NULL;
3233
3234         spin_lock(&pmap->pm_spin);
3235         for (;;) {
3236                 /*
3237                  * Shortcut cache
3238                  */
3239                 pv = pmap->pm_pvhint;
3240                 cpu_ccfence();
3241                 if (pv == NULL ||
3242                     pv->pv_pmap != pmap ||
3243                     pv->pv_pindex != pindex) {
3244                         pv = pv_entry_rb_tree_RB_LOOKUP(&pmap->pm_pvroot,
3245                                                         pindex);
3246                 }
3247                 if (pv == NULL) {
3248                         vm_pindex_t *pmark;
3249
3250                         /*
3251                          * We need to stage a new pv entry
3252                          */
3253                         if (pnew == NULL) {
3254                                 spin_unlock(&pmap->pm_spin);
3255                                 pnew = zalloc(pvzone);
3256                                 spin_lock(&pmap->pm_spin);
3257                                 continue;
3258                         }
3259
3260                         /*
3261                          * We need to block if someone is holding a
3262                          * placemarker.
3263                          */
3264                         pmark = pmap_placemarker_hash(pmap, pindex);
3265
3266                         if (((*pmark ^ pindex) & ~PM_PLACEMARK_WAKEUP) == 0) {
3267                                 atomic_set_long(pmark, PM_PLACEMARK_WAKEUP);
3268                                 ssleep(pmark, &pmap->pm_spin, 0, "pvplc", 0);
3269                                 continue;
3270                         }
3271
3272                         /*
3273                          * Setup the new entry
3274                          */
3275                         pnew->pv_pmap = pmap;
3276                         pnew->pv_pindex = pindex;
3277                         pnew->pv_hold = PV_HOLD_LOCKED | 2;
3278 #ifdef PMAP_DEBUG
3279                         pnew->pv_func = func;
3280                         pnew->pv_line = lineno;
3281 #endif
3282                         pv = pv_entry_rb_tree_RB_INSERT(&pmap->pm_pvroot, pnew);
3283                         atomic_add_long(&pmap->pm_stats.resident_count, 1);
3284                         spin_unlock(&pmap->pm_spin);
3285                         *isnew = 1;
3286
3287                         KKASSERT(pv == NULL);
3288
3289                         return(pnew);
3290                 }
3291
3292                 /*
3293                  * We have an entry, clean up any staged pv we had allocated,
3294                  * then block until we can lock the entry.
3295                  */
3296                 if (pnew) {
3297                         spin_unlock(&pmap->pm_spin);
3298                         zfree(pvzone, pnew);
3299                         pnew = NULL;
3300                         spin_lock(&pmap->pm_spin);
3301                         continue;
3302                 }
3303                 if (_pv_hold_try(pv PMAP_DEBUG_COPY)) {
3304                         spin_unlock(&pmap->pm_spin);
3305                 } else {
3306                         spin_unlock(&pmap->pm_spin);
3307                         _pv_lock(pv PMAP_DEBUG_COPY);
3308                 }
3309
3310                 /*
3311                  * Make sure the pv is still in good shape for return,
3312                  * otherwise retry.
3313                  */
3314                 if (pv->pv_pmap == pmap && pv->pv_pindex == pindex) {
3315                         *isnew = 0;
3316                         return(pv);
3317                 }
3318                 pv_put(pv);
3319                 spin_lock(&pmap->pm_spin);
3320         }
3321 }
3322
3323 /*
3324  * Find the requested PV entry, returning a locked+held pv or NULL
3325  */
3326 static
3327 pv_entry_t
3328 _pv_get(pmap_t pmap, vm_pindex_t pindex, vm_pindex_t **pmarkp PMAP_DEBUG_DECL)
3329 {
3330         pv_entry_t pv;
3331
3332         spin_lock(&pmap->pm_spin);
3333         for (;;) {
3334                 /*
3335                  * Shortcut cache
3336                  */
3337                 pv = pmap->pm_pvhint;
3338                 cpu_ccfence();
3339                 if (pv == NULL ||
3340                     pv->pv_pmap != pmap ||
3341                     pv->pv_pindex != pindex) {
3342                         pv = pv_entry_rb_tree_RB_LOOKUP(&pmap->pm_pvroot,
3343                                                         pindex);
3344                 }
3345                 if (pv == NULL) {
3346                         /*
3347                          * Block if there is a placemarker
3348                          */
3349                         vm_pindex_t *pmark;
3350
3351                         pmark = pmap_placemarker_hash(pmap, pindex);
3352
3353                         if ((pmarkp && *pmark != PM_NOPLACEMARK) ||
3354                             ((*pmark ^ pindex) & ~PM_PLACEMARK_WAKEUP) == 0) {
3355                                 atomic_set_long(pmark, PM_PLACEMARK_WAKEUP);
3356                                 ssleep(pmark, &pmap->pm_spin, 0, "pvpld", 0);
3357                                 continue;
3358                         }
3359                         if (pmarkp) {
3360                                 *pmark = pindex;
3361                                 *pmarkp = pmark;
3362                         }
3363                         spin_unlock(&pmap->pm_spin);
3364                         return NULL;
3365                 }
3366                 if (_pv_hold_try(pv PMAP_DEBUG_COPY)) {
3367                         spin_unlock(&pmap->pm_spin);
3368                 } else {
3369                         spin_unlock(&pmap->pm_spin);
3370                         _pv_lock(pv PMAP_DEBUG_COPY);
3371                 }
3372                 if (pv->pv_pmap == pmap && pv->pv_pindex == pindex) {
3373                         pv_cache(pv, pindex);
3374                         return(pv);
3375                 }
3376                 pv_put(pv);
3377                 spin_lock(&pmap->pm_spin);
3378         }
3379 }
3380
3381 /*
3382  * Lookup, hold, and attempt to lock (pmap,pindex).
3383  *
3384  * If the entry does not exist NULL is returned and *errorp is set to 0
3385  *
3386  * If the entry exists and could be successfully locked it is returned and
3387  * errorp is set to 0.
3388  *
3389  * If the entry exists but could NOT be successfully locked it is returned
3390  * held and *errorp is set to 1.
3391  *
3392  * If the entry is placemarked by someone else NULL is returned and *errorp
3393  * is set to 1.
3394  */
3395 static
3396 pv_entry_t
3397 pv_get_try(pmap_t pmap, vm_pindex_t pindex, vm_pindex_t **pmarkp, int *errorp)
3398 {
3399         pv_entry_t pv;
3400
3401         spin_lock_shared(&pmap->pm_spin);
3402
3403         pv = pmap->pm_pvhint;
3404         cpu_ccfence();
3405         if (pv == NULL ||
3406             pv->pv_pmap != pmap ||
3407             pv->pv_pindex != pindex) {
3408                 pv = pv_entry_rb_tree_RB_LOOKUP(&pmap->pm_pvroot, pindex);
3409         }
3410
3411         if (pv == NULL) {
3412                 vm_pindex_t *pmark;
3413
3414                 pmark = pmap_placemarker_hash(pmap, pindex);
3415
3416                 if (((*pmark ^ pindex) & ~PM_PLACEMARK_WAKEUP) == 0) {
3417                         *errorp = 1;
3418                 } else if (pmarkp &&
3419                            atomic_cmpset_long(pmark, PM_NOPLACEMARK, pindex)) {
3420                         *errorp = 0;
3421                 } else {
3422                         /*
3423                          * Can't set a placemark with a NULL pmarkp, return
3424                          * an error.
3425                          */
3426                         *errorp = 1;
3427                 }
3428                 if (pmarkp)
3429                         *pmarkp = pmark;
3430                 spin_unlock_shared(&pmap->pm_spin);
3431
3432                 return NULL;
3433         }
3434         if (pv_hold_try(pv)) {
3435                 pv_cache(pv, pindex);
3436                 spin_unlock_shared(&pmap->pm_spin);
3437                 *errorp = 0;
3438                 KKASSERT(pv->pv_pmap == pmap && pv->pv_pindex == pindex);
3439                 return(pv);     /* lock succeeded */
3440         }
3441         spin_unlock_shared(&pmap->pm_spin);
3442         *errorp = 1;
3443         return (pv);            /* lock failed */
3444 }
3445
3446 /*
3447  * Lock a held pv, keeping the hold count
3448  */
3449 static
3450 void
3451 _pv_lock(pv_entry_t pv PMAP_DEBUG_DECL)
3452 {
3453         u_int count;
3454
3455         for (;;) {
3456                 count = pv->pv_hold;
3457                 cpu_ccfence();
3458                 if ((count & PV_HOLD_LOCKED) == 0) {
3459                         if (atomic_cmpset_int(&pv->pv_hold, count,
3460                                               count | PV_HOLD_LOCKED)) {
3461 #ifdef PMAP_DEBUG
3462                                 pv->pv_func = func;
3463                                 pv->pv_line = lineno;
3464 #endif
3465                                 return;
3466                         }
3467                         continue;
3468                 }
3469                 tsleep_interlock(pv, 0);
3470                 if (atomic_cmpset_int(&pv->pv_hold, count,
3471                                       count | PV_HOLD_WAITING)) {
3472 #ifdef PMAP_DEBUG
3473                         kprintf("pv waiting on %s:%d\n",
3474                                         pv->pv_func, pv->pv_line);
3475 #endif
3476                         tsleep(pv, PINTERLOCKED, "pvwait", hz);
3477                 }
3478                 /* retry */
3479         }
3480 }
3481
3482 /*
3483  * Unlock a held and locked pv, keeping the hold count.
3484  */
3485 static
3486 void
3487 pv_unlock(pv_entry_t pv)
3488 {
3489         u_int count;
3490
3491         for (;;) {
3492                 count = pv->pv_hold;
3493                 cpu_ccfence();
3494                 KKASSERT((count & (PV_HOLD_LOCKED | PV_HOLD_MASK)) >=
3495                          (PV_HOLD_LOCKED | 1));
3496                 if (atomic_cmpset_int(&pv->pv_hold, count,
3497                                       count &
3498                                       ~(PV_HOLD_LOCKED | PV_HOLD_WAITING))) {
3499                         if (count & PV_HOLD_WAITING)
3500                                 wakeup(pv);
3501                         break;
3502                 }
3503         }
3504 }
3505
3506 /*
3507  * Unlock and drop a pv.  If the pv is no longer associated with a pmap
3508  * and the hold count drops to zero we will free it.
3509  *
3510  * Caller should not hold any spin locks.  We are protected from hold races
3511  * by virtue of holds only occuring only with a pmap_spin or vm_page_spin
3512  * lock held.  A pv cannot be located otherwise.
3513  */
3514 static
3515 void
3516 pv_put(pv_entry_t pv)
3517 {
3518 #ifdef PMAP_DEBUG2
3519         if (pmap_enter_debug > 0) {
3520                 --pmap_enter_debug;
3521                 kprintf("pv_put pv=%p hold=%08x\n", pv, pv->pv_hold);
3522         }
3523 #endif
3524
3525         /*
3526          * Fast - shortcut most common condition
3527          */
3528         if (atomic_cmpset_int(&pv->pv_hold, PV_HOLD_LOCKED | 2, 1))
3529                 return;
3530
3531         /*
3532          * Slow
3533          */
3534         pv_unlock(pv);
3535         pv_drop(pv);
3536 }
3537
3538 /*
3539  * Remove the pmap association from a pv, require that pv_m already be removed,
3540  * then unlock and drop the pv.  Any pte operations must have already been
3541  * completed.  This call may result in a last-drop which will physically free
3542  * the pv.
3543  *
3544  * Removing the pmap association entails an additional drop.
3545  *
3546  * pv must be exclusively locked on call and will be disposed of on return.
3547  */
3548 static
3549 void
3550 pv_free(pv_entry_t pv, pv_entry_t pvp)
3551 {
3552         pmap_t pmap;
3553
3554         KKASSERT(pv->pv_m == NULL);
3555         KKASSERT((pv->pv_hold & PV_HOLD_MASK) >= 2);
3556         if ((pmap = pv->pv_pmap) != NULL) {
3557                 spin_lock(&pmap->pm_spin);
3558                 KKASSERT(pv->pv_pmap == pmap);
3559                 if (pmap->pm_pvhint == pv)
3560                         pmap->pm_pvhint = NULL;
3561                 pv_entry_rb_tree_RB_REMOVE(&pmap->pm_pvroot, pv);
3562                 atomic_add_long(&pmap->pm_stats.resident_count, -1);
3563                 pv->pv_pmap = NULL;
3564                 pv->pv_pindex = 0;
3565                 spin_unlock(&pmap->pm_spin);
3566
3567                 /*
3568                  * Try to shortcut three atomic ops, otherwise fall through
3569                  * and do it normally.  Drop two refs and the lock all in
3570                  * one go.
3571                  */
3572                 if (atomic_cmpset_int(&pv->pv_hold, PV_HOLD_LOCKED | 2, 0)) {
3573 #ifdef PMAP_DEBUG2
3574                         if (pmap_enter_debug > 0) {
3575                                 --pmap_enter_debug;
3576                                 kprintf("pv_free: free pv %p\n", pv);
3577                         }
3578 #endif
3579                         zfree(pvzone, pv);
3580                         if (pvp)
3581                                 vm_page_unwire_quick(pvp->pv_m);
3582                         return;
3583                 }
3584                 pv_drop(pv);    /* ref for pv_pmap */
3585                 if (pvp)
3586                         vm_page_unwire_quick(pvp->pv_m);
3587         }
3588         pv_put(pv);
3589 }
3590
3591 /*
3592  * This routine is very drastic, but can save the system
3593  * in a pinch.
3594  */
3595 void
3596 pmap_collect(void)
3597 {
3598         int i;
3599         vm_page_t m;
3600         static int warningdone=0;
3601
3602         if (pmap_pagedaemon_waken == 0)
3603                 return;
3604         pmap_pagedaemon_waken = 0;
3605         if (warningdone < 5) {
3606                 kprintf("pmap_collect: collecting pv entries -- "
3607                         "suggest increasing PMAP_SHPGPERPROC\n");
3608                 warningdone++;
3609         }
3610
3611         for (i = 0; i < vm_page_array_size; i++) {
3612                 m = &vm_page_array[i];
3613                 if (m->wire_count || m->hold_count)
3614                         continue;
3615                 if (vm_page_busy_try(m, TRUE) == 0) {
3616                         if (m->wire_count == 0 && m->hold_count == 0) {
3617                                 pmap_remove_all(m);
3618                         }
3619                         vm_page_wakeup(m);
3620                 }
3621         }
3622 }
3623
3624 /*
3625  * Scan the pmap for active page table entries and issue a callback.
3626  * The callback must dispose of pte_pv, whos PTE entry is at *ptep in
3627  * its parent page table.
3628  *
3629  * pte_pv will be NULL if the page or page table is unmanaged.
3630  * pt_pv will point to the page table page containing the pte for the page.
3631  *
3632  * NOTE! If we come across an unmanaged page TABLE (verses an unmanaged page),
3633  *       we pass a NULL pte_pv and we pass a pt_pv pointing to the passed
3634  *       process pmap's PD and page to the callback function.  This can be
3635  *       confusing because the pt_pv is really a pd_pv, and the target page
3636  *       table page is simply aliased by the pmap and not owned by it.
3637  *
3638  * It is assumed that the start and end are properly rounded to the page size.
3639  *
3640  * It is assumed that PD pages and above are managed and thus in the RB tree,
3641  * allowing us to use RB_SCAN from the PD pages down for ranged scans.
3642  */
3643 struct pmap_scan_info {
3644         struct pmap *pmap;
3645         vm_offset_t sva;
3646         vm_offset_t eva;
3647         vm_pindex_t sva_pd_pindex;
3648         vm_pindex_t eva_pd_pindex;
3649         void (*func)(pmap_t, struct pmap_scan_info *,
3650                      pv_entry_t, vm_pindex_t *, pv_entry_t,
3651                      int, vm_offset_t,
3652                      pt_entry_t *, void *);
3653         void *arg;
3654         pmap_inval_bulk_t bulk_core;
3655         pmap_inval_bulk_t *bulk;
3656         int count;
3657         int stop;
3658 };
3659
3660 static int pmap_scan_cmp(pv_entry_t pv, void *data);
3661 static int pmap_scan_callback(pv_entry_t pv, void *data);
3662
3663 static void
3664 pmap_scan(struct pmap_scan_info *info, int smp_inval)
3665 {
3666         struct pmap *pmap = info->pmap;
3667         pv_entry_t pd_pv;       /* A page directory PV */
3668         pv_entry_t pt_pv;       /* A page table PV */
3669         pv_entry_t pte_pv;      /* A page table entry PV */
3670         vm_pindex_t *pte_placemark;
3671         vm_pindex_t *pt_placemark;
3672         pt_entry_t *ptep;
3673         pt_entry_t oldpte;
3674         struct pv_entry dummy_pv;
3675
3676         info->stop = 0;
3677         if (pmap == NULL)
3678                 return;
3679         if (smp_inval) {
3680                 info->bulk = &info->bulk_core;
3681                 pmap_inval_bulk_init(&info->bulk_core, pmap);
3682         } else {
3683                 info->bulk = NULL;
3684         }
3685
3686         /*
3687          * Hold the token for stability; if the pmap is empty we have nothing
3688          * to do.
3689          */
3690 #if 0
3691         if (pmap->pm_stats.resident_count == 0) {
3692                 return;
3693         }
3694 #endif
3695
3696         info->count = 0;
3697
3698         /*
3699          * Special handling for scanning one page, which is a very common
3700          * operation (it is?).
3701          *
3702          * NOTE: Locks must be ordered bottom-up. pte,pt,pd,pdp,pml4
3703          */
3704         if (info->sva + PAGE_SIZE == info->eva) {
3705                 if (info->sva >= VM_MAX_USER_ADDRESS) {
3706                         /*
3707                          * Kernel mappings do not track wire counts on
3708                          * page table pages and only maintain pd_pv and
3709                          * pte_pv levels so pmap_scan() works.
3710                          */
3711                         pt_pv = NULL;
3712                         pte_pv = pv_get(pmap, pmap_pte_pindex(info->sva),
3713                                         &pte_placemark);
3714                         ptep = vtopte(info->sva);
3715                 } else {
3716                         /*
3717                          * User pages which are unmanaged will not have a
3718                          * pte_pv.  User page table pages which are unmanaged
3719                          * (shared from elsewhere) will also not have a pt_pv.
3720                          * The func() callback will pass both pte_pv and pt_pv
3721                          * as NULL in that case.
3722                          *
3723                          * We hold pte_placemark across the operation for
3724                          * unmanaged pages.
3725                          *
3726                          * WARNING!  We must hold pt_placemark across the
3727                          *           *ptep test to prevent misintepreting
3728                          *           a non-zero *ptep as a shared page
3729                          *           table page.  Hold it across the function
3730                          *           callback as well for SMP safety.
3731                          */
3732                         pte_pv = pv_get(pmap, pmap_pte_pindex(info->sva),
3733                                         &pte_placemark);
3734                         pt_pv = pv_get(pmap, pmap_pt_pindex(info->sva),
3735                                         &pt_placemark);
3736                         if (pt_pv == NULL) {
3737                                 KKASSERT(pte_pv == NULL);
3738                                 pd_pv = pv_get(pmap,
3739                                                pmap_pd_pindex(info->sva),
3740                                                NULL);
3741                                 if (pd_pv) {
3742                                         ptep = pv_pte_lookup(pd_pv,
3743                                                     pmap_pt_index(info->sva));
3744                                         if (*ptep) {
3745                                                 info->func(pmap, info,
3746                                                      NULL, pt_placemark,
3747                                                      pd_pv, 1,
3748                                                      info->sva, ptep,
3749                                                      info->arg);
3750                                         } else {
3751                                                 pv_placemarker_wakeup(pmap,
3752                                                                   pt_placemark);
3753                                         }
3754                                         pv_put(pd_pv);
3755                                 } else {
3756                                         pv_placemarker_wakeup(pmap,
3757                                                               pt_placemark);
3758                                 }
3759                                 pv_placemarker_wakeup(pmap, pte_placemark);
3760                                 goto fast_skip;
3761                         }
3762                         ptep = pv_pte_lookup(pt_pv, pmap_pte_index(info->sva));
3763                 }
3764
3765                 /*
3766                  * NOTE: *ptep can't be ripped out from under us if we hold
3767                  *       pte_pv (or pte_placemark) locked, but bits can
3768                  *       change.
3769                  */
3770                 oldpte = *ptep;
3771                 cpu_ccfence();
3772                 if (oldpte == 0) {
3773                         KKASSERT(pte_pv == NULL);
3774                         pv_placemarker_wakeup(pmap, pte_placemark);
3775                 } else if (pte_pv) {
3776                         KASSERT((oldpte & (pmap->pmap_bits[PG_MANAGED_IDX] |
3777                                            pmap->pmap_bits[PG_V_IDX])) ==
3778                                 (pmap->pmap_bits[PG_MANAGED_IDX] |
3779                                  pmap->pmap_bits[PG_V_IDX]),
3780                             ("badA *ptep %016lx/%016lx sva %016lx pte_pv %p",
3781                             *ptep, oldpte, info->sva, pte_pv));
3782                         info->func(pmap, info, pte_pv, NULL, pt_pv, 0,
3783                                    info->sva, ptep, info->arg);
3784                 } else {
3785                         KASSERT((oldpte & (pmap->pmap_bits[PG_MANAGED_IDX] |
3786                                            pmap->pmap_bits[PG_V_IDX])) ==
3787                             pmap->pmap_bits[PG_V_IDX],
3788                             ("badB *ptep %016lx/%016lx sva %016lx pte_pv NULL",
3789                             *ptep, oldpte, info->sva));
3790                         info->func(pmap, info, NULL, pte_placemark, pt_pv, 0,
3791                                    info->sva, ptep, info->arg);
3792                 }
3793                 if (pt_pv)
3794                         pv_put(pt_pv);
3795 fast_skip:
3796                 pmap_inval_bulk_flush(info->bulk);
3797                 return;
3798         }
3799
3800         /*
3801          * Nominal scan case, RB_SCAN() for PD pages and iterate from
3802          * there.
3803          */
3804         info->sva_pd_pindex = pmap_pd_pindex(info->sva);
3805         info->eva_pd_pindex = pmap_pd_pindex(info->eva + NBPDP - 1);
3806
3807         if (info->sva >= VM_MAX_USER_ADDRESS) {
3808                 /*
3809                  * The kernel does not currently maintain any pv_entry's for
3810                  * higher-level page tables.
3811                  */
3812                 bzero(&dummy_pv, sizeof(dummy_pv));
3813                 dummy_pv.pv_pindex = info->sva_pd_pindex;
3814                 spin_lock(&pmap->pm_spin);
3815                 while (dummy_pv.pv_pindex < info->eva_pd_pindex) {
3816                         pmap_scan_callback(&dummy_pv, info);
3817                         ++dummy_pv.pv_pindex;
3818                 }
3819                 spin_unlock(&pmap->pm_spin);
3820         } else {
3821                 /*
3822                  * User page tables maintain local PML4, PDP, and PD
3823                  * pv_entry's at the very least.  PT pv's might be
3824                  * unmanaged and thus not exist.  PTE pv's might be
3825                  * unmanaged and thus not exist.
3826                  */
3827                 spin_lock(&pmap->pm_spin);
3828                 pv_entry_rb_tree_RB_SCAN(&pmap->pm_pvroot, pmap_scan_cmp,
3829                                          pmap_scan_callback, info);
3830                 spin_unlock(&pmap->pm_spin);
3831         }
3832         pmap_inval_bulk_flush(info->bulk);
3833 }
3834
3835 /*
3836  * WARNING! pmap->pm_spin held
3837  */
3838 static int
3839 pmap_scan_cmp(pv_entry_t pv, void *data)
3840 {
3841         struct pmap_scan_info *info = data;
3842         if (pv->pv_pindex < info->sva_pd_pindex)
3843                 return(-1);
3844         if (pv->pv_pindex >= info->eva_pd_pindex)
3845                 return(1);
3846         return(0);
3847 }
3848
3849 /*
3850  * pmap_scan() by PDs
3851  *
3852  * WARNING! pmap->pm_spin held
3853  */
3854 static int
3855 pmap_scan_callback(pv_entry_t pv, void *data)
3856 {
3857         struct pmap_scan_info *info = data;
3858         struct pmap *pmap = info->pmap;
3859         pv_entry_t pd_pv;       /* A page directory PV */
3860         pv_entry_t pt_pv;       /* A page table PV */
3861         vm_pindex_t *pt_placemark;
3862         pt_entry_t *ptep;
3863         pt_entry_t oldpte;
3864         vm_offset_t sva;
3865         vm_offset_t eva;
3866         vm_offset_t va_next;
3867         vm_pindex_t pd_pindex;
3868         int error;
3869
3870         /*
3871          * Stop if requested
3872          */
3873         if (info->stop)
3874                 return -1;
3875
3876         /*
3877          * Pull the PD pindex from the pv before releasing the spinlock.
3878          *
3879          * WARNING: pv is faked for kernel pmap scans.
3880          */
3881         pd_pindex = pv->pv_pindex;
3882         spin_unlock(&pmap->pm_spin);
3883         pv = NULL;      /* invalid after spinlock unlocked */
3884
3885         /*
3886          * Calculate the page range within the PD.  SIMPLE pmaps are
3887          * direct-mapped for the entire 2^64 address space.  Normal pmaps
3888          * reflect the user and kernel address space which requires
3889          * cannonicalization w/regards to converting pd_pindex's back
3890          * into addresses.
3891          */
3892         sva = (pd_pindex - pmap_pd_pindex(0)) << PDPSHIFT;
3893         if ((pmap->pm_flags & PMAP_FLAG_SIMPLE) == 0 &&
3894             (sva & PML4_SIGNMASK)) {
3895                 sva |= PML4_SIGNMASK;
3896         }
3897         eva = sva + NBPDP;      /* can overflow */
3898         if (sva < info->sva)
3899                 sva = info->sva;
3900         if (eva < info->sva || eva > info->eva)