2 * Copyright (c) 1991 Regents of the University of California.
3 * Copyright (c) 1994 John S. Dyson
4 * Copyright (c) 1994 David Greenman
5 * Copyright (c) 2003 Peter Wemm
6 * Copyright (c) 2005-2008 Alan L. Cox <alc@cs.rice.edu>
7 * Copyright (c) 2008, 2009 The DragonFly Project.
8 * Copyright (c) 2008, 2009 Jordan Gordeev.
9 * Copyright (c) 2011-2012 Matthew Dillon
10 * All rights reserved.
12 * This code is derived from software contributed to Berkeley by
13 * the Systems Programming Group of the University of Utah Computer
14 * Science Department and William Jolitz of UUNET Technologies Inc.
16 * Redistribution and use in source and binary forms, with or without
17 * modification, are permitted provided that the following conditions
19 * 1. Redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer.
21 * 2. Redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution.
24 * 3. All advertising materials mentioning features or use of this software
25 * must display the following acknowledgement:
26 * This product includes software developed by the University of
27 * California, Berkeley and its contributors.
28 * 4. Neither the name of the University nor the names of its contributors
29 * may be used to endorse or promote products derived from this software
30 * without specific prior written permission.
32 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
33 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
34 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
35 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
36 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
37 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
38 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
39 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
40 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
41 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
45 * Manage physical address maps for x86-64 systems.
49 #include "opt_disable_pse.h"
52 #include "opt_msgbuf.h"
54 #include <sys/param.h>
55 #include <sys/systm.h>
56 #include <sys/kernel.h>
58 #include <sys/msgbuf.h>
59 #include <sys/vmmeter.h>
63 #include <vm/vm_param.h>
64 #include <sys/sysctl.h>
66 #include <vm/vm_kern.h>
67 #include <vm/vm_page.h>
68 #include <vm/vm_map.h>
69 #include <vm/vm_object.h>
70 #include <vm/vm_extern.h>
71 #include <vm/vm_pageout.h>
72 #include <vm/vm_pager.h>
73 #include <vm/vm_zone.h>
76 #include <sys/thread2.h>
77 #include <sys/sysref2.h>
78 #include <sys/spinlock2.h>
79 #include <vm/vm_page2.h>
81 #include <machine/cputypes.h>
82 #include <machine/md_var.h>
83 #include <machine/specialreg.h>
84 #include <machine/smp.h>
85 #include <machine_base/apic/apicreg.h>
86 #include <machine/globaldata.h>
87 #include <machine/pmap.h>
88 #include <machine/pmap_inval.h>
89 #include <machine/inttypes.h>
93 #define PMAP_KEEP_PDIRS
94 #ifndef PMAP_SHPGPERPROC
95 #define PMAP_SHPGPERPROC 2000
98 #if defined(DIAGNOSTIC)
99 #define PMAP_DIAGNOSTIC
105 * pmap debugging will report who owns a pv lock when blocking.
109 #define PMAP_DEBUG_DECL ,const char *func, int lineno
110 #define PMAP_DEBUG_ARGS , __func__, __LINE__
111 #define PMAP_DEBUG_COPY , func, lineno
113 #define pv_get(pmap, pindex) _pv_get(pmap, pindex \
115 #define pv_lock(pv) _pv_lock(pv \
117 #define pv_hold_try(pv) _pv_hold_try(pv \
119 #define pv_alloc(pmap, pindex, isnewp) _pv_alloc(pmap, pindex, isnewp \
124 #define PMAP_DEBUG_DECL
125 #define PMAP_DEBUG_ARGS
126 #define PMAP_DEBUG_COPY
128 #define pv_get(pmap, pindex) _pv_get(pmap, pindex)
129 #define pv_lock(pv) _pv_lock(pv)
130 #define pv_hold_try(pv) _pv_hold_try(pv)
131 #define pv_alloc(pmap, pindex, isnewp) _pv_alloc(pmap, pindex, isnewp)
136 * Get PDEs and PTEs for user/kernel address space
138 #define pdir_pde(m, v) (m[(vm_offset_t)(v) >> PDRSHIFT])
140 #define pmap_pde_v(pte) ((*(pd_entry_t *)pte & PG_V) != 0)
141 #define pmap_pte_w(pte) ((*(pt_entry_t *)pte & PG_W) != 0)
142 #define pmap_pte_m(pte) ((*(pt_entry_t *)pte & PG_M) != 0)
143 #define pmap_pte_u(pte) ((*(pt_entry_t *)pte & PG_A) != 0)
144 #define pmap_pte_v(pte) ((*(pt_entry_t *)pte & PG_V) != 0)
147 * Given a map and a machine independent protection code,
148 * convert to a vax protection code.
150 #define pte_prot(m, p) \
151 (protection_codes[p & (VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE)])
152 static int protection_codes[8];
154 struct pmap kernel_pmap;
155 static TAILQ_HEAD(,pmap) pmap_list = TAILQ_HEAD_INITIALIZER(pmap_list);
157 MALLOC_DEFINE(M_OBJPMAP, "objpmap", "pmaps associated with VM objects");
159 vm_paddr_t avail_start; /* PA of first available physical page */
160 vm_paddr_t avail_end; /* PA of last available physical page */
161 vm_offset_t virtual2_start; /* cutout free area prior to kernel start */
162 vm_offset_t virtual2_end;
163 vm_offset_t virtual_start; /* VA of first avail page (after kernel bss) */
164 vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */
165 vm_offset_t KvaStart; /* VA start of KVA space */
166 vm_offset_t KvaEnd; /* VA end of KVA space (non-inclusive) */
167 vm_offset_t KvaSize; /* max size of kernel virtual address space */
168 static boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */
169 static int pgeflag; /* PG_G or-in */
170 static int pseflag; /* PG_PS or-in */
174 static vm_paddr_t dmaplimit;
176 vm_offset_t kernel_vm_end = VM_MIN_KERNEL_ADDRESS;
178 #define PAT_INDEX_SIZE 8
179 static pt_entry_t pat_pte_index[PAT_INDEX_SIZE]; /* PAT -> PG_ bits */
180 /*static pt_entry_t pat_pde_index[PAT_INDEX_SIZE];*/ /* PAT -> PG_ bits */
182 static uint64_t KPTbase;
183 static uint64_t KPTphys;
184 static uint64_t KPDphys; /* phys addr of kernel level 2 */
185 static uint64_t KPDbase; /* phys addr of kernel level 2 @ KERNBASE */
186 uint64_t KPDPphys; /* phys addr of kernel level 3 */
187 uint64_t KPML4phys; /* phys addr of kernel level 4 */
189 static uint64_t DMPDphys; /* phys addr of direct mapped level 2 */
190 static uint64_t DMPDPphys; /* phys addr of direct mapped level 3 */
193 * Data for the pv entry allocation mechanism
195 static vm_zone_t pvzone;
196 static struct vm_zone pvzone_store;
197 static struct vm_object pvzone_obj;
198 static int pv_entry_max=0, pv_entry_high_water=0;
199 static int pmap_pagedaemon_waken = 0;
200 static struct pv_entry *pvinit;
203 * All those kernel PT submaps that BSD is so fond of
205 pt_entry_t *CMAP1 = NULL, *ptmmap;
206 caddr_t CADDR1 = NULL, ptvmmap = NULL;
207 static pt_entry_t *msgbufmap;
208 struct msgbuf *msgbufp=NULL;
213 static pt_entry_t *pt_crashdumpmap;
214 static caddr_t crashdumpmap;
216 static int pmap_yield_count = 64;
217 SYSCTL_INT(_machdep, OID_AUTO, pmap_yield_count, CTLFLAG_RW,
218 &pmap_yield_count, 0, "Yield during init_pt/release");
219 static int pmap_mmu_optimize = 0;
220 SYSCTL_INT(_machdep, OID_AUTO, pmap_mmu_optimize, CTLFLAG_RW,
221 &pmap_mmu_optimize, 0, "Share page table pages when possible");
225 static void pv_hold(pv_entry_t pv);
226 static int _pv_hold_try(pv_entry_t pv
228 static void pv_drop(pv_entry_t pv);
229 static void _pv_lock(pv_entry_t pv
231 static void pv_unlock(pv_entry_t pv);
232 static pv_entry_t _pv_alloc(pmap_t pmap, vm_pindex_t pindex, int *isnew
234 static pv_entry_t _pv_get(pmap_t pmap, vm_pindex_t pindex
236 static pv_entry_t pv_get_try(pmap_t pmap, vm_pindex_t pindex, int *errorp);
237 static pv_entry_t pv_find(pmap_t pmap, vm_pindex_t pindex);
238 static void pv_put(pv_entry_t pv);
239 static void pv_free(pv_entry_t pv);
240 static void *pv_pte_lookup(pv_entry_t pv, vm_pindex_t pindex);
241 static pv_entry_t pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex,
243 static pv_entry_t pmap_allocpte_seg(pmap_t pmap, vm_pindex_t ptepindex,
244 pv_entry_t *pvpp, vm_map_entry_t entry, vm_offset_t va);
245 static void pmap_remove_pv_pte(pv_entry_t pv, pv_entry_t pvp,
246 struct pmap_inval_info *info);
247 static vm_page_t pmap_remove_pv_page(pv_entry_t pv);
248 static int pmap_release_pv(pv_entry_t pv, pv_entry_t pvp);
250 struct pmap_scan_info;
251 static void pmap_remove_callback(pmap_t pmap, struct pmap_scan_info *info,
252 pv_entry_t pte_pv, pv_entry_t pt_pv, int sharept,
253 vm_offset_t va, pt_entry_t *ptep, void *arg __unused);
254 static void pmap_protect_callback(pmap_t pmap, struct pmap_scan_info *info,
255 pv_entry_t pte_pv, pv_entry_t pt_pv, int sharept,
256 vm_offset_t va, pt_entry_t *ptep, void *arg __unused);
258 static void i386_protection_init (void);
259 static void create_pagetables(vm_paddr_t *firstaddr);
260 static void pmap_remove_all (vm_page_t m);
261 static boolean_t pmap_testbit (vm_page_t m, int bit);
263 static pt_entry_t * pmap_pte_quick (pmap_t pmap, vm_offset_t va);
264 static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
266 static unsigned pdir4mb;
269 pv_entry_compare(pv_entry_t pv1, pv_entry_t pv2)
271 if (pv1->pv_pindex < pv2->pv_pindex)
273 if (pv1->pv_pindex > pv2->pv_pindex)
278 RB_GENERATE2(pv_entry_rb_tree, pv_entry, pv_entry,
279 pv_entry_compare, vm_pindex_t, pv_pindex);
282 * Move the kernel virtual free pointer to the next
283 * 2MB. This is used to help improve performance
284 * by using a large (2MB) page for much of the kernel
285 * (.text, .data, .bss)
289 pmap_kmem_choose(vm_offset_t addr)
291 vm_offset_t newaddr = addr;
293 newaddr = (addr + (NBPDR - 1)) & ~(NBPDR - 1);
300 * Super fast pmap_pte routine best used when scanning the pv lists.
301 * This eliminates many course-grained invltlb calls. Note that many of
302 * the pv list scans are across different pmaps and it is very wasteful
303 * to do an entire invltlb when checking a single mapping.
305 static __inline pt_entry_t *pmap_pte(pmap_t pmap, vm_offset_t va);
309 pmap_pte_quick(pmap_t pmap, vm_offset_t va)
311 return pmap_pte(pmap, va);
315 * Returns the pindex of a page table entry (representing a terminal page).
316 * There are NUPTE_TOTAL page table entries possible (a huge number)
318 * x86-64 has a 48-bit address space, where bit 47 is sign-extended out.
319 * We want to properly translate negative KVAs.
323 pmap_pte_pindex(vm_offset_t va)
325 return ((va >> PAGE_SHIFT) & (NUPTE_TOTAL - 1));
329 * Returns the pindex of a page table.
333 pmap_pt_pindex(vm_offset_t va)
335 return (NUPTE_TOTAL + ((va >> PDRSHIFT) & (NUPT_TOTAL - 1)));
339 * Returns the pindex of a page directory.
343 pmap_pd_pindex(vm_offset_t va)
345 return (NUPTE_TOTAL + NUPT_TOTAL +
346 ((va >> PDPSHIFT) & (NUPD_TOTAL - 1)));
351 pmap_pdp_pindex(vm_offset_t va)
353 return (NUPTE_TOTAL + NUPT_TOTAL + NUPD_TOTAL +
354 ((va >> PML4SHIFT) & (NUPDP_TOTAL - 1)));
359 pmap_pml4_pindex(void)
361 return (NUPTE_TOTAL + NUPT_TOTAL + NUPD_TOTAL + NUPDP_TOTAL);
365 * Return various clipped indexes for a given VA
367 * Returns the index of a pte in a page table, representing a terminal
372 pmap_pte_index(vm_offset_t va)
374 return ((va >> PAGE_SHIFT) & ((1ul << NPTEPGSHIFT) - 1));
378 * Returns the index of a pt in a page directory, representing a page
383 pmap_pt_index(vm_offset_t va)
385 return ((va >> PDRSHIFT) & ((1ul << NPDEPGSHIFT) - 1));
389 * Returns the index of a pd in a page directory page, representing a page
394 pmap_pd_index(vm_offset_t va)
396 return ((va >> PDPSHIFT) & ((1ul << NPDPEPGSHIFT) - 1));
400 * Returns the index of a pdp in the pml4 table, representing a page
405 pmap_pdp_index(vm_offset_t va)
407 return ((va >> PML4SHIFT) & ((1ul << NPML4EPGSHIFT) - 1));
411 * Generic procedure to index a pte from a pt, pd, or pdp.
413 * NOTE: Normally passed pindex as pmap_xx_index(). pmap_xx_pindex() is NOT
414 * a page table page index but is instead of PV lookup index.
418 pv_pte_lookup(pv_entry_t pv, vm_pindex_t pindex)
422 pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pv->pv_m));
423 return(&pte[pindex]);
427 * Return pointer to PDP slot in the PML4
431 pmap_pdp(pmap_t pmap, vm_offset_t va)
433 return (&pmap->pm_pml4[pmap_pdp_index(va)]);
437 * Return pointer to PD slot in the PDP given a pointer to the PDP
441 pmap_pdp_to_pd(pml4_entry_t pdp_pte, vm_offset_t va)
445 pd = (pdp_entry_t *)PHYS_TO_DMAP(pdp_pte & PG_FRAME);
446 return (&pd[pmap_pd_index(va)]);
450 * Return pointer to PD slot in the PDP.
454 pmap_pd(pmap_t pmap, vm_offset_t va)
458 pdp = pmap_pdp(pmap, va);
459 if ((*pdp & PG_V) == 0)
461 return (pmap_pdp_to_pd(*pdp, va));
465 * Return pointer to PT slot in the PD given a pointer to the PD
469 pmap_pd_to_pt(pdp_entry_t pd_pte, vm_offset_t va)
473 pt = (pd_entry_t *)PHYS_TO_DMAP(pd_pte & PG_FRAME);
474 return (&pt[pmap_pt_index(va)]);
478 * Return pointer to PT slot in the PD
480 * SIMPLE PMAP NOTE: Simple pmaps (embedded in objects) do not have PDPs,
481 * so we cannot lookup the PD via the PDP. Instead we
482 * must look it up via the pmap.
486 pmap_pt(pmap_t pmap, vm_offset_t va)
490 vm_pindex_t pd_pindex;
492 if (pmap->pm_flags & PMAP_FLAG_SIMPLE) {
493 pd_pindex = pmap_pd_pindex(va);
494 spin_lock(&pmap->pm_spin);
495 pv = pv_entry_rb_tree_RB_LOOKUP(&pmap->pm_pvroot, pd_pindex);
496 spin_unlock(&pmap->pm_spin);
497 if (pv == NULL || pv->pv_m == NULL)
499 return (pmap_pd_to_pt(VM_PAGE_TO_PHYS(pv->pv_m), va));
501 pd = pmap_pd(pmap, va);
502 if (pd == NULL || (*pd & PG_V) == 0)
504 return (pmap_pd_to_pt(*pd, va));
509 * Return pointer to PTE slot in the PT given a pointer to the PT
513 pmap_pt_to_pte(pd_entry_t pt_pte, vm_offset_t va)
517 pte = (pt_entry_t *)PHYS_TO_DMAP(pt_pte & PG_FRAME);
518 return (&pte[pmap_pte_index(va)]);
522 * Return pointer to PTE slot in the PT
526 pmap_pte(pmap_t pmap, vm_offset_t va)
530 pt = pmap_pt(pmap, va);
531 if (pt == NULL || (*pt & PG_V) == 0)
533 if ((*pt & PG_PS) != 0)
534 return ((pt_entry_t *)pt);
535 return (pmap_pt_to_pte(*pt, va));
539 * Of all the layers (PTE, PT, PD, PDP, PML4) the best one to cache is
540 * the PT layer. This will speed up core pmap operations considerably.
544 pv_cache(pv_entry_t pv, vm_pindex_t pindex)
546 if (pindex >= pmap_pt_pindex(0) && pindex <= pmap_pd_pindex(0))
547 pv->pv_pmap->pm_pvhint = pv;
552 * KVM - return address of PT slot in PD
556 vtopt(vm_offset_t va)
558 uint64_t mask = ((1ul << (NPDEPGSHIFT + NPDPEPGSHIFT +
559 NPML4EPGSHIFT)) - 1);
561 return (PDmap + ((va >> PDRSHIFT) & mask));
565 * KVM - return address of PTE slot in PT
569 vtopte(vm_offset_t va)
571 uint64_t mask = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT +
572 NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
574 return (PTmap + ((va >> PAGE_SHIFT) & mask));
578 allocpages(vm_paddr_t *firstaddr, long n)
583 bzero((void *)ret, n * PAGE_SIZE);
584 *firstaddr += n * PAGE_SIZE;
590 create_pagetables(vm_paddr_t *firstaddr)
592 long i; /* must be 64 bits */
598 * We are running (mostly) V=P at this point
600 * Calculate NKPT - number of kernel page tables. We have to
601 * accomodoate prealloction of the vm_page_array, dump bitmap,
602 * MSGBUF_SIZE, and other stuff. Be generous.
604 * Maxmem is in pages.
606 * ndmpdp is the number of 1GB pages we wish to map.
608 ndmpdp = (ptoa(Maxmem) + NBPDP - 1) >> PDPSHIFT;
609 if (ndmpdp < 4) /* Minimum 4GB of dirmap */
611 KKASSERT(ndmpdp <= NKPDPE * NPDEPG);
614 * Starting at the beginning of kvm (not KERNBASE).
616 nkpt_phys = (Maxmem * sizeof(struct vm_page) + NBPDR - 1) / NBPDR;
617 nkpt_phys += (Maxmem * sizeof(struct pv_entry) + NBPDR - 1) / NBPDR;
618 nkpt_phys += ((nkpt + nkpt + 1 + NKPML4E + NKPDPE + NDMPML4E +
619 ndmpdp) + 511) / 512;
623 * Starting at KERNBASE - map 2G worth of page table pages.
624 * KERNBASE is offset -2G from the end of kvm.
626 nkpt_base = (NPDPEPG - KPDPI) * NPTEPG; /* typically 2 x 512 */
631 KPTbase = allocpages(firstaddr, nkpt_base);
632 KPTphys = allocpages(firstaddr, nkpt_phys);
633 KPML4phys = allocpages(firstaddr, 1);
634 KPDPphys = allocpages(firstaddr, NKPML4E);
635 KPDphys = allocpages(firstaddr, NKPDPE);
638 * Calculate the page directory base for KERNBASE,
639 * that is where we start populating the page table pages.
640 * Basically this is the end - 2.
642 KPDbase = KPDphys + ((NKPDPE - (NPDPEPG - KPDPI)) << PAGE_SHIFT);
644 DMPDPphys = allocpages(firstaddr, NDMPML4E);
645 if ((amd_feature & AMDID_PAGE1GB) == 0)
646 DMPDphys = allocpages(firstaddr, ndmpdp);
647 dmaplimit = (vm_paddr_t)ndmpdp << PDPSHIFT;
650 * Fill in the underlying page table pages for the area around
651 * KERNBASE. This remaps low physical memory to KERNBASE.
653 * Read-only from zero to physfree
654 * XXX not fully used, underneath 2M pages
656 for (i = 0; (i << PAGE_SHIFT) < *firstaddr; i++) {
657 ((pt_entry_t *)KPTbase)[i] = i << PAGE_SHIFT;
658 ((pt_entry_t *)KPTbase)[i] |= PG_RW | PG_V | PG_G;
662 * Now map the initial kernel page tables. One block of page
663 * tables is placed at the beginning of kernel virtual memory,
664 * and another block is placed at KERNBASE to map the kernel binary,
665 * data, bss, and initial pre-allocations.
667 for (i = 0; i < nkpt_base; i++) {
668 ((pd_entry_t *)KPDbase)[i] = KPTbase + (i << PAGE_SHIFT);
669 ((pd_entry_t *)KPDbase)[i] |= PG_RW | PG_V;
671 for (i = 0; i < nkpt_phys; i++) {
672 ((pd_entry_t *)KPDphys)[i] = KPTphys + (i << PAGE_SHIFT);
673 ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V;
677 * Map from zero to end of allocations using 2M pages as an
678 * optimization. This will bypass some of the KPTBase pages
679 * above in the KERNBASE area.
681 for (i = 0; (i << PDRSHIFT) < *firstaddr; i++) {
682 ((pd_entry_t *)KPDbase)[i] = i << PDRSHIFT;
683 ((pd_entry_t *)KPDbase)[i] |= PG_RW | PG_V | PG_PS | PG_G;
687 * And connect up the PD to the PDP. The kernel pmap is expected
688 * to pre-populate all of its PDs. See NKPDPE in vmparam.h.
690 for (i = 0; i < NKPDPE; i++) {
691 ((pdp_entry_t *)KPDPphys)[NPDPEPG - NKPDPE + i] =
692 KPDphys + (i << PAGE_SHIFT);
693 ((pdp_entry_t *)KPDPphys)[NPDPEPG - NKPDPE + i] |=
698 * Now set up the direct map space using either 2MB or 1GB pages
699 * Preset PG_M and PG_A because demotion expects it.
701 * When filling in entries in the PD pages make sure any excess
702 * entries are set to zero as we allocated enough PD pages
704 if ((amd_feature & AMDID_PAGE1GB) == 0) {
705 for (i = 0; i < NPDEPG * ndmpdp; i++) {
706 ((pd_entry_t *)DMPDphys)[i] = i << PDRSHIFT;
707 ((pd_entry_t *)DMPDphys)[i] |= PG_RW | PG_V | PG_PS |
712 * And the direct map space's PDP
714 for (i = 0; i < ndmpdp; i++) {
715 ((pdp_entry_t *)DMPDPphys)[i] = DMPDphys +
717 ((pdp_entry_t *)DMPDPphys)[i] |= PG_RW | PG_V | PG_U;
720 for (i = 0; i < ndmpdp; i++) {
721 ((pdp_entry_t *)DMPDPphys)[i] =
722 (vm_paddr_t)i << PDPSHIFT;
723 ((pdp_entry_t *)DMPDPphys)[i] |= PG_RW | PG_V | PG_PS |
728 /* And recursively map PML4 to itself in order to get PTmap */
729 ((pdp_entry_t *)KPML4phys)[PML4PML4I] = KPML4phys;
730 ((pdp_entry_t *)KPML4phys)[PML4PML4I] |= PG_RW | PG_V | PG_U;
733 * Connect the Direct Map slots up to the PML4
735 for (j = 0; j < NDMPML4E; ++j) {
736 ((pdp_entry_t *)KPML4phys)[DMPML4I + j] =
737 (DMPDPphys + ((vm_paddr_t)j << PML4SHIFT)) |
742 * Connect the KVA slot up to the PML4
744 ((pdp_entry_t *)KPML4phys)[KPML4I] = KPDPphys;
745 ((pdp_entry_t *)KPML4phys)[KPML4I] |= PG_RW | PG_V | PG_U;
749 * Bootstrap the system enough to run with virtual memory.
751 * On the i386 this is called after mapping has already been enabled
752 * and just syncs the pmap module with what has already been done.
753 * [We can't call it easily with mapping off since the kernel is not
754 * mapped with PA == VA, hence we would have to relocate every address
755 * from the linked base (virtual) address "KERNBASE" to the actual
756 * (physical) address starting relative to 0]
759 pmap_bootstrap(vm_paddr_t *firstaddr)
764 KvaStart = VM_MIN_KERNEL_ADDRESS;
765 KvaEnd = VM_MAX_KERNEL_ADDRESS;
766 KvaSize = KvaEnd - KvaStart;
768 avail_start = *firstaddr;
771 * Create an initial set of page tables to run the kernel in.
773 create_pagetables(firstaddr);
775 virtual2_start = KvaStart;
776 virtual2_end = PTOV_OFFSET;
778 virtual_start = (vm_offset_t) PTOV_OFFSET + *firstaddr;
779 virtual_start = pmap_kmem_choose(virtual_start);
781 virtual_end = VM_MAX_KERNEL_ADDRESS;
783 /* XXX do %cr0 as well */
784 load_cr4(rcr4() | CR4_PGE | CR4_PSE);
788 * Initialize protection array.
790 i386_protection_init();
793 * The kernel's pmap is statically allocated so we don't have to use
794 * pmap_create, which is unlikely to work correctly at this part of
795 * the boot sequence (XXX and which no longer exists).
797 kernel_pmap.pm_pml4 = (pdp_entry_t *) (PTOV_OFFSET + KPML4phys);
798 kernel_pmap.pm_count = 1;
799 kernel_pmap.pm_active = (cpumask_t)-1 & ~CPUMASK_LOCK;
800 RB_INIT(&kernel_pmap.pm_pvroot);
801 spin_init(&kernel_pmap.pm_spin);
802 lwkt_token_init(&kernel_pmap.pm_token, "kpmap_tok");
805 * Reserve some special page table entries/VA space for temporary
808 #define SYSMAP(c, p, v, n) \
809 v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n);
815 * CMAP1/CMAP2 are used for zeroing and copying pages.
817 SYSMAP(caddr_t, CMAP1, CADDR1, 1)
822 SYSMAP(caddr_t, pt_crashdumpmap, crashdumpmap, MAXDUMPPGS);
825 * ptvmmap is used for reading arbitrary physical pages via
828 SYSMAP(caddr_t, ptmmap, ptvmmap, 1)
831 * msgbufp is used to map the system message buffer.
832 * XXX msgbufmap is not used.
834 SYSMAP(struct msgbuf *, msgbufmap, msgbufp,
835 atop(round_page(MSGBUF_SIZE)))
842 * PG_G is terribly broken on SMP because we IPI invltlb's in some
843 * cases rather then invl1pg. Actually, I don't even know why it
844 * works under UP because self-referential page table mappings
849 * Initialize the 4MB page size flag
853 * The 4MB page version of the initial
854 * kernel page mapping.
858 #if !defined(DISABLE_PSE)
859 if (cpu_feature & CPUID_PSE) {
862 * Note that we have enabled PSE mode
865 ptditmp = *(PTmap + x86_64_btop(KERNBASE));
866 ptditmp &= ~(NBPDR - 1);
867 ptditmp |= PG_V | PG_RW | PG_PS | PG_U | pgeflag;
873 /* Initialize the PAT MSR */
887 * Default values mapping PATi,PCD,PWT bits at system reset.
888 * The default values effectively ignore the PATi bit by
889 * repeating the encodings for 0-3 in 4-7, and map the PCD
890 * and PWT bit combinations to the expected PAT types.
892 pat_msr = PAT_VALUE(0, PAT_WRITE_BACK) | /* 000 */
893 PAT_VALUE(1, PAT_WRITE_THROUGH) | /* 001 */
894 PAT_VALUE(2, PAT_UNCACHED) | /* 010 */
895 PAT_VALUE(3, PAT_UNCACHEABLE) | /* 011 */
896 PAT_VALUE(4, PAT_WRITE_BACK) | /* 100 */
897 PAT_VALUE(5, PAT_WRITE_THROUGH) | /* 101 */
898 PAT_VALUE(6, PAT_UNCACHED) | /* 110 */
899 PAT_VALUE(7, PAT_UNCACHEABLE); /* 111 */
900 pat_pte_index[PAT_WRITE_BACK] = 0;
901 pat_pte_index[PAT_WRITE_THROUGH]= 0 | PG_NC_PWT;
902 pat_pte_index[PAT_UNCACHED] = PG_NC_PCD;
903 pat_pte_index[PAT_UNCACHEABLE] = PG_NC_PCD | PG_NC_PWT;
904 pat_pte_index[PAT_WRITE_PROTECTED] = pat_pte_index[PAT_UNCACHEABLE];
905 pat_pte_index[PAT_WRITE_COMBINING] = pat_pte_index[PAT_UNCACHEABLE];
907 if (cpu_feature & CPUID_PAT) {
909 * If we support the PAT then set-up entries for
910 * WRITE_PROTECTED and WRITE_COMBINING using bit patterns
913 pat_msr = (pat_msr & ~PAT_MASK(4)) |
914 PAT_VALUE(4, PAT_WRITE_PROTECTED);
915 pat_msr = (pat_msr & ~PAT_MASK(5)) |
916 PAT_VALUE(5, PAT_WRITE_COMBINING);
917 pat_pte_index[PAT_WRITE_PROTECTED] = PG_PTE_PAT | 0;
918 pat_pte_index[PAT_WRITE_COMBINING] = PG_PTE_PAT | PG_NC_PWT;
921 * Then enable the PAT
926 load_cr4(cr4 & ~CR4_PGE);
928 /* Disable caches (CD = 1, NW = 0). */
930 load_cr0((cr0 & ~CR0_NW) | CR0_CD);
932 /* Flushes caches and TLBs. */
936 /* Update PAT and index table. */
937 wrmsr(MSR_PAT, pat_msr);
939 /* Flush caches and TLBs again. */
943 /* Restore caches and PGE. */
951 * Set 4mb pdir for mp startup
956 if (pseflag && (cpu_feature & CPUID_PSE)) {
957 load_cr4(rcr4() | CR4_PSE);
958 if (pdir4mb && mycpu->gd_cpuid == 0) { /* only on BSP */
965 * Initialize the pmap module.
966 * Called by vm_init, to initialize any structures that the pmap
967 * system needs to map virtual memory.
968 * pmap_init has been enhanced to support in a fairly consistant
969 * way, discontiguous physical memory.
978 * Allocate memory for random pmap data structures. Includes the
982 for (i = 0; i < vm_page_array_size; i++) {
985 m = &vm_page_array[i];
986 TAILQ_INIT(&m->md.pv_list);
990 * init the pv free list
992 initial_pvs = vm_page_array_size;
993 if (initial_pvs < MINPV)
995 pvzone = &pvzone_store;
996 pvinit = (void *)kmem_alloc(&kernel_map,
997 initial_pvs * sizeof (struct pv_entry));
998 zbootinit(pvzone, "PV ENTRY", sizeof (struct pv_entry),
999 pvinit, initial_pvs);
1002 * Now it is safe to enable pv_table recording.
1004 pmap_initialized = TRUE;
1008 * Initialize the address space (zone) for the pv_entries. Set a
1009 * high water mark so that the system can recover from excessive
1010 * numbers of pv entries.
1015 int shpgperproc = PMAP_SHPGPERPROC;
1018 TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
1019 pv_entry_max = shpgperproc * maxproc + vm_page_array_size;
1020 TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
1021 pv_entry_high_water = 9 * (pv_entry_max / 10);
1024 * Subtract out pages already installed in the zone (hack)
1026 entry_max = pv_entry_max - vm_page_array_size;
1030 zinitna(pvzone, &pvzone_obj, NULL, 0, entry_max, ZONE_INTERRUPT, 1);
1034 * Typically used to initialize a fictitious page by vm/device_pager.c
1037 pmap_page_init(struct vm_page *m)
1040 TAILQ_INIT(&m->md.pv_list);
1043 /***************************************************
1044 * Low level helper routines.....
1045 ***************************************************/
1048 * this routine defines the region(s) of memory that should
1049 * not be tested for the modified bit.
1053 pmap_track_modified(vm_pindex_t pindex)
1055 vm_offset_t va = (vm_offset_t)pindex << PAGE_SHIFT;
1056 if ((va < clean_sva) || (va >= clean_eva))
1063 * Extract the physical page address associated with the map/VA pair.
1064 * The page must be wired for this to work reliably.
1066 * XXX for the moment we're using pv_find() instead of pv_get(), as
1067 * callers might be expecting non-blocking operation.
1070 pmap_extract(pmap_t pmap, vm_offset_t va)
1077 if (va >= VM_MAX_USER_ADDRESS) {
1079 * Kernel page directories might be direct-mapped and
1080 * there is typically no PV tracking of pte's
1084 pt = pmap_pt(pmap, va);
1085 if (pt && (*pt & PG_V)) {
1087 rtval = *pt & PG_PS_FRAME;
1088 rtval |= va & PDRMASK;
1090 ptep = pmap_pt_to_pte(*pt, va);
1092 rtval = *ptep & PG_FRAME;
1093 rtval |= va & PAGE_MASK;
1099 * User pages currently do not direct-map the page directory
1100 * and some pages might not used managed PVs. But all PT's
1103 pt_pv = pv_find(pmap, pmap_pt_pindex(va));
1105 ptep = pv_pte_lookup(pt_pv, pmap_pte_index(va));
1107 rtval = *ptep & PG_FRAME;
1108 rtval |= va & PAGE_MASK;
1117 * Extract the physical page address associated kernel virtual address.
1120 pmap_kextract(vm_offset_t va)
1122 pd_entry_t pt; /* pt entry in pd */
1125 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
1126 pa = DMAP_TO_PHYS(va);
1130 pa = (pt & PG_PS_FRAME) | (va & PDRMASK);
1133 * Beware of a concurrent promotion that changes the
1134 * PDE at this point! For example, vtopte() must not
1135 * be used to access the PTE because it would use the
1136 * new PDE. It is, however, safe to use the old PDE
1137 * because the page table page is preserved by the
1140 pa = *pmap_pt_to_pte(pt, va);
1141 pa = (pa & PG_FRAME) | (va & PAGE_MASK);
1147 /***************************************************
1148 * Low level mapping routines.....
1149 ***************************************************/
1152 * Routine: pmap_kenter
1154 * Add a wired page to the KVA
1155 * NOTE! note that in order for the mapping to take effect -- you
1156 * should do an invltlb after doing the pmap_kenter().
1159 pmap_kenter(vm_offset_t va, vm_paddr_t pa)
1163 pmap_inval_info info;
1165 pmap_inval_init(&info); /* XXX remove */
1166 npte = pa | PG_RW | PG_V | pgeflag;
1168 pmap_inval_interlock(&info, &kernel_pmap, va); /* XXX remove */
1170 pmap_inval_deinterlock(&info, &kernel_pmap); /* XXX remove */
1171 pmap_inval_done(&info); /* XXX remove */
1175 * Routine: pmap_kenter_quick
1177 * Similar to pmap_kenter(), except we only invalidate the
1178 * mapping on the current CPU.
1181 pmap_kenter_quick(vm_offset_t va, vm_paddr_t pa)
1186 npte = pa | PG_RW | PG_V | pgeflag;
1189 cpu_invlpg((void *)va);
1193 pmap_kenter_sync(vm_offset_t va)
1195 pmap_inval_info info;
1197 pmap_inval_init(&info);
1198 pmap_inval_interlock(&info, &kernel_pmap, va);
1199 pmap_inval_deinterlock(&info, &kernel_pmap);
1200 pmap_inval_done(&info);
1204 pmap_kenter_sync_quick(vm_offset_t va)
1206 cpu_invlpg((void *)va);
1210 * remove a page from the kernel pagetables
1213 pmap_kremove(vm_offset_t va)
1216 pmap_inval_info info;
1218 pmap_inval_init(&info);
1220 pmap_inval_interlock(&info, &kernel_pmap, va);
1221 (void)pte_load_clear(pte);
1222 pmap_inval_deinterlock(&info, &kernel_pmap);
1223 pmap_inval_done(&info);
1227 pmap_kremove_quick(vm_offset_t va)
1231 (void)pte_load_clear(pte);
1232 cpu_invlpg((void *)va);
1236 * XXX these need to be recoded. They are not used in any critical path.
1239 pmap_kmodify_rw(vm_offset_t va)
1241 atomic_set_long(vtopte(va), PG_RW);
1242 cpu_invlpg((void *)va);
1246 pmap_kmodify_nc(vm_offset_t va)
1248 atomic_set_long(vtopte(va), PG_N);
1249 cpu_invlpg((void *)va);
1253 * Used to map a range of physical addresses into kernel virtual
1254 * address space during the low level boot, typically to map the
1255 * dump bitmap, message buffer, and vm_page_array.
1257 * These mappings are typically made at some pointer after the end of the
1260 * We could return PHYS_TO_DMAP(start) here and not allocate any
1261 * via (*virtp), but then kmem from userland and kernel dumps won't
1262 * have access to the related pointers.
1265 pmap_map(vm_offset_t *virtp, vm_paddr_t start, vm_paddr_t end, int prot)
1268 vm_offset_t va_start;
1270 /*return PHYS_TO_DMAP(start);*/
1275 while (start < end) {
1276 pmap_kenter_quick(va, start);
1284 #define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024)
1287 * Remove the specified set of pages from the data and instruction caches.
1289 * In contrast to pmap_invalidate_cache_range(), this function does not
1290 * rely on the CPU's self-snoop feature, because it is intended for use
1291 * when moving pages into a different cache domain.
1294 pmap_invalidate_cache_pages(vm_page_t *pages, int count)
1296 vm_offset_t daddr, eva;
1299 if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE ||
1300 (cpu_feature & CPUID_CLFSH) == 0)
1304 for (i = 0; i < count; i++) {
1305 daddr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pages[i]));
1306 eva = daddr + PAGE_SIZE;
1307 for (; daddr < eva; daddr += cpu_clflush_line_size)
1315 pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
1317 KASSERT((sva & PAGE_MASK) == 0,
1318 ("pmap_invalidate_cache_range: sva not page-aligned"));
1319 KASSERT((eva & PAGE_MASK) == 0,
1320 ("pmap_invalidate_cache_range: eva not page-aligned"));
1322 if (cpu_feature & CPUID_SS) {
1323 ; /* If "Self Snoop" is supported, do nothing. */
1325 /* Globally invalidate caches */
1326 cpu_wbinvd_on_all_cpus();
1330 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
1332 smp_invlpg_range(pmap->pm_active, sva, eva);
1336 * Add a list of wired pages to the kva
1337 * this routine is only used for temporary
1338 * kernel mappings that do not need to have
1339 * page modification or references recorded.
1340 * Note that old mappings are simply written
1341 * over. The page *must* be wired.
1344 pmap_qenter(vm_offset_t va, vm_page_t *m, int count)
1348 end_va = va + count * PAGE_SIZE;
1350 while (va < end_va) {
1354 *pte = VM_PAGE_TO_PHYS(*m) | PG_RW | PG_V |
1355 pat_pte_index[(*m)->pat_mode] | pgeflag;
1356 cpu_invlpg((void *)va);
1364 * This routine jerks page mappings from the
1365 * kernel -- it is meant only for temporary mappings.
1367 * MPSAFE, INTERRUPT SAFE (cluster callback)
1370 pmap_qremove(vm_offset_t va, int count)
1374 end_va = va + count * PAGE_SIZE;
1376 while (va < end_va) {
1380 (void)pte_load_clear(pte);
1381 cpu_invlpg((void *)va);
1388 * Create a new thread and optionally associate it with a (new) process.
1389 * NOTE! the new thread's cpu may not equal the current cpu.
1392 pmap_init_thread(thread_t td)
1394 /* enforce pcb placement & alignment */
1395 td->td_pcb = (struct pcb *)(td->td_kstack + td->td_kstack_size) - 1;
1396 td->td_pcb = (struct pcb *)((intptr_t)td->td_pcb & ~(intptr_t)0xF);
1397 td->td_savefpu = &td->td_pcb->pcb_save;
1398 td->td_sp = (char *)td->td_pcb; /* no -16 */
1402 * This routine directly affects the fork perf for a process.
1405 pmap_init_proc(struct proc *p)
1410 * Initialize pmap0/vmspace0. This pmap is not added to pmap_list because
1411 * it, and IdlePTD, represents the template used to update all other pmaps.
1413 * On architectures where the kernel pmap is not integrated into the user
1414 * process pmap, this pmap represents the process pmap, not the kernel pmap.
1415 * kernel_pmap should be used to directly access the kernel_pmap.
1418 pmap_pinit0(struct pmap *pmap)
1420 pmap->pm_pml4 = (pml4_entry_t *)(PTOV_OFFSET + KPML4phys);
1422 pmap->pm_active = 0;
1423 pmap->pm_pvhint = NULL;
1424 RB_INIT(&pmap->pm_pvroot);
1425 spin_init(&pmap->pm_spin);
1426 lwkt_token_init(&pmap->pm_token, "pmap_tok");
1427 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
1431 * Initialize a preallocated and zeroed pmap structure,
1432 * such as one in a vmspace structure.
1435 pmap_pinit_simple(struct pmap *pmap)
1438 * Misc initialization
1441 pmap->pm_active = 0;
1442 pmap->pm_pvhint = NULL;
1443 pmap->pm_flags = PMAP_FLAG_SIMPLE;
1446 * Don't blow up locks/tokens on re-use (XXX fix/use drop code
1449 if (pmap->pm_pmlpv == NULL) {
1450 RB_INIT(&pmap->pm_pvroot);
1451 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
1452 spin_init(&pmap->pm_spin);
1453 lwkt_token_init(&pmap->pm_token, "pmap_tok");
1458 pmap_pinit(struct pmap *pmap)
1463 pmap_pinit_simple(pmap);
1464 pmap->pm_flags &= ~PMAP_FLAG_SIMPLE;
1467 * No need to allocate page table space yet but we do need a valid
1468 * page directory table.
1470 if (pmap->pm_pml4 == NULL) {
1472 (pml4_entry_t *)kmem_alloc_pageable(&kernel_map, PAGE_SIZE);
1476 * Allocate the page directory page, which wires it even though
1477 * it isn't being entered into some higher level page table (it
1478 * being the highest level). If one is already cached we don't
1479 * have to do anything.
1481 if ((pv = pmap->pm_pmlpv) == NULL) {
1482 pv = pmap_allocpte(pmap, pmap_pml4_pindex(), NULL);
1483 pmap->pm_pmlpv = pv;
1484 pmap_kenter((vm_offset_t)pmap->pm_pml4,
1485 VM_PAGE_TO_PHYS(pv->pv_m));
1489 * Install DMAP and KMAP.
1491 for (j = 0; j < NDMPML4E; ++j) {
1492 pmap->pm_pml4[DMPML4I + j] =
1493 (DMPDPphys + ((vm_paddr_t)j << PML4SHIFT)) |
1494 PG_RW | PG_V | PG_U;
1496 pmap->pm_pml4[KPML4I] = KPDPphys | PG_RW | PG_V | PG_U;
1499 * install self-referential address mapping entry
1501 pmap->pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pv->pv_m) |
1502 PG_V | PG_RW | PG_A | PG_M;
1504 KKASSERT(pv->pv_m->flags & PG_MAPPED);
1505 KKASSERT(pv->pv_m->flags & PG_WRITEABLE);
1507 KKASSERT(pmap->pm_pml4[255] == 0);
1508 KKASSERT(RB_ROOT(&pmap->pm_pvroot) == pv);
1509 KKASSERT(pv->pv_entry.rbe_left == NULL);
1510 KKASSERT(pv->pv_entry.rbe_right == NULL);
1514 * Clean up a pmap structure so it can be physically freed. This routine
1515 * is called by the vmspace dtor function. A great deal of pmap data is
1516 * left passively mapped to improve vmspace management so we have a bit
1517 * of cleanup work to do here.
1520 pmap_puninit(pmap_t pmap)
1525 KKASSERT(pmap->pm_active == 0);
1526 if ((pv = pmap->pm_pmlpv) != NULL) {
1527 if (pv_hold_try(pv) == 0)
1529 p = pmap_remove_pv_page(pv);
1531 pmap_kremove((vm_offset_t)pmap->pm_pml4);
1532 vm_page_busy_wait(p, FALSE, "pgpun");
1533 KKASSERT(p->flags & (PG_FICTITIOUS|PG_UNMANAGED));
1534 vm_page_unwire(p, 0);
1535 vm_page_flag_clear(p, PG_MAPPED | PG_WRITEABLE);
1538 * XXX eventually clean out PML4 static entries and
1539 * use vm_page_free_zero()
1542 pmap->pm_pmlpv = NULL;
1544 if (pmap->pm_pml4) {
1545 KKASSERT(pmap->pm_pml4 != (void *)(PTOV_OFFSET + KPML4phys));
1546 kmem_free(&kernel_map, (vm_offset_t)pmap->pm_pml4, PAGE_SIZE);
1547 pmap->pm_pml4 = NULL;
1549 KKASSERT(pmap->pm_stats.resident_count == 0);
1550 KKASSERT(pmap->pm_stats.wired_count == 0);
1554 * Wire in kernel global address entries. To avoid a race condition
1555 * between pmap initialization and pmap_growkernel, this procedure
1556 * adds the pmap to the master list (which growkernel scans to update),
1557 * then copies the template.
1560 pmap_pinit2(struct pmap *pmap)
1562 spin_lock(&pmap_spin);
1563 TAILQ_INSERT_TAIL(&pmap_list, pmap, pm_pmnode);
1564 spin_unlock(&pmap_spin);
1568 * This routine is called when various levels in the page table need to
1569 * be populated. This routine cannot fail.
1571 * This function returns two locked pv_entry's, one representing the
1572 * requested pv and one representing the requested pv's parent pv. If
1573 * the pv did not previously exist it will be mapped into its parent
1574 * and wired, otherwise no additional wire count will be added.
1578 pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, pv_entry_t *pvpp)
1583 vm_pindex_t pt_pindex;
1589 * If the pv already exists and we aren't being asked for the
1590 * parent page table page we can just return it. A locked+held pv
1594 pv = pv_alloc(pmap, ptepindex, &isnew);
1595 if (isnew == 0 && pvpp == NULL)
1599 * This is a new PV, we have to resolve its parent page table and
1600 * add an additional wiring to the page if necessary.
1604 * Special case terminal PVs. These are not page table pages so
1605 * no vm_page is allocated (the caller supplied the vm_page). If
1606 * pvpp is non-NULL we are being asked to also removed the pt_pv
1609 * Note that pt_pv's are only returned for user VAs. We assert that
1610 * a pt_pv is not being requested for kernel VAs.
1612 if (ptepindex < pmap_pt_pindex(0)) {
1613 if (ptepindex >= NUPTE_USER)
1614 KKASSERT(pvpp == NULL);
1616 KKASSERT(pvpp != NULL);
1618 pt_pindex = NUPTE_TOTAL + (ptepindex >> NPTEPGSHIFT);
1619 pvp = pmap_allocpte(pmap, pt_pindex, NULL);
1621 vm_page_wire_quick(pvp->pv_m);
1630 * Non-terminal PVs allocate a VM page to represent the page table,
1631 * so we have to resolve pvp and calculate ptepindex for the pvp
1632 * and then for the page table entry index in the pvp for
1635 if (ptepindex < pmap_pd_pindex(0)) {
1637 * pv is PT, pvp is PD
1639 ptepindex = (ptepindex - pmap_pt_pindex(0)) >> NPDEPGSHIFT;
1640 ptepindex += NUPTE_TOTAL + NUPT_TOTAL;
1641 pvp = pmap_allocpte(pmap, ptepindex, NULL);
1648 ptepindex = pv->pv_pindex - pmap_pt_pindex(0);
1649 ptepindex &= ((1ul << NPDEPGSHIFT) - 1);
1651 } else if (ptepindex < pmap_pdp_pindex(0)) {
1653 * pv is PD, pvp is PDP
1655 * SIMPLE PMAP NOTE: Simple pmaps do not allocate above
1658 ptepindex = (ptepindex - pmap_pd_pindex(0)) >> NPDPEPGSHIFT;
1659 ptepindex += NUPTE_TOTAL + NUPT_TOTAL + NUPD_TOTAL;
1661 if (pmap->pm_flags & PMAP_FLAG_SIMPLE) {
1662 KKASSERT(pvpp == NULL);
1665 pvp = pmap_allocpte(pmap, ptepindex, NULL);
1673 ptepindex = pv->pv_pindex - pmap_pd_pindex(0);
1674 ptepindex &= ((1ul << NPDPEPGSHIFT) - 1);
1675 } else if (ptepindex < pmap_pml4_pindex()) {
1677 * pv is PDP, pvp is the root pml4 table
1679 pvp = pmap_allocpte(pmap, pmap_pml4_pindex(), NULL);
1686 ptepindex = pv->pv_pindex - pmap_pdp_pindex(0);
1687 ptepindex &= ((1ul << NPML4EPGSHIFT) - 1);
1690 * pv represents the top-level PML4, there is no parent.
1698 * This code is only reached if isnew is TRUE and this is not a
1699 * terminal PV. We need to allocate a vm_page for the page table
1700 * at this level and enter it into the parent page table.
1702 * page table pages are marked PG_WRITEABLE and PG_MAPPED.
1705 m = vm_page_alloc(NULL, pv->pv_pindex,
1706 VM_ALLOC_NORMAL | VM_ALLOC_SYSTEM |
1707 VM_ALLOC_INTERRUPT);
1712 vm_page_spin_lock(m);
1713 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
1715 vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE);
1716 vm_page_spin_unlock(m);
1717 vm_page_unmanage(m); /* m must be spinunlocked */
1719 if ((m->flags & PG_ZERO) == 0) {
1720 pmap_zero_page(VM_PAGE_TO_PHYS(m));
1724 pmap_page_assertzero(VM_PAGE_TO_PHYS(m));
1727 m->valid = VM_PAGE_BITS_ALL;
1728 vm_page_flag_clear(m, PG_ZERO);
1729 vm_page_wire(m); /* wire for mapping in parent */
1732 * Wire the page into pvp, bump the wire-count for pvp's page table
1733 * page. Bump the resident_count for the pmap. There is no pvp
1734 * for the top level, address the pm_pml4[] array directly.
1736 * If the caller wants the parent we return it, otherwise
1737 * we just put it away.
1739 * No interlock is needed for pte 0 -> non-zero.
1741 * In the situation where *ptep is valid we might have an unmanaged
1742 * page table page shared from another page table which we need to
1743 * unshare before installing our private page table page.
1746 ptep = pv_pte_lookup(pvp, ptepindex);
1749 pmap_inval_info info;
1752 panic("pmap_allocpte: unexpected pte %p/%d",
1753 pvp, (int)ptepindex);
1755 pmap_inval_init(&info);
1756 pmap_inval_interlock(&info, pmap, (vm_offset_t)-1);
1757 pte = pte_load_clear(ptep);
1758 pmap_inval_deinterlock(&info, pmap);
1759 pmap_inval_done(&info);
1760 if (vm_page_unwire_quick(
1761 PHYS_TO_VM_PAGE(pte & PG_FRAME))) {
1762 panic("pmap_allocpte: shared pgtable "
1763 "pg bad wirecount");
1765 atomic_add_long(&pmap->pm_stats.resident_count, -1);
1767 vm_page_wire_quick(pvp->pv_m);
1769 *ptep = VM_PAGE_TO_PHYS(m) | (PG_U | PG_RW | PG_V |
1782 * This version of pmap_allocpte() checks for possible segment optimizations
1783 * that would allow page-table sharing. It can be called for terminal
1784 * page or page table page ptepindex's.
1786 * The function is called with page table page ptepindex's for fictitious
1787 * and unmanaged terminal pages. That is, we don't want to allocate a
1788 * terminal pv, we just want the pt_pv. pvpp is usually passed as NULL
1791 * This function can return a pv and *pvpp associated with the passed in pmap
1792 * OR a pv and *pvpp associated with the shared pmap. In the latter case
1793 * an unmanaged page table page will be entered into the pass in pmap.
1797 pmap_allocpte_seg(pmap_t pmap, vm_pindex_t ptepindex, pv_entry_t *pvpp,
1798 vm_map_entry_t entry, vm_offset_t va)
1800 struct pmap_inval_info info;
1805 pv_entry_t pte_pv; /* in original or shared pmap */
1806 pv_entry_t pt_pv; /* in original or shared pmap */
1807 pv_entry_t proc_pd_pv; /* in original pmap */
1808 pv_entry_t proc_pt_pv; /* in original pmap */
1809 pv_entry_t xpv; /* PT in shared pmap */
1810 pd_entry_t *pt; /* PT entry in PD of original pmap */
1811 pd_entry_t opte; /* contents of *pt */
1812 pd_entry_t npte; /* contents of *pt */
1817 * Basic tests, require a non-NULL vm_map_entry, require proper
1818 * alignment and type for the vm_map_entry, require that the
1819 * underlying object already be allocated.
1821 * We currently allow any type of object to use this optimization.
1822 * The object itself does NOT have to be sized to a multiple of the
1823 * segment size, but the memory mapping does.
1825 * XXX don't handle devices currently, because VM_PAGE_TO_PHYS()
1826 * won't work as expected.
1828 if (entry == NULL ||
1829 pmap_mmu_optimize == 0 || /* not enabled */
1830 ptepindex >= pmap_pd_pindex(0) || /* not terminal */
1831 entry->inheritance != VM_INHERIT_SHARE || /* not shared */
1832 entry->maptype != VM_MAPTYPE_NORMAL || /* weird map type */
1833 entry->object.vm_object == NULL || /* needs VM object */
1834 entry->object.vm_object->type == OBJT_DEVICE || /* ick */
1835 entry->object.vm_object->type == OBJT_MGTDEVICE || /* ick */
1836 (entry->offset & SEG_MASK) || /* must be aligned */
1837 (entry->start & SEG_MASK)) {
1838 return(pmap_allocpte(pmap, ptepindex, pvpp));
1842 * Make sure the full segment can be represented.
1844 b = va & ~(vm_offset_t)SEG_MASK;
1845 if (b < entry->start && b + SEG_SIZE > entry->end)
1846 return(pmap_allocpte(pmap, ptepindex, pvpp));
1849 * If the full segment can be represented dive the VM object's
1850 * shared pmap, allocating as required.
1852 object = entry->object.vm_object;
1854 if (entry->protection & VM_PROT_WRITE)
1855 obpmapp = &object->md.pmap_rw;
1857 obpmapp = &object->md.pmap_ro;
1860 * We allocate what appears to be a normal pmap but because portions
1861 * of this pmap are shared with other unrelated pmaps we have to
1862 * set pm_active to point to all cpus.
1864 * XXX Currently using pmap_spin to interlock the update, can't use
1865 * vm_object_hold/drop because the token might already be held
1866 * shared OR exclusive and we don't know.
1868 while ((obpmap = *obpmapp) == NULL) {
1869 obpmap = kmalloc(sizeof(*obpmap), M_OBJPMAP, M_WAITOK|M_ZERO);
1870 pmap_pinit_simple(obpmap);
1871 pmap_pinit2(obpmap);
1872 spin_lock(&pmap_spin);
1873 if (*obpmapp != NULL) {
1877 spin_unlock(&pmap_spin);
1878 pmap_release(obpmap);
1879 pmap_puninit(obpmap);
1880 kfree(obpmap, M_OBJPMAP);
1882 obpmap->pm_active = smp_active_mask;
1884 spin_unlock(&pmap_spin);
1889 * Layering is: PTE, PT, PD, PDP, PML4. We have to return the
1890 * pte/pt using the shared pmap from the object but also adjust
1891 * the process pmap's page table page as a side effect.
1895 * Resolve the terminal PTE and PT in the shared pmap. This is what
1896 * we will return. This is true if ptepindex represents a terminal
1897 * page, otherwise pte_pv is actually the PT and pt_pv is actually
1901 pte_pv = pmap_allocpte(obpmap, ptepindex, &pt_pv);
1902 if (ptepindex >= pmap_pt_pindex(0))
1908 * Resolve the PD in the process pmap so we can properly share the
1909 * page table page. Lock order is bottom-up (leaf first)!
1911 * NOTE: proc_pt_pv can be NULL.
1913 proc_pt_pv = pv_get(pmap, pmap_pt_pindex(b));
1914 proc_pd_pv = pmap_allocpte(pmap, pmap_pd_pindex(b), NULL);
1917 * xpv is the page table page pv from the shared object
1918 * (for convenience).
1920 * Calculate the pte value for the PT to load into the process PD.
1921 * If we have to change it we must properly dispose of the previous
1924 pt = pv_pte_lookup(proc_pd_pv, pmap_pt_index(b));
1925 npte = VM_PAGE_TO_PHYS(xpv->pv_m) |
1926 (PG_U | PG_RW | PG_V | PG_A | PG_M);
1929 * Dispose of previous page table page if it was local to the
1930 * process pmap. If the old pt is not empty we cannot dispose of it
1931 * until we clean it out. This case should not arise very often so
1932 * it is not optimized.
1935 if (proc_pt_pv->pv_m->wire_count != 1) {
1941 va & ~(vm_offset_t)SEG_MASK,
1942 (va + SEG_SIZE) & ~(vm_offset_t)SEG_MASK);
1945 pmap_release_pv(proc_pt_pv, proc_pd_pv);
1948 pt = pv_pte_lookup(proc_pd_pv, pmap_pt_index(b));
1952 * Handle remaining cases.
1956 vm_page_wire_quick(xpv->pv_m);
1957 vm_page_wire_quick(proc_pd_pv->pv_m);
1958 atomic_add_long(&pmap->pm_stats.resident_count, 1);
1959 } else if (*pt != npte) {
1960 pmap_inval_init(&info);
1961 pmap_inval_interlock(&info, pmap, (vm_offset_t)-1);
1963 opte = pte_load_clear(pt);
1964 KKASSERT(opte && opte != npte);
1967 vm_page_wire_quick(xpv->pv_m); /* pgtable pg that is npte */
1970 * Clean up opte, bump the wire_count for the process
1971 * PD page representing the new entry if it was
1974 * If the entry was not previously empty and we have
1975 * a PT in the proc pmap then opte must match that
1976 * pt. The proc pt must be retired (this is done
1977 * later on in this procedure).
1979 * NOTE: replacing valid pte, wire_count on proc_pd_pv
1982 KKASSERT(opte & PG_V);
1983 m = PHYS_TO_VM_PAGE(opte & PG_FRAME);
1984 if (vm_page_unwire_quick(m)) {
1985 panic("pmap_allocpte_seg: "
1986 "bad wire count %p",
1990 pmap_inval_deinterlock(&info, pmap);
1991 pmap_inval_done(&info);
1995 * The existing process page table was replaced and must be destroyed
2009 * Release any resources held by the given physical map.
2011 * Called when a pmap initialized by pmap_pinit is being released. Should
2012 * only be called if the map contains no valid mappings.
2014 * Caller must hold pmap->pm_token
2016 struct pmap_release_info {
2021 static int pmap_release_callback(pv_entry_t pv, void *data);
2024 pmap_release(struct pmap *pmap)
2026 struct pmap_release_info info;
2028 KASSERT(pmap->pm_active == 0,
2029 ("pmap still active! %016jx", (uintmax_t)pmap->pm_active));
2031 spin_lock(&pmap_spin);
2032 TAILQ_REMOVE(&pmap_list, pmap, pm_pmnode);
2033 spin_unlock(&pmap_spin);
2036 * Pull pv's off the RB tree in order from low to high and release
2042 spin_lock(&pmap->pm_spin);
2043 RB_SCAN(pv_entry_rb_tree, &pmap->pm_pvroot, NULL,
2044 pmap_release_callback, &info);
2045 spin_unlock(&pmap->pm_spin);
2046 } while (info.retry);
2050 * One resident page (the pml4 page) should remain.
2051 * No wired pages should remain.
2053 KKASSERT(pmap->pm_stats.resident_count ==
2054 ((pmap->pm_flags & PMAP_FLAG_SIMPLE) ? 0 : 1));
2056 KKASSERT(pmap->pm_stats.wired_count == 0);
2060 pmap_release_callback(pv_entry_t pv, void *data)
2062 struct pmap_release_info *info = data;
2063 pmap_t pmap = info->pmap;
2066 if (pv_hold_try(pv)) {
2067 spin_unlock(&pmap->pm_spin);
2069 spin_unlock(&pmap->pm_spin);
2071 if (pv->pv_pmap != pmap) {
2073 spin_lock(&pmap->pm_spin);
2078 r = pmap_release_pv(pv, NULL);
2079 spin_lock(&pmap->pm_spin);
2084 * Called with held (i.e. also locked) pv. This function will dispose of
2085 * the lock along with the pv.
2087 * If the caller already holds the locked parent page table for pv it
2088 * must pass it as pvp, allowing us to avoid a deadlock, else it can
2089 * pass NULL for pvp.
2092 pmap_release_pv(pv_entry_t pv, pv_entry_t pvp)
2097 * The pmap is currently not spinlocked, pv is held+locked.
2098 * Remove the pv's page from its parent's page table. The
2099 * parent's page table page's wire_count will be decremented.
2101 pmap_remove_pv_pte(pv, pvp, NULL);
2104 * Terminal pvs are unhooked from their vm_pages. Because
2105 * terminal pages aren't page table pages they aren't wired
2106 * by us, so we have to be sure not to unwire them either.
2108 if (pv->pv_pindex < pmap_pt_pindex(0)) {
2109 pmap_remove_pv_page(pv);
2114 * We leave the top-level page table page cached, wired, and
2115 * mapped in the pmap until the dtor function (pmap_puninit())
2118 * Since we are leaving the top-level pv intact we need
2119 * to break out of what would otherwise be an infinite loop.
2121 if (pv->pv_pindex == pmap_pml4_pindex()) {
2127 * For page table pages (other than the top-level page),
2128 * remove and free the vm_page. The representitive mapping
2129 * removed above by pmap_remove_pv_pte() did not undo the
2130 * last wire_count so we have to do that as well.
2132 p = pmap_remove_pv_page(pv);
2133 vm_page_busy_wait(p, FALSE, "pmaprl");
2134 if (p->wire_count != 1) {
2135 kprintf("p->wire_count was %016lx %d\n",
2136 pv->pv_pindex, p->wire_count);
2138 KKASSERT(p->wire_count == 1);
2139 KKASSERT(p->flags & PG_UNMANAGED);
2141 vm_page_unwire(p, 0);
2142 KKASSERT(p->wire_count == 0);
2145 * Theoretically this page, if not the pml4 page, should contain
2146 * all-zeros. But its just too dangerous to mark it PG_ZERO. Free
2156 * This function will remove the pte associated with a pv from its parent.
2157 * Terminal pv's are supported. The removal will be interlocked if info
2158 * is non-NULL. The caller must dispose of pv instead of just unlocking
2161 * The wire count will be dropped on the parent page table. The wire
2162 * count on the page being removed (pv->pv_m) from the parent page table
2163 * is NOT touched. Note that terminal pages will not have any additional
2164 * wire counts while page table pages will have at least one representing
2165 * the mapping, plus others representing sub-mappings.
2167 * NOTE: Cannot be called on kernel page table pages, only KVM terminal
2168 * pages and user page table and terminal pages.
2170 * The pv must be locked.
2172 * XXX must lock parent pv's if they exist to remove pte XXX
2176 pmap_remove_pv_pte(pv_entry_t pv, pv_entry_t pvp, struct pmap_inval_info *info)
2178 vm_pindex_t ptepindex = pv->pv_pindex;
2179 pmap_t pmap = pv->pv_pmap;
2185 if (ptepindex == pmap_pml4_pindex()) {
2187 * We are the top level pml4 table, there is no parent.
2189 p = pmap->pm_pmlpv->pv_m;
2190 } else if (ptepindex >= pmap_pdp_pindex(0)) {
2192 * Remove a PDP page from the pml4e. This can only occur
2193 * with user page tables. We do not have to lock the
2194 * pml4 PV so just ignore pvp.
2196 vm_pindex_t pml4_pindex;
2197 vm_pindex_t pdp_index;
2200 pdp_index = ptepindex - pmap_pdp_pindex(0);
2202 pml4_pindex = pmap_pml4_pindex();
2203 pvp = pv_get(pv->pv_pmap, pml4_pindex);
2207 pdp = &pmap->pm_pml4[pdp_index & ((1ul << NPML4EPGSHIFT) - 1)];
2208 KKASSERT((*pdp & PG_V) != 0);
2209 p = PHYS_TO_VM_PAGE(*pdp & PG_FRAME);
2211 KKASSERT(info == NULL);
2212 } else if (ptepindex >= pmap_pd_pindex(0)) {
2214 * Remove a PD page from the pdp
2216 * SIMPLE PMAP NOTE: Non-existant pvp's are ok in the case
2217 * of a simple pmap because it stops at
2220 vm_pindex_t pdp_pindex;
2221 vm_pindex_t pd_index;
2224 pd_index = ptepindex - pmap_pd_pindex(0);
2227 pdp_pindex = NUPTE_TOTAL + NUPT_TOTAL + NUPD_TOTAL +
2228 (pd_index >> NPML4EPGSHIFT);
2229 pvp = pv_get(pv->pv_pmap, pdp_pindex);
2234 pd = pv_pte_lookup(pvp, pd_index &
2235 ((1ul << NPDPEPGSHIFT) - 1));
2236 KKASSERT((*pd & PG_V) != 0);
2237 p = PHYS_TO_VM_PAGE(*pd & PG_FRAME);
2240 KKASSERT(pmap->pm_flags & PMAP_FLAG_SIMPLE);
2241 p = pv->pv_m; /* degenerate test later */
2243 KKASSERT(info == NULL);
2244 } else if (ptepindex >= pmap_pt_pindex(0)) {
2246 * Remove a PT page from the pd
2248 vm_pindex_t pd_pindex;
2249 vm_pindex_t pt_index;
2252 pt_index = ptepindex - pmap_pt_pindex(0);
2255 pd_pindex = NUPTE_TOTAL + NUPT_TOTAL +
2256 (pt_index >> NPDPEPGSHIFT);
2257 pvp = pv_get(pv->pv_pmap, pd_pindex);
2261 pt = pv_pte_lookup(pvp, pt_index & ((1ul << NPDPEPGSHIFT) - 1));
2262 KKASSERT((*pt & PG_V) != 0);
2263 p = PHYS_TO_VM_PAGE(*pt & PG_FRAME);
2265 KKASSERT(info == NULL);
2268 * Remove a PTE from the PT page
2270 * NOTE: pv's must be locked bottom-up to avoid deadlocking.
2271 * pv is a pte_pv so we can safely lock pt_pv.
2273 * NOTE: FICTITIOUS pages may have multiple physical mappings
2274 * so PHYS_TO_VM_PAGE() will not necessarily work for
2277 vm_pindex_t pt_pindex;
2282 pt_pindex = ptepindex >> NPTEPGSHIFT;
2283 va = (vm_offset_t)ptepindex << PAGE_SHIFT;
2285 if (ptepindex >= NUPTE_USER) {
2286 ptep = vtopte(ptepindex << PAGE_SHIFT);
2287 KKASSERT(pvp == NULL);
2290 pt_pindex = NUPTE_TOTAL +
2291 (ptepindex >> NPDPEPGSHIFT);
2292 pvp = pv_get(pv->pv_pmap, pt_pindex);
2296 ptep = pv_pte_lookup(pvp, ptepindex &
2297 ((1ul << NPDPEPGSHIFT) - 1));
2301 pmap_inval_interlock(info, pmap, va);
2302 pte = pte_load_clear(ptep);
2304 pmap_inval_deinterlock(info, pmap);
2306 cpu_invlpg((void *)va);
2309 * Now update the vm_page_t
2311 if ((pte & (PG_MANAGED|PG_V)) != (PG_MANAGED|PG_V)) {
2312 kprintf("remove_pte badpte %016lx %016lx %d\n",
2314 pv->pv_pindex < pmap_pt_pindex(0));
2316 /* PHYS_TO_VM_PAGE() will not work for FICTITIOUS pages */
2317 /*KKASSERT((pte & (PG_MANAGED|PG_V)) == (PG_MANAGED|PG_V));*/
2318 if (pte & PG_DEVICE)
2321 p = PHYS_TO_VM_PAGE(pte & PG_FRAME);
2325 if (pmap_track_modified(ptepindex))
2329 vm_page_flag_set(p, PG_REFERENCED);
2332 atomic_add_long(&pmap->pm_stats.wired_count, -1);
2334 cpu_invlpg((void *)va);
2338 * Unwire the parent page table page. The wire_count cannot go below
2339 * 1 here because the parent page table page is itself still mapped.
2341 * XXX remove the assertions later.
2343 KKASSERT(pv->pv_m == p);
2344 if (pvp && vm_page_unwire_quick(pvp->pv_m))
2345 panic("pmap_remove_pv_pte: Insufficient wire_count");
2353 pmap_remove_pv_page(pv_entry_t pv)
2359 vm_page_spin_lock(m);
2361 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
2364 atomic_add_int(&m->object->agg_pv_list_count, -1);
2366 if (TAILQ_EMPTY(&m->md.pv_list))
2367 vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE);
2368 vm_page_spin_unlock(m);
2373 * Grow the number of kernel page table entries, if needed.
2375 * This routine is always called to validate any address space
2376 * beyond KERNBASE (for kldloads). kernel_vm_end only governs the address
2377 * space below KERNBASE.
2380 pmap_growkernel(vm_offset_t kstart, vm_offset_t kend)
2383 vm_offset_t ptppaddr;
2385 pd_entry_t *pt, newpt;
2387 int update_kernel_vm_end;
2390 * bootstrap kernel_vm_end on first real VM use
2392 if (kernel_vm_end == 0) {
2393 kernel_vm_end = VM_MIN_KERNEL_ADDRESS;
2395 while ((*pmap_pt(&kernel_pmap, kernel_vm_end) & PG_V) != 0) {
2396 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) &
2397 ~(PAGE_SIZE * NPTEPG - 1);
2399 if (kernel_vm_end - 1 >= kernel_map.max_offset) {
2400 kernel_vm_end = kernel_map.max_offset;
2407 * Fill in the gaps. kernel_vm_end is only adjusted for ranges
2408 * below KERNBASE. Ranges above KERNBASE are kldloaded and we
2409 * do not want to force-fill 128G worth of page tables.
2411 if (kstart < KERNBASE) {
2412 if (kstart > kernel_vm_end)
2413 kstart = kernel_vm_end;
2414 KKASSERT(kend <= KERNBASE);
2415 update_kernel_vm_end = 1;
2417 update_kernel_vm_end = 0;
2420 kstart = rounddown2(kstart, PAGE_SIZE * NPTEPG);
2421 kend = roundup2(kend, PAGE_SIZE * NPTEPG);
2423 if (kend - 1 >= kernel_map.max_offset)
2424 kend = kernel_map.max_offset;
2426 while (kstart < kend) {
2427 pt = pmap_pt(&kernel_pmap, kstart);
2429 /* We need a new PDP entry */
2430 nkpg = vm_page_alloc(NULL, nkpt,
2433 VM_ALLOC_INTERRUPT);
2435 panic("pmap_growkernel: no memory to grow "
2438 paddr = VM_PAGE_TO_PHYS(nkpg);
2439 if ((nkpg->flags & PG_ZERO) == 0)
2440 pmap_zero_page(paddr);
2441 vm_page_flag_clear(nkpg, PG_ZERO);
2442 newpd = (pdp_entry_t)
2443 (paddr | PG_V | PG_RW | PG_A | PG_M);
2444 *pmap_pd(&kernel_pmap, kstart) = newpd;
2446 continue; /* try again */
2448 if ((*pt & PG_V) != 0) {
2449 kstart = (kstart + PAGE_SIZE * NPTEPG) &
2450 ~(PAGE_SIZE * NPTEPG - 1);
2451 if (kstart - 1 >= kernel_map.max_offset) {
2452 kstart = kernel_map.max_offset;
2459 * This index is bogus, but out of the way
2461 nkpg = vm_page_alloc(NULL, nkpt,
2464 VM_ALLOC_INTERRUPT);
2466 panic("pmap_growkernel: no memory to grow kernel");
2469 ptppaddr = VM_PAGE_TO_PHYS(nkpg);
2470 pmap_zero_page(ptppaddr);
2471 vm_page_flag_clear(nkpg, PG_ZERO);
2472 newpt = (pd_entry_t) (ptppaddr | PG_V | PG_RW | PG_A | PG_M);
2473 *pmap_pt(&kernel_pmap, kstart) = newpt;
2476 kstart = (kstart + PAGE_SIZE * NPTEPG) &
2477 ~(PAGE_SIZE * NPTEPG - 1);
2479 if (kstart - 1 >= kernel_map.max_offset) {
2480 kstart = kernel_map.max_offset;
2486 * Only update kernel_vm_end for areas below KERNBASE.
2488 if (update_kernel_vm_end && kernel_vm_end < kstart)
2489 kernel_vm_end = kstart;
2493 * Add a reference to the specified pmap.
2496 pmap_reference(pmap_t pmap)
2499 lwkt_gettoken(&pmap->pm_token);
2501 lwkt_reltoken(&pmap->pm_token);
2505 /***************************************************
2506 * page management routines.
2507 ***************************************************/
2510 * Hold a pv without locking it
2513 pv_hold(pv_entry_t pv)
2517 if (atomic_cmpset_int(&pv->pv_hold, 0, 1))
2521 count = pv->pv_hold;
2523 if (atomic_cmpset_int(&pv->pv_hold, count, count + 1))
2530 * Hold a pv_entry, preventing its destruction. TRUE is returned if the pv
2531 * was successfully locked, FALSE if it wasn't. The caller must dispose of
2534 * Either the pmap->pm_spin or the related vm_page_spin (if traversing a
2535 * pv list via its page) must be held by the caller.
2538 _pv_hold_try(pv_entry_t pv PMAP_DEBUG_DECL)
2542 if (atomic_cmpset_int(&pv->pv_hold, 0, PV_HOLD_LOCKED | 1)) {
2545 pv->pv_line = lineno;
2551 count = pv->pv_hold;
2553 if ((count & PV_HOLD_LOCKED) == 0) {
2554 if (atomic_cmpset_int(&pv->pv_hold, count,
2555 (count + 1) | PV_HOLD_LOCKED)) {
2558 pv->pv_line = lineno;
2563 if (atomic_cmpset_int(&pv->pv_hold, count, count + 1))
2571 * Drop a previously held pv_entry which could not be locked, allowing its
2574 * Must not be called with a spinlock held as we might zfree() the pv if it
2575 * is no longer associated with a pmap and this was the last hold count.
2578 pv_drop(pv_entry_t pv)
2582 if (atomic_cmpset_int(&pv->pv_hold, 1, 0)) {
2583 if (pv->pv_pmap == NULL)
2589 count = pv->pv_hold;
2591 KKASSERT((count & PV_HOLD_MASK) > 0);
2592 KKASSERT((count & (PV_HOLD_LOCKED | PV_HOLD_MASK)) !=
2593 (PV_HOLD_LOCKED | 1));
2594 if (atomic_cmpset_int(&pv->pv_hold, count, count - 1)) {
2595 if (count == 1 && pv->pv_pmap == NULL)
2604 * Find or allocate the requested PV entry, returning a locked pv
2608 _pv_alloc(pmap_t pmap, vm_pindex_t pindex, int *isnew PMAP_DEBUG_DECL)
2611 pv_entry_t pnew = NULL;
2613 spin_lock(&pmap->pm_spin);
2615 if ((pv = pmap->pm_pvhint) == NULL || pv->pv_pindex != pindex) {
2616 pv = pv_entry_rb_tree_RB_LOOKUP(&pmap->pm_pvroot,
2621 spin_unlock(&pmap->pm_spin);
2622 pnew = zalloc(pvzone);
2623 spin_lock(&pmap->pm_spin);
2626 pnew->pv_pmap = pmap;
2627 pnew->pv_pindex = pindex;
2628 pnew->pv_hold = PV_HOLD_LOCKED | 1;
2630 pnew->pv_func = func;
2631 pnew->pv_line = lineno;
2633 pv_entry_rb_tree_RB_INSERT(&pmap->pm_pvroot, pnew);
2634 atomic_add_long(&pmap->pm_stats.resident_count, 1);
2635 spin_unlock(&pmap->pm_spin);
2640 spin_unlock(&pmap->pm_spin);
2641 zfree(pvzone, pnew);
2643 spin_lock(&pmap->pm_spin);
2646 if (_pv_hold_try(pv PMAP_DEBUG_COPY)) {
2647 spin_unlock(&pmap->pm_spin);
2651 spin_unlock(&pmap->pm_spin);
2652 _pv_lock(pv PMAP_DEBUG_COPY);
2653 if (pv->pv_pmap == pmap && pv->pv_pindex == pindex) {
2658 spin_lock(&pmap->pm_spin);
2665 * Find the requested PV entry, returning a locked+held pv or NULL
2669 _pv_get(pmap_t pmap, vm_pindex_t pindex PMAP_DEBUG_DECL)
2673 spin_lock(&pmap->pm_spin);
2678 if ((pv = pmap->pm_pvhint) == NULL || pv->pv_pindex != pindex) {
2679 pv = pv_entry_rb_tree_RB_LOOKUP(&pmap->pm_pvroot,
2683 spin_unlock(&pmap->pm_spin);
2686 if (_pv_hold_try(pv PMAP_DEBUG_COPY)) {
2687 pv_cache(pv, pindex);
2688 spin_unlock(&pmap->pm_spin);
2691 spin_unlock(&pmap->pm_spin);
2692 _pv_lock(pv PMAP_DEBUG_COPY);
2693 if (pv->pv_pmap == pmap && pv->pv_pindex == pindex)
2696 spin_lock(&pmap->pm_spin);
2701 * Lookup, hold, and attempt to lock (pmap,pindex).
2703 * If the entry does not exist NULL is returned and *errorp is set to 0
2705 * If the entry exists and could be successfully locked it is returned and
2706 * errorp is set to 0.
2708 * If the entry exists but could NOT be successfully locked it is returned
2709 * held and *errorp is set to 1.
2713 pv_get_try(pmap_t pmap, vm_pindex_t pindex, int *errorp)
2717 spin_lock(&pmap->pm_spin);
2718 if ((pv = pmap->pm_pvhint) == NULL || pv->pv_pindex != pindex)
2719 pv = pv_entry_rb_tree_RB_LOOKUP(&pmap->pm_pvroot, pindex);
2721 spin_unlock(&pmap->pm_spin);
2725 if (pv_hold_try(pv)) {
2726 pv_cache(pv, pindex);
2727 spin_unlock(&pmap->pm_spin);
2729 return(pv); /* lock succeeded */
2731 spin_unlock(&pmap->pm_spin);
2733 return (pv); /* lock failed */
2737 * Find the requested PV entry, returning a held pv or NULL
2741 pv_find(pmap_t pmap, vm_pindex_t pindex)
2745 spin_lock(&pmap->pm_spin);
2747 if ((pv = pmap->pm_pvhint) == NULL || pv->pv_pindex != pindex)
2748 pv = pv_entry_rb_tree_RB_LOOKUP(&pmap->pm_pvroot, pindex);
2750 spin_unlock(&pmap->pm_spin);
2754 pv_cache(pv, pindex);
2755 spin_unlock(&pmap->pm_spin);
2760 * Lock a held pv, keeping the hold count
2764 _pv_lock(pv_entry_t pv PMAP_DEBUG_DECL)
2769 count = pv->pv_hold;
2771 if ((count & PV_HOLD_LOCKED) == 0) {
2772 if (atomic_cmpset_int(&pv->pv_hold, count,
2773 count | PV_HOLD_LOCKED)) {
2776 pv->pv_line = lineno;
2782 tsleep_interlock(pv, 0);
2783 if (atomic_cmpset_int(&pv->pv_hold, count,
2784 count | PV_HOLD_WAITING)) {
2786 kprintf("pv waiting on %s:%d\n",
2787 pv->pv_func, pv->pv_line);
2789 tsleep(pv, PINTERLOCKED, "pvwait", hz);
2796 * Unlock a held and locked pv, keeping the hold count.
2800 pv_unlock(pv_entry_t pv)
2804 if (atomic_cmpset_int(&pv->pv_hold, PV_HOLD_LOCKED | 1, 1))
2808 count = pv->pv_hold;
2810 KKASSERT((count & (PV_HOLD_LOCKED|PV_HOLD_MASK)) >=
2811 (PV_HOLD_LOCKED | 1));
2812 if (atomic_cmpset_int(&pv->pv_hold, count,
2814 ~(PV_HOLD_LOCKED | PV_HOLD_WAITING))) {
2815 if (count & PV_HOLD_WAITING)
2823 * Unlock and drop a pv. If the pv is no longer associated with a pmap
2824 * and the hold count drops to zero we will free it.
2826 * Caller should not hold any spin locks. We are protected from hold races
2827 * by virtue of holds only occuring only with a pmap_spin or vm_page_spin
2828 * lock held. A pv cannot be located otherwise.
2832 pv_put(pv_entry_t pv)
2834 if (atomic_cmpset_int(&pv->pv_hold, PV_HOLD_LOCKED | 1, 0)) {
2835 if (pv->pv_pmap == NULL)
2844 * Unlock, drop, and free a pv, destroying it. The pv is removed from its
2845 * pmap. Any pte operations must have already been completed.
2849 pv_free(pv_entry_t pv)
2853 KKASSERT(pv->pv_m == NULL);
2854 if ((pmap = pv->pv_pmap) != NULL) {
2855 spin_lock(&pmap->pm_spin);
2856 pv_entry_rb_tree_RB_REMOVE(&pmap->pm_pvroot, pv);
2857 if (pmap->pm_pvhint == pv)
2858 pmap->pm_pvhint = NULL;
2859 atomic_add_long(&pmap->pm_stats.resident_count, -1);
2862 spin_unlock(&pmap->pm_spin);
2868 * This routine is very drastic, but can save the system
2876 static int warningdone=0;
2878 if (pmap_pagedaemon_waken == 0)
2880 pmap_pagedaemon_waken = 0;
2881 if (warningdone < 5) {
2882 kprintf("pmap_collect: collecting pv entries -- "
2883 "suggest increasing PMAP_SHPGPERPROC\n");
2887 for (i = 0; i < vm_page_array_size; i++) {
2888 m = &vm_page_array[i];
2889 if (m->wire_count || m->hold_count)
2891 if (vm_page_busy_try(m, TRUE) == 0) {
2892 if (m->wire_count == 0 && m->hold_count == 0) {
2901 * Scan the pmap for active page table entries and issue a callback.
2902 * The callback must dispose of pte_pv, whos PTE entry is at *ptep in
2903 * its parent page table.
2905 * pte_pv will be NULL if the page or page table is unmanaged.
2906 * pt_pv will point to the page table page containing the pte for the page.
2908 * NOTE! If we come across an unmanaged page TABLE (verses an unmanaged page),
2909 * we pass a NULL pte_pv and we pass a pt_pv pointing to the passed
2910 * process pmap's PD and page to the callback function. This can be
2911 * confusing because the pt_pv is really a pd_pv, and the target page
2912 * table page is simply aliased by the pmap and not owned by it.
2914 * It is assumed that the start and end are properly rounded to the page size.
2916 * It is assumed that PD pages and above are managed and thus in the RB tree,
2917 * allowing us to use RB_SCAN from the PD pages down for ranged scans.
2919 struct pmap_scan_info {
2923 vm_pindex_t sva_pd_pindex;
2924 vm_pindex_t eva_pd_pindex;
2925 void (*func)(pmap_t, struct pmap_scan_info *,
2926 pv_entry_t, pv_entry_t, int, vm_offset_t,
2927 pt_entry_t *, void *);
2930 struct pmap_inval_info inval;
2933 static int pmap_scan_cmp(pv_entry_t pv, void *data);
2934 static int pmap_scan_callback(pv_entry_t pv, void *data);
2937 pmap_scan(struct pmap_scan_info *info)
2939 struct pmap *pmap = info->pmap;
2940 pv_entry_t pd_pv; /* A page directory PV */
2941 pv_entry_t pt_pv; /* A page table PV */
2942 pv_entry_t pte_pv; /* A page table entry PV */
2944 struct pv_entry dummy_pv;
2950 * Hold the token for stability; if the pmap is empty we have nothing
2953 lwkt_gettoken(&pmap->pm_token);
2955 if (pmap->pm_stats.resident_count == 0) {
2956 lwkt_reltoken(&pmap->pm_token);
2961 pmap_inval_init(&info->inval);
2964 * Special handling for scanning one page, which is a very common
2965 * operation (it is?).
2967 * NOTE: Locks must be ordered bottom-up. pte,pt,pd,pdp,pml4
2969 if (info->sva + PAGE_SIZE == info->eva) {
2970 if (info->sva >= VM_MAX_USER_ADDRESS) {
2972 * Kernel mappings do not track wire counts on
2973 * page table pages and only maintain pd_pv and
2974 * pte_pv levels so pmap_scan() works.
2977 pte_pv = pv_get(pmap, pmap_pte_pindex(info->sva));
2978 ptep = vtopte(info->sva);
2981 * User pages which are unmanaged will not have a
2982 * pte_pv. User page table pages which are unmanaged
2983 * (shared from elsewhere) will also not have a pt_pv.
2984 * The func() callback will pass both pte_pv and pt_pv
2985 * as NULL in that case.
2987 pte_pv = pv_get(pmap, pmap_pte_pindex(info->sva));
2988 pt_pv = pv_get(pmap, pmap_pt_pindex(info->sva));
2989 if (pt_pv == NULL) {
2990 KKASSERT(pte_pv == NULL);
2991 pd_pv = pv_get(pmap, pmap_pd_pindex(info->sva));
2993 ptep = pv_pte_lookup(pd_pv,
2994 pmap_pt_index(info->sva));
2996 info->func(pmap, info,
3005 ptep = pv_pte_lookup(pt_pv, pmap_pte_index(info->sva));
3009 * Unlike the pv_find() case below we actually
3010 * acquired a locked pv in this case so any
3011 * race should have been resolved. It is expected
3014 KKASSERT(pte_pv == NULL);
3015 } else if (pte_pv) {
3016 KASSERT((*ptep & (PG_MANAGED|PG_V)) == (PG_MANAGED|
3018 ("bad *ptep %016lx sva %016lx pte_pv %p",
3019 *ptep, info->sva, pte_pv));
3020 info->func(pmap, info, pte_pv, pt_pv, 0,
3021 info->sva, ptep, info->arg);
3023 KASSERT((*ptep & (PG_MANAGED|PG_V)) == PG_V,
3024 ("bad *ptep %016lx sva %016lx pte_pv NULL",
3026 info->func(pmap, info, NULL, pt_pv, 0,
3027 info->sva, ptep, info->arg);
3032 pmap_inval_done(&info->inval);
3033 lwkt_reltoken(&pmap->pm_token);
3038 * Nominal scan case, RB_SCAN() for PD pages and iterate from
3041 info->sva_pd_pindex = pmap_pd_pindex(info->sva);
3042 info->eva_pd_pindex = pmap_pd_pindex(info->eva + NBPDP - 1);
3044 if (info->sva >= VM_MAX_USER_ADDRESS) {
3046 * The kernel does not currently maintain any pv_entry's for
3047 * higher-level page tables.
3049 bzero(&dummy_pv, sizeof(dummy_pv));
3050 dummy_pv.pv_pindex = info->sva_pd_pindex;
3051 spin_lock(&pmap->pm_spin);
3052 while (dummy_pv.pv_pindex < info->eva_pd_pindex) {
3053 pmap_scan_callback(&dummy_pv, info);
3054 ++dummy_pv.pv_pindex;
3056 spin_unlock(&pmap->pm_spin);
3059 * User page tables maintain local PML4, PDP, and PD
3060 * pv_entry's at the very least. PT pv's might be
3061 * unmanaged and thus not exist. PTE pv's might be
3062 * unmanaged and thus not exist.
3064 spin_lock(&pmap->pm_spin);
3065 pv_entry_rb_tree_RB_SCAN(&pmap->pm_pvroot,
3066 pmap_scan_cmp, pmap_scan_callback, info);
3067 spin_unlock(&pmap->pm_spin);
3069 pmap_inval_done(&info->inval);
3070 lwkt_reltoken(&pmap->pm_token);
3074 * WARNING! pmap->pm_spin held
3077 pmap_scan_cmp(pv_entry_t pv, void *data)
3079 struct pmap_scan_info *info = data;
3080 if (pv->pv_pindex < info->sva_pd_pindex)
3082 if (pv->pv_pindex >= info->eva_pd_pindex)
3088 * WARNING! pmap->pm_spin held
3091 pmap_scan_callback(pv_entry_t pv, void *data)
3093 struct pmap_scan_info *info = data;
3094 struct pmap *pmap = info->pmap;
3095 pv_entry_t pd_pv; /* A page directory PV */
3096 pv_entry_t pt_pv; /* A page table PV */
3097 pv_entry_t pte_pv; /* A page table entry PV */
3101 vm_offset_t va_next;
3102 vm_pindex_t pd_pindex;
3106 * Pull the PD pindex from the pv before releasing the spinlock.
3108 * WARNING: pv is faked for kernel pmap scans.
3110 pd_pindex = pv->pv_pindex;
3111 spin_unlock(&pmap->pm_spin);
3112 pv = NULL; /* invalid after spinlock unlocked */
3115 * Calculate the page range within the PD. SIMPLE pmaps are
3116 * direct-mapped for the entire 2^64 address space. Normal pmaps
3117 * reflect the user and kernel address space which requires
3118 * cannonicalization w/regards to converting pd_pindex's back
3121 sva = (pd_pindex - NUPTE_TOTAL - NUPT_TOTAL) << PDPSHIFT;
3122 if ((pmap->pm_flags & PMAP_FLAG_SIMPLE) == 0 &&
3123 (sva & PML4_SIGNMASK)) {
3124 sva |= PML4_SIGNMASK;
3126 eva = sva + NBPDP; /* can overflow */
3127 if (sva < info->sva)
3129 if (eva < info->sva || eva > info->eva)
3133 * NOTE: kernel mappings do not track page table pages, only
3136 * NOTE: Locks must be ordered bottom-up. pte,pt,pd,pdp,pml4.
3137 * However, for the scan to be efficient we try to
3138 * cache items top-down.
3143 for (; sva < eva; sva = va_next) {
3144 if (sva >= VM_MAX_USER_ADDRESS) {
3153 * PD cache (degenerate case if we skip). It is possible
3154 * for the PD to not exist due to races. This is ok.
3156 if (pd_pv == NULL) {
3157 pd_pv = pv_get(pmap, pmap_pd_pindex(sva));
3158 } else if (pd_pv->pv_pindex != pmap_pd_pindex(sva)) {
3160 pd_pv = pv_get(pmap, pmap_pd_pindex(sva));
3162 if (pd_pv == NULL) {
3163 va_next = (sva + NBPDP) & ~PDPMASK;
3172 if (pt_pv == NULL) {
3177 pt_pv = pv_get(pmap, pmap_pt_pindex(sva));
3178 } else if (pt_pv->pv_pindex != pmap_pt_pindex(sva)) {
3184 pt_pv = pv_get(pmap, pmap_pt_pindex(sva));
3188 * If pt_pv is NULL we either have an shared page table
3189 * page and must issue a callback specific to that case,
3190 * or there is no page table page.
3192 * Either way we can skip the page table page.
3194 if (pt_pv == NULL) {
3196 * Possible unmanaged (shared from another pmap)
3200 pd_pv = pv_get(pmap, pmap_pd_pindex(sva));
3201 KKASSERT(pd_pv != NULL);
3202 ptep = pv_pte_lookup(pd_pv, pmap_pt_index(sva));
3204 info->func(pmap, info, NULL, pd_pv, 1,
3205 sva, ptep, info->arg);
3209 * Done, move to next page table page.
3211 va_next = (sva + NBPDR) & ~PDRMASK;
3218 * From this point in the loop testing pt_pv for non-NULL
3219 * means we are in UVM, else if it is NULL we are in KVM.
3221 * Limit our scan to either the end of the va represented
3222 * by the current page table page, or to the end of the
3223 * range being removed.
3226 va_next = (sva + NBPDR) & ~PDRMASK;
3233 * Scan the page table for pages. Some pages may not be
3234 * managed (might not have a pv_entry).
3236 * There is no page table management for kernel pages so
3237 * pt_pv will be NULL in that case, but otherwise pt_pv
3238 * is non-NULL, locked, and referenced.
3242 * At this point a non-NULL pt_pv means a UVA, and a NULL
3243 * pt_pv means a KVA.
3246 ptep = pv_pte_lookup(pt_pv, pmap_pte_index(sva));
3250 while (sva < va_next) {
3252 * Acquire the related pte_pv, if any. If *ptep == 0
3253 * the related pte_pv should not exist, but if *ptep
3254 * is not zero the pte_pv may or may not exist (e.g.
3255 * will not exist for an unmanaged page).
3257 * However a multitude of races are possible here.
3259 * In addition, the (pt_pv, pte_pv) lock order is
3260 * backwards, so we have to be careful in aquiring
3261 * a properly locked pte_pv.
3264 pte_pv = pv_get_try(pmap, pmap_pte_pindex(sva),
3271 pv_put(pt_pv); /* must be non-NULL */
3273 pv_lock(pte_pv); /* safe to block now */
3276 pt_pv = pv_get(pmap,
3277 pmap_pt_pindex(sva));
3279 * pt_pv reloaded, need new ptep
3281 KKASSERT(pt_pv != NULL);
3282 ptep = pv_pte_lookup(pt_pv,
3283 pmap_pte_index(sva));
3287 pte_pv = pv_get(pmap, pmap_pte_pindex(sva));
3291 * Ok, if *ptep == 0 we had better NOT have a pte_pv.
3295 kprintf("Unexpected non-NULL pte_pv "
3296 "%p pt_pv %p *ptep = %016lx\n",
3297 pte_pv, pt_pv, *ptep);
3298 panic("Unexpected non-NULL pte_pv");
3306 * Ready for the callback. The locked pte_pv (if any)
3307 * is consumed by the callback. pte_pv will exist if
3308 * the page is managed, and will not exist if it
3312 KASSERT((*ptep & (PG_MANAGED|PG_V)) ==
3314 ("bad *ptep %016lx sva %016lx "
3316 *ptep, sva, pte_pv));
3317 info->func(pmap, info, pte_pv, pt_pv, 0,
3318 sva, ptep, info->arg);
3320 KASSERT((*ptep & (PG_MANAGED|PG_V)) ==
3322 ("bad *ptep %016lx sva %016lx "
3325 info->func(pmap, info, NULL, pt_pv, 0,
3326 sva, ptep, info->arg);
3345 * Relock before returning.
3347 spin_lock(&pmap->pm_spin);
3352 pmap_remove(struct pmap *pmap, vm_offset_t sva, vm_offset_t eva)
3354 struct pmap_scan_info info;
3359 info.func = pmap_remove_callback;
3361 info.doinval = 1; /* normal remove requires pmap inval */
3366 pmap_remove_noinval(struct pmap *pmap, vm_offset_t sva, vm_offset_t eva)
3368 struct pmap_scan_info info;
3373 info.func = pmap_remove_callback;
3375 info.doinval = 0; /* normal remove requires pmap inval */
3380 pmap_remove_callback(pmap_t pmap, struct pmap_scan_info *info,
3381 pv_entry_t pte_pv, pv_entry_t pt_pv, int sharept,
3382 vm_offset_t va, pt_entry_t *ptep, void *arg __unused)
3388 * This will also drop pt_pv's wire_count. Note that
3389 * terminal pages are not wired based on mmu presence.
3392 pmap_remove_pv_pte(pte_pv, pt_pv, &info->inval);
3394 pmap_remove_pv_pte(pte_pv, pt_pv, NULL);
3395 pmap_remove_pv_page(pte_pv);
3397 } else if (sharept == 0) {
3401 * pt_pv's wire_count is still bumped by unmanaged pages
3402 * so we must decrement it manually.
3405 pmap_inval_interlock(&info->inval, pmap, va);
3406 pte = pte_load_clear(ptep);
3408 pmap_inval_deinterlock(&info->inval, pmap);
3410 atomic_add_long(&pmap->pm_stats.wired_count, -1);
3411 atomic_add_long(&pmap->pm_stats.resident_count, -1);
3412 if (vm_page_unwire_quick(pt_pv->pv_m))
3413 panic("pmap_remove: insufficient wirecount");
3416 * Unmanaged page table, pt_pv is actually the pd_pv
3417 * for our pmap (not the share object pmap).
3419 * We have to unwire the target page table page and we
3420 * have to unwire our page directory page.
3423 pmap_inval_interlock(&info->inval, pmap, va);
3424 pte = pte_load_clear(ptep);
3426 pmap_inval_deinterlock(&info->inval, pmap);
3427 atomic_add_long(&pmap->pm_stats.resident_count, -1);
3428 KKASSERT((pte & PG_DEVICE) == 0);
3429 if (vm_page_unwire_quick(PHYS_TO_VM_PAGE(pte & PG_FRAME)))
3430 panic("pmap_remove: shared pgtable1 bad wirecount");
3431 if (vm_page_unwire_quick(pt_pv->pv_m))
3432 panic("pmap_remove: shared pgtable2 bad wirecount");
3437 * Removes this physical page from all physical maps in which it resides.
3438 * Reflects back modify bits to the pager.
3440 * This routine may not be called from an interrupt.
3444 pmap_remove_all(vm_page_t m)
3446 struct pmap_inval_info info;
3449 if (!pmap_initialized /* || (m->flags & PG_FICTITIOUS)*/)
3452 pmap_inval_init(&info);
3453 vm_page_spin_lock(m);
3454 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
3455 KKASSERT(pv->pv_m == m);
3456 if (pv_hold_try(pv)) {
3457 vm_page_spin_unlock(m);
3459 vm_page_spin_unlock(m);
3461 if (pv->pv_m != m) {
3463 vm_page_spin_lock(m);
3468 * Holding no spinlocks, pv is locked.
3470 pmap_remove_pv_pte(pv, NULL, &info);
3471 pmap_remove_pv_page(pv);
3473 vm_page_spin_lock(m);
3475 KKASSERT((m->flags & (PG_MAPPED|PG_WRITEABLE)) == 0);
3476 vm_page_spin_unlock(m);
3477 pmap_inval_done(&info);
3481 * Set the physical protection on the specified range of this map
3482 * as requested. This function is typically only used for debug watchpoints
3485 * This function may not be called from an interrupt if the map is
3486 * not the kernel_pmap.
3488 * NOTE! For shared page table pages we just unmap the page.
3491 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
3493 struct pmap_scan_info info;
3494 /* JG review for NX */
3498 if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
3499 pmap_remove(pmap, sva, eva);
3502 if (prot & VM_PROT_WRITE)
3507 info.func = pmap_protect_callback;
3515 pmap_protect_callback(pmap_t pmap, struct pmap_scan_info *info,
3516 pv_entry_t pte_pv, pv_entry_t pt_pv, int sharept,
3517 vm_offset_t va, pt_entry_t *ptep, void *arg __unused)
3527 pmap_inval_interlock(&info->inval, pmap, va);
3534 if ((pbits & PG_DEVICE) == 0) {
3535 m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
3536 KKASSERT(m == pte_pv->pv_m);
3537 vm_page_flag_set(m, PG_REFERENCED);
3542 if (pmap_track_modified(pte_pv->pv_pindex)) {
3543 if ((pbits & PG_DEVICE) == 0) {
3545 m = PHYS_TO_VM_PAGE(pbits &
3553 } else if (sharept) {
3555 * Unmanaged page table, pt_pv is actually the pd_pv
3556 * for our pmap (not the share object pmap).
3558 * When asked to protect something in a shared page table
3559 * page we just unmap the page table page. We have to
3560 * invalidate the tlb in this situation.
3562 * XXX Warning, shared page tables will not be used for
3563 * OBJT_DEVICE or OBJT_MGTDEVICE (PG_FICTITIOUS) mappings
3564 * so PHYS_TO_VM_PAGE() should be safe here.
3566 pte = pte_load_clear(ptep);
3567 pmap_inval_invltlb(&info->inval);
3568 if (vm_page_unwire_quick(PHYS_TO_VM_PAGE(pte & PG_FRAME)))
3569 panic("pmap_protect: pgtable1 pg bad wirecount");
3570 if (vm_page_unwire_quick(pt_pv->pv_m))
3571 panic("pmap_protect: pgtable2 pg bad wirecount");
3574 /* else unmanaged page, adjust bits, no wire changes */
3578 if (pbits != cbits && !atomic_cmpset_long(ptep, pbits, cbits)) {
3582 pmap_inval_deinterlock(&info->inval, pmap);
3588 * Insert the vm_page (m) at the virtual address (va), replacing any prior
3589 * mapping at that address. Set protection and wiring as requested.
3591 * If entry is non-NULL we check to see if the SEG_SIZE optimization is
3592 * possible. If it is we enter the page into the appropriate shared pmap
3593 * hanging off the related VM object instead of the passed pmap, then we
3594 * share the page table page from the VM object's pmap into the current pmap.
3596 * NOTE: This routine MUST insert the page into the pmap now, it cannot
3600 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
3601 boolean_t wired, vm_map_entry_t entry)
3603 pmap_inval_info info;
3604 pv_entry_t pt_pv; /* page table */
3605 pv_entry_t pte_pv; /* page table entry */
3608 pt_entry_t origpte, newpte;
3613 va = trunc_page(va);
3614 #ifdef PMAP_DIAGNOSTIC
3616 panic("pmap_enter: toobig");
3617 if ((va >= UPT_MIN_ADDRESS) && (va < UPT_MAX_ADDRESS))
3618 panic("pmap_enter: invalid to pmap_enter page table "
3619 "pages (va: 0x%lx)", va);
3621 if (va < UPT_MAX_ADDRESS && pmap == &kernel_pmap) {
3622 kprintf("Warning: pmap_enter called on UVA with "
3625 db_print_backtrace();
3628 if (va >= UPT_MAX_ADDRESS && pmap != &kernel_pmap) {
3629 kprintf("Warning: pmap_enter called on KVA without"
3632 db_print_backtrace();
3637 * Get locked PV entries for our new page table entry (pte_pv)
3638 * and for its parent page table (pt_pv). We need the parent
3639 * so we can resolve the location of the ptep.
3641 * Only hardware MMU actions can modify the ptep out from
3644 * if (m) is fictitious or unmanaged we do not create a managing
3645 * pte_pv for it. Any pre-existing page's management state must
3646 * match (avoiding code complexity).
3648 * If the pmap is still being initialized we assume existing
3651 * Kernel mapppings do not track page table pages (i.e. pt_pv).
3652 * pmap_allocpte() checks the
3654 if (pmap_initialized == FALSE) {
3658 } else if (m->flags & (/*PG_FICTITIOUS |*/ PG_UNMANAGED)) { /* XXX */
3660 if (va >= VM_MAX_USER_ADDRESS) {
3664 pt_pv = pmap_allocpte_seg(pmap, pmap_pt_pindex(va),
3666 ptep = pv_pte_lookup(pt_pv, pmap_pte_index(va));
3668 KKASSERT(*ptep == 0 || (*ptep & PG_MANAGED) == 0);
3670 if (va >= VM_MAX_USER_ADDRESS) {
3672 * Kernel map, pv_entry-tracked.
3675 pte_pv = pmap_allocpte(pmap, pmap_pte_pindex(va), NULL);
3681 pte_pv = pmap_allocpte_seg(pmap, pmap_pte_pindex(va),
3683 ptep = pv_pte_lookup(pt_pv, pmap_pte_index(va));
3685 KKASSERT(*ptep == 0 || (*ptep & PG_MANAGED));
3688 pa = VM_PAGE_TO_PHYS(m);
3690 opa = origpte & PG_FRAME;
3692 newpte = (pt_entry_t)(pa | pte_prot(pmap, prot) | PG_V | PG_A);
3695 if (va < VM_MAX_USER_ADDRESS)
3698 newpte |= PG_MANAGED;
3699 if (pmap == &kernel_pmap)
3701 newpte |= pat_pte_index[m->pat_mode];
3702 if (m->flags & PG_FICTITIOUS)
3703 newpte |= PG_DEVICE;
3706 * It is possible for multiple faults to occur in threaded
3707 * environments, the existing pte might be correct.
3709 if (((origpte ^ newpte) & ~(pt_entry_t)(PG_M|PG_A)) == 0)
3712 if ((prot & VM_PROT_NOSYNC) == 0)
3713 pmap_inval_init(&info);
3716 * Ok, either the address changed or the protection or wiring
3719 * Clear the current entry, interlocking the removal. For managed
3720 * pte's this will also flush the modified state to the vm_page.
3721 * Atomic ops are mandatory in order to ensure that PG_M events are
3722 * not lost during any transition.
3727 * pmap_remove_pv_pte() unwires pt_pv and assumes
3728 * we will free pte_pv, but since we are reusing
3729 * pte_pv we want to retain the wire count.
3731 * pt_pv won't exist for a kernel page (managed or
3735 vm_page_wire_quick(pt_pv->pv_m);
3736 if (prot & VM_PROT_NOSYNC)
3737 pmap_remove_pv_pte(pte_pv, pt_pv, NULL);
3739 pmap_remove_pv_pte(pte_pv, pt_pv, &info);
3741 pmap_remove_pv_page(pte_pv);
3742 } else if (prot & VM_PROT_NOSYNC) {
3744 * Unmanaged page, NOSYNC (no mmu sync) requested.
3746 * Leave wire count on PT page intact.
3748 (void)pte_load_clear(ptep);
3749 cpu_invlpg((void *)va);
3750 atomic_add_long(&pmap->pm_stats.resident_count, -1);
3753 * Unmanaged page, normal enter.
3755 * Leave wire count on PT page intact.
3757 pmap_inval_interlock(&info, pmap, va);
3758 (void)pte_load_clear(ptep);
3759 pmap_inval_deinterlock(&info, pmap);
3760 atomic_add_long(&pmap->pm_stats.resident_count, -1);
3762 KKASSERT(*ptep == 0);
3767 * Enter on the PV list if part of our managed memory.
3768 * Wiring of the PT page is already handled.
3770 KKASSERT(pte_pv->pv_m == NULL);
3771 vm_page_spin_lock(m);
3773 TAILQ_INSERT_TAIL(&m->md.pv_list, pte_pv, pv_list);
3776 atomic_add_int(&m->object->agg_pv_list_count, 1);
3778 vm_page_flag_set(m, PG_MAPPED);
3779 vm_page_spin_unlock(m);
3780 } else if (pt_pv && opa == 0) {
3782 * We have to adjust the wire count on the PT page ourselves
3783 * for unmanaged entries. If opa was non-zero we retained
3784 * the existing wire count from the removal.
3786 vm_page_wire_quick(pt_pv->pv_m);
3790 * Kernel VMAs (pt_pv == NULL) require pmap invalidation interlocks.
3792 * User VMAs do not because those will be zero->non-zero, so no
3793 * stale entries to worry about at this point.
3795 * For KVM there appear to still be issues. Theoretically we
3796 * should be able to scrap the interlocks entirely but we
3799 if ((prot & VM_PROT_NOSYNC) == 0 && pt_pv == NULL)
3800 pmap_inval_interlock(&info, pmap, va);
3805 *(volatile pt_entry_t *)ptep = newpte;
3807 if ((prot & VM_PROT_NOSYNC) == 0 && pt_pv == NULL)
3808 pmap_inval_deinterlock(&info, pmap);
3809 else if (pt_pv == NULL)
3810 cpu_invlpg((void *)va);
3814 atomic_add_long(&pte_pv->pv_pmap->pm_stats.wired_count,
3817 atomic_add_long(&pmap->pm_stats.wired_count, 1);
3821 vm_page_flag_set(m, PG_WRITEABLE);
3824 * Unmanaged pages need manual resident_count tracking.
3826 if (pte_pv == NULL && pt_pv)
3827 atomic_add_long(&pt_pv->pv_pmap->pm_stats.resident_count, 1);
3832 if ((prot & VM_PROT_NOSYNC) == 0 || pte_pv == NULL)
3833 pmap_inval_done(&info);
3835 KKASSERT((newpte & PG_MANAGED) == 0 || (m->flags & PG_MAPPED));
3838 * Cleanup the pv entry, allowing other accessors.
3847 * This code works like pmap_enter() but assumes VM_PROT_READ and not-wired.
3848 * This code also assumes that the pmap has no pre-existing entry for this
3851 * This code currently may only be used on user pmaps, not kernel_pmap.
3854 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m)
3856 pmap_enter(pmap, va, m, VM_PROT_READ, FALSE, NULL);
3860 * Make a temporary mapping for a physical address. This is only intended
3861 * to be used for panic dumps.
3863 * The caller is responsible for calling smp_invltlb().
3866 pmap_kenter_temporary(vm_paddr_t pa, long i)
3868 pmap_kenter_quick((vm_offset_t)crashdumpmap + (i * PAGE_SIZE), pa);
3869 return ((void *)crashdumpmap);
3872 #define MAX_INIT_PT (96)
3875 * This routine preloads the ptes for a given object into the specified pmap.
3876 * This eliminates the blast of soft faults on process startup and
3877 * immediately after an mmap.
3879 static int pmap_object_init_pt_callback(vm_page_t p, void *data);
3882 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_prot_t prot,
3883 vm_object_t object, vm_pindex_t pindex,
3884 vm_size_t size, int limit)
3886 struct rb_vm_page_scan_info info;
3891 * We can't preinit if read access isn't set or there is no pmap
3894 if ((prot & VM_PROT_READ) == 0 || pmap == NULL || object == NULL)
3898 * We can't preinit if the pmap is not the current pmap
3900 lp = curthread->td_lwp;
3901 if (lp == NULL || pmap != vmspace_pmap(lp->lwp_vmspace))
3905 * Misc additional checks
3907 psize = x86_64_btop(size);
3909 if ((object->type != OBJT_VNODE) ||
3910 ((limit & MAP_PREFAULT_PARTIAL) && (psize > MAX_INIT_PT) &&
3911 (object->resident_page_count > MAX_INIT_PT))) {
3915 if (pindex + psize > object->size) {
3916 if (object->size < pindex)
3918 psize = object->size - pindex;
3925 * If everything is segment-aligned do not pre-init here. Instead
3926 * allow the normal vm_fault path to pass a segment hint to
3927 * pmap_enter() which will then use an object-referenced shared
3930 if ((addr & SEG_MASK) == 0 &&
3931 (ctob(psize) & SEG_MASK) == 0 &&
3932 (ctob(pindex) & SEG_MASK) == 0) {
3937 * Use a red-black scan to traverse the requested range and load
3938 * any valid pages found into the pmap.
3940 * We cannot safely scan the object's memq without holding the
3943 info.start_pindex = pindex;
3944 info.end_pindex = pindex + psize - 1;
3950 vm_object_hold_shared(object);
3951 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp,
3952 pmap_object_init_pt_callback, &info);
3953 vm_object_drop(object);
3958 pmap_object_init_pt_callback(vm_page_t p, void *data)
3960 struct rb_vm_page_scan_info *info = data;
3961 vm_pindex_t rel_index;
3964 * don't allow an madvise to blow away our really
3965 * free pages allocating pv entries.
3967 if ((info->limit & MAP_PREFAULT_MADVISE) &&
3968 vmstats.v_free_count < vmstats.v_free_reserved) {
3973 * Ignore list markers and ignore pages we cannot instantly
3974 * busy (while holding the object token).
3976 if (p->flags & PG_MARKER)
3978 if (vm_page_busy_try(p, TRUE))
3980 if (((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) &&
3981 (p->flags & PG_FICTITIOUS) == 0) {
3982 if ((p->queue - p->pc) == PQ_CACHE)
3983 vm_page_deactivate(p);
3984 rel_index = p->pindex - info->start_pindex;
3985 pmap_enter_quick(info->pmap,
3986 info->addr + x86_64_ptob(rel_index), p);
3994 * Return TRUE if the pmap is in shape to trivially pre-fault the specified
3997 * Returns FALSE if it would be non-trivial or if a pte is already loaded
4000 * XXX This is safe only because page table pages are not freed.
4003 pmap_prefault_ok(pmap_t pmap, vm_offset_t addr)
4007 /*spin_lock(&pmap->pm_spin);*/
4008 if ((pte = pmap_pte(pmap, addr)) != NULL) {
4010 /*spin_unlock(&pmap->pm_spin);*/
4014 /*spin_unlock(&pmap->pm_spin);*/
4019 * Change the wiring attribute for a pmap/va pair. The mapping must already
4020 * exist in the pmap. The mapping may or may not be managed.
4023 pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired,
4024 vm_map_entry_t entry)
4031 lwkt_gettoken(&pmap->pm_token);
4032 pv = pmap_allocpte_seg(pmap, pmap_pt_pindex(va), NULL, entry, va);
4033 ptep = pv_pte_lookup(pv, pmap_pte_index(va));
4035 if (wired && !pmap_pte_w(ptep))
4036 atomic_add_long(&pv->pv_pmap->pm_stats.wired_count, 1);
4037 else if (!wired && pmap_pte_w(ptep))
4038 atomic_add_long(&pv->pv_pmap->pm_stats.wired_count, -1);
4041 * Wiring is not a hardware characteristic so there is no need to
4042 * invalidate TLB. However, in an SMP environment we must use
4043 * a locked bus cycle to update the pte (if we are not using
4044 * the pmap_inval_*() API that is)... it's ok to do this for simple
4048 atomic_set_long(ptep, PG_W);
4050 atomic_clear_long(ptep, PG_W);
4052 lwkt_reltoken(&pmap->pm_token);
4058 * Copy the range specified by src_addr/len from the source map to
4059 * the range dst_addr/len in the destination map.
4061 * This routine is only advisory and need not do anything.
4064 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
4065 vm_size_t len, vm_offset_t src_addr)
4072 * Zero the specified physical page.
4074 * This function may be called from an interrupt and no locking is
4078 pmap_zero_page(vm_paddr_t phys)
4080 vm_offset_t va = PHYS_TO_DMAP(phys);
4082 pagezero((void *)va);
4086 * pmap_page_assertzero:
4088 * Assert that a page is empty, panic if it isn't.
4091 pmap_page_assertzero(vm_paddr_t phys)
4093 vm_offset_t va = PHYS_TO_DMAP(phys);
4096 for (i = 0; i < PAGE_SIZE; i += sizeof(long)) {
4097 if (*(long *)((char *)va + i) != 0) {
4098 panic("pmap_page_assertzero() @ %p not zero!",
4099 (void *)(intptr_t)va);
4107 * Zero part of a physical page by mapping it into memory and clearing
4108 * its contents with bzero.
4110 * off and size may not cover an area beyond a single hardware page.
4113 pmap_zero_page_area(vm_paddr_t phys, int off, int size)
4115 vm_offset_t virt = PHYS_TO_DMAP(phys);
4117 bzero((char *)virt + off, size);
4123 * Copy the physical page from the source PA to the target PA.
4124 * This function may be called from an interrupt. No locking
4128 pmap_copy_page(vm_paddr_t src, vm_paddr_t dst)
4130 vm_offset_t src_virt, dst_virt;
4132 src_virt = PHYS_TO_DMAP(src);
4133 dst_virt = PHYS_TO_DMAP(dst);
4134 bcopy((void *)src_virt, (void *)dst_virt, PAGE_SIZE);
4138 * pmap_copy_page_frag:
4140 * Copy the physical page from the source PA to the target PA.
4141 * This function may be called from an interrupt. No locking
4145 pmap_copy_page_frag(vm_paddr_t src, vm_paddr_t dst, size_t bytes)
4147 vm_offset_t src_virt, dst_virt;
4149 src_virt = PHYS_TO_DMAP(src);
4150 dst_virt = PHYS_TO_DMAP(dst);
4152 bcopy((char *)src_virt + (src & PAGE_MASK),
4153 (char *)dst_virt + (dst & PAGE_MASK),
4158 * Returns true if the pmap's pv is one of the first 16 pvs linked to from
4159 * this page. This count may be changed upwards or downwards in the future;
4160 * it is only necessary that true be returned for a small subset of pmaps
4161 * for proper page aging.
4164 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
4169 if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
4172 vm_page_spin_lock(m);
4173 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
4174 if (pv->pv_pmap == pmap) {
4175 vm_page_spin_unlock(m);
4182 vm_page_spin_unlock(m);
4187 * Remove all pages from specified address space this aids process exit
4188 * speeds. Also, this code may be special cased for the current process
4192 pmap_remove_pages(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
4194 pmap_remove_noinval(pmap, sva, eva);
4199 * pmap_testbit tests bits in pte's note that the testbit/clearbit
4200 * routines are inline, and a lot of things compile-time evaluate.
4204 pmap_testbit(vm_page_t m, int bit)
4209 if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
4212 if (TAILQ_FIRST(&m->md.pv_list) == NULL)
4214 vm_page_spin_lock(m);
4215 if (TAILQ_FIRST(&m->md.pv_list) == NULL) {
4216 vm_page_spin_unlock(m);
4220 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
4222 * if the bit being tested is the modified bit, then
4223 * mark clean_map and ptes as never
4226 if (bit & (PG_A|PG_M)) {
4227 if (!pmap_track_modified(pv->pv_pindex))
4231 #if defined(PMAP_DIAGNOSTIC)
4232 if (pv->pv_pmap == NULL) {
4233 kprintf("Null pmap (tb) at pindex: %"PRIu64"\n",
4238 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_pindex << PAGE_SHIFT);
4240 vm_page_spin_unlock(m);
4244 vm_page_spin_unlock(m);
4249 * This routine is used to modify bits in ptes. Only one bit should be
4250 * specified. PG_RW requires special handling.
4252 * Caller must NOT hold any spin locks
4256 pmap_clearbit(vm_page_t m, int bit)
4258 struct pmap_inval_info info;
4265 vm_page_flag_clear(m, PG_WRITEABLE);
4266 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) {
4273 * Loop over all current mappings setting/clearing as appropos If
4274 * setting RO do we need to clear the VAC?
4276 * NOTE: When clearing PG_M we could also (not implemented) drop
4277 * through to the PG_RW code and clear PG_RW too, forcing
4278 * a fault on write to redetect PG_M for virtual kernels, but
4279 * it isn't necessary since virtual kernels invalidate the
4280 * pte when they clear the VPTE_M bit in their virtual page
4283 * NOTE: Does not re-dirty the page when clearing only PG_M.
4285 if ((bit & PG_RW) == 0) {
4286 vm_page_spin_lock(m);
4287 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
4288 #if defined(PMAP_DIAGNOSTIC)
4289 if (pv->pv_pmap == NULL) {
4290 kprintf("Null pmap (cb) at pindex: %"PRIu64"\n",
4295 pte = pmap_pte_quick(pv->pv_pmap,
4296 pv->pv_pindex << PAGE_SHIFT);
4299 atomic_clear_long(pte, bit);
4301 vm_page_spin_unlock(m);
4306 * Clear PG_RW. Also clears PG_M and marks the page dirty if PG_M
4309 pmap_inval_init(&info);
4312 vm_page_spin_lock(m);
4313 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
4315 * don't write protect pager mappings
4317 if (!pmap_track_modified(pv->pv_pindex))
4320 #if defined(PMAP_DIAGNOSTIC)
4321 if (pv->pv_pmap == NULL) {
4322 kprintf("Null pmap (cb) at pindex: %"PRIu64"\n",
4328 * Skip pages which do not have PG_RW set.
4330 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_pindex << PAGE_SHIFT);
4331 if ((*pte & PG_RW) == 0)
4337 if (pv_hold_try(pv) == 0) {
4338 vm_page_spin_unlock(m);
4339 pv_lock(pv); /* held, now do a blocking lock */
4340 pv_put(pv); /* and release */
4341 goto restart; /* anything could have happened */
4344 save_pmap = pv->pv_pmap;
4345 vm_page_spin_unlock(m);
4346 pmap_inval_interlock(&info, save_pmap,
4347 (vm_offset_t)pv->pv_pindex << PAGE_SHIFT);
4348 KKASSERT(pv->pv_pmap == save_pmap);
4352 if (atomic_cmpset_long(pte, pbits,
4353 pbits & ~(PG_RW|PG_M))) {
4357 pmap_inval_deinterlock(&info, save_pmap);
4358 vm_page_spin_lock(m);
4361 * If PG_M was found to be set while we were clearing PG_RW
4362 * we also clear PG_M (done above) and mark the page dirty.
4363 * Callers expect this behavior.
4369 vm_page_spin_unlock(m);
4370 pmap_inval_done(&info);
4374 * Lower the permission for all mappings to a given page.
4376 * Page must be busied by caller.
4379 pmap_page_protect(vm_page_t m, vm_prot_t prot)
4381 /* JG NX support? */
4382 if ((prot & VM_PROT_WRITE) == 0) {
4383 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) {
4385 * NOTE: pmap_clearbit(.. PG_RW) also clears
4386 * the PG_WRITEABLE flag in (m).
4388 pmap_clearbit(m, PG_RW);
4396 pmap_phys_address(vm_pindex_t ppn)
4398 return (x86_64_ptob(ppn));
4402 * Return a count of reference bits for a page, clearing those bits.
4403 * It is not necessary for every reference bit to be cleared, but it
4404 * is necessary that 0 only be returned when there are truly no
4405 * reference bits set.
4407 * XXX: The exact number of bits to check and clear is a matter that
4408 * should be tested and standardized at some point in the future for
4409 * optimal aging of shared pages.
4411 * This routine may not block.
4414 pmap_ts_referenced(vm_page_t m)
4420 if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
4423 vm_page_spin_lock(m);
4424 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
4425 if (!pmap_track_modified(pv->pv_pindex))
4427 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_pindex << PAGE_SHIFT);
4428 if (pte && (*pte & PG_A)) {
4429 atomic_clear_long(pte, PG_A);
4435 vm_page_spin_unlock(m);
4442 * Return whether or not the specified physical page was modified
4443 * in any physical maps.
4446 pmap_is_modified(vm_page_t m)
4450 res = pmap_testbit(m, PG_M);
4455 * Clear the modify bits on the specified physical page.
4458 pmap_clear_modify(vm_page_t m)
4460 pmap_clearbit(m, PG_M);
4464 * pmap_clear_reference:
4466 * Clear the reference bit on the specified physical page.
4469 pmap_clear_reference(vm_page_t m)
4471 pmap_clearbit(m, PG_A);
4475 * Miscellaneous support routines follow
4480 i386_protection_init(void)
4484 /* JG NX support may go here; No VM_PROT_EXECUTE ==> set NX bit */
4485 kp = protection_codes;
4486 for (prot = 0; prot < 8; prot++) {
4488 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE:
4490 * Read access is also 0. There isn't any execute bit,
4491 * so just make it readable.
4493 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_NONE:
4494 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_EXECUTE:
4495 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_EXECUTE:
4498 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE:
4499 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_EXECUTE:
4500 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_NONE:
4501 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE:
4509 * Map a set of physical memory pages into the kernel virtual
4510 * address space. Return a pointer to where it is mapped. This
4511 * routine is intended to be used for mapping device memory,
4514 * NOTE: We can't use pgeflag unless we invalidate the pages one at
4517 * NOTE: The PAT attributes {WRITE_BACK, WRITE_THROUGH, UNCACHED, UNCACHEABLE}
4518 * work whether the cpu supports PAT or not. The remaining PAT
4519 * attributes {WRITE_PROTECTED, WRITE_COMBINING} only work if the cpu
4523 pmap_mapdev(vm_paddr_t pa, vm_size_t size)
4525 return(pmap_mapdev_attr(pa, size, PAT_WRITE_BACK));
4529 pmap_mapdev_uncacheable(vm_paddr_t pa, vm_size_t size)
4531 return(pmap_mapdev_attr(pa, size, PAT_UNCACHEABLE));
4535 pmap_mapbios(vm_paddr_t pa, vm_size_t size)
4537 return (pmap_mapdev_attr(pa, size, PAT_WRITE_BACK));
4541 * Map a set of physical memory pages into the kernel virtual
4542 * address space. Return a pointer to where it is mapped. This
4543 * routine is intended to be used for mapping device memory,
4547 pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode)
4549 vm_offset_t va, tmpva, offset;
4553 offset = pa & PAGE_MASK;
4554 size = roundup(offset + size, PAGE_SIZE);
4556 va = kmem_alloc_nofault(&kernel_map, size, PAGE_SIZE);
4558 panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
4560 pa = pa & ~PAGE_MASK;
4561 for (tmpva = va, tmpsize = size; tmpsize > 0;) {
4562 pte = vtopte(tmpva);
4563 *pte = pa | PG_RW | PG_V | /* pgeflag | */
4564 pat_pte_index[mode];
4565 tmpsize -= PAGE_SIZE;
4569 pmap_invalidate_range(&kernel_pmap, va, va + size);
4570 pmap_invalidate_cache_range(va, va + size);
4572 return ((void *)(va + offset));
4576 pmap_unmapdev(vm_offset_t va, vm_size_t size)
4578 vm_offset_t base, offset;
4580 base = va & ~PAGE_MASK;
4581 offset = va & PAGE_MASK;
4582 size = roundup(offset + size, PAGE_SIZE);
4583 pmap_qremove(va, size >> PAGE_SHIFT);
4584 kmem_free(&kernel_map, base, size);
4588 * Sets the memory attribute for the specified page.
4591 pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
4597 * If "m" is a normal page, update its direct mapping. This update
4598 * can be relied upon to perform any cache operations that are
4599 * required for data coherence.
4601 if ((m->flags & PG_FICTITIOUS) == 0)
4602 pmap_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)), PAGE_SIZE,
4607 * Change the PAT attribute on an existing kernel memory map. Caller
4608 * must ensure that the virtual memory in question is not accessed
4609 * during the adjustment.
4612 pmap_change_attr(vm_offset_t va, vm_size_t count, int mode)
4619 panic("pmap_change_attr: va is NULL");
4620 base = trunc_page(va);
4624 *pte = (*pte & ~(pt_entry_t)(PG_PTE_PAT | PG_NC_PCD |
4626 pat_pte_index[mode];
4631 changed = 1; /* XXX: not optimal */
4634 * Flush CPU caches if required to make sure any data isn't cached that
4635 * shouldn't be, etc.
4638 pmap_invalidate_range(&kernel_pmap, base, va);
4639 pmap_invalidate_cache_range(base, va);
4644 * perform the pmap work for mincore
4647 pmap_mincore(pmap_t pmap, vm_offset_t addr)
4649 pt_entry_t *ptep, pte;
4653 lwkt_gettoken(&pmap->pm_token);
4654 ptep = pmap_pte(pmap, addr);
4656 if (ptep && (pte = *ptep) != 0) {
4659 val = MINCORE_INCORE;
4660 if ((pte & PG_MANAGED) == 0)
4663 pa = pte & PG_FRAME;
4665 if (pte & PG_DEVICE)
4668 m = PHYS_TO_VM_PAGE(pa);
4674 val |= MINCORE_MODIFIED|MINCORE_MODIFIED_OTHER;
4676 * Modified by someone
4678 else if (m && (m->dirty || pmap_is_modified(m)))
4679 val |= MINCORE_MODIFIED_OTHER;
4684 val |= MINCORE_REFERENCED|MINCORE_REFERENCED_OTHER;
4687 * Referenced by someone
4689 else if (m && ((m->flags & PG_REFERENCED) ||
4690 pmap_ts_referenced(m))) {
4691 val |= MINCORE_REFERENCED_OTHER;
4692 vm_page_flag_set(m, PG_REFERENCED);
4696 lwkt_reltoken(&pmap->pm_token);
4702 * Replace p->p_vmspace with a new one. If adjrefs is non-zero the new
4703 * vmspace will be ref'd and the old one will be deref'd.
4705 * The vmspace for all lwps associated with the process will be adjusted
4706 * and cr3 will be reloaded if any lwp is the current lwp.
4708 * The process must hold the vmspace->vm_map.token for oldvm and newvm
4711 pmap_replacevm(struct proc *p, struct vmspace *newvm, int adjrefs)
4713 struct vmspace *oldvm;
4716 oldvm = p->p_vmspace;
4717 if (oldvm != newvm) {
4719 sysref_get(&newvm->vm_sysref);
4720 p->p_vmspace = newvm;
4721 KKASSERT(p->p_nthreads == 1);
4722 lp = RB_ROOT(&p->p_lwp_tree);
4723 pmap_setlwpvm(lp, newvm);
4725 sysref_put(&oldvm->vm_sysref);
4730 * Set the vmspace for a LWP. The vmspace is almost universally set the
4731 * same as the process vmspace, but virtual kernels need to swap out contexts
4732 * on a per-lwp basis.
4734 * Caller does not necessarily hold any vmspace tokens. Caller must control
4735 * the lwp (typically be in the context of the lwp). We use a critical
4736 * section to protect against statclock and hardclock (statistics collection).
4739 pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm)
4741 struct vmspace *oldvm;
4744 oldvm = lp->lwp_vmspace;
4746 if (oldvm != newvm) {
4748 lp->lwp_vmspace = newvm;
4749 if (curthread->td_lwp == lp) {
4750 pmap = vmspace_pmap(newvm);
4751 atomic_set_cpumask(&pmap->pm_active, mycpu->gd_cpumask);
4752 if (pmap->pm_active & CPUMASK_LOCK)
4753 pmap_interlock_wait(newvm);
4754 #if defined(SWTCH_OPTIM_STATS)
4757 curthread->td_pcb->pcb_cr3 = vtophys(pmap->pm_pml4);
4758 load_cr3(curthread->td_pcb->pcb_cr3);
4759 pmap = vmspace_pmap(oldvm);
4760 atomic_clear_cpumask(&pmap->pm_active, mycpu->gd_cpumask);
4767 * Called when switching to a locked pmap, used to interlock against pmaps
4768 * undergoing modifications to prevent us from activating the MMU for the
4769 * target pmap until all such modifications have completed. We have to do
4770 * this because the thread making the modifications has already set up its
4771 * SMP synchronization mask.
4773 * This function cannot sleep!
4778 pmap_interlock_wait(struct vmspace *vm)
4780 struct pmap *pmap = &vm->vm_pmap;
4782 if (pmap->pm_active & CPUMASK_LOCK) {
4784 KKASSERT(curthread->td_critcount >= 2);
4785 DEBUG_PUSH_INFO("pmap_interlock_wait");
4786 while (pmap->pm_active & CPUMASK_LOCK) {
4788 lwkt_process_ipiq();
4796 pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size)
4799 if ((obj == NULL) || (size < NBPDR) ||
4800 ((obj->type != OBJT_DEVICE) && (obj->type != OBJT_MGTDEVICE))) {
4804 addr = (addr + (NBPDR - 1)) & ~(NBPDR - 1);
4809 * Used by kmalloc/kfree, page already exists at va
4812 pmap_kvtom(vm_offset_t va)
4814 pt_entry_t *ptep = vtopte(va);
4816 KKASSERT((*ptep & PG_DEVICE) == 0);
4817 return(PHYS_TO_VM_PAGE(*ptep & PG_FRAME));
4821 * Initialize machine-specific shared page directory support. This
4822 * is executed when a VM object is created.
4825 pmap_object_init(vm_object_t object)
4827 object->md.pmap_rw = NULL;
4828 object->md.pmap_ro = NULL;
4832 * Clean up machine-specific shared page directory support. This
4833 * is executed when a VM object is destroyed.
4836 pmap_object_free(vm_object_t object)
4840 if ((pmap = object->md.pmap_rw) != NULL) {
4841 object->md.pmap_rw = NULL;
4842 pmap_remove_noinval(pmap,
4843 VM_MIN_USER_ADDRESS, VM_MAX_USER_ADDRESS);
4844 pmap->pm_active = 0;
4847 kfree(pmap, M_OBJPMAP);
4849 if ((pmap = object->md.pmap_ro) != NULL) {
4850 object->md.pmap_ro = NULL;
4851 pmap_remove_noinval(pmap,
4852 VM_MIN_USER_ADDRESS, VM_MAX_USER_ADDRESS);
4853 pmap->pm_active = 0;
4856 kfree(pmap, M_OBJPMAP);