Testing - fix bugs in fsx related to 64-bit systems.
[dragonfly.git] / sys / platform / pc64 / amd64 / pmap.c
CommitLineData
d7f50089 1/*
d7f50089 2 * Copyright (c) 1991 Regents of the University of California.
d7f50089 3 * Copyright (c) 1994 John S. Dyson
d7f50089 4 * Copyright (c) 1994 David Greenman
48ffc236
JG
5 * Copyright (c) 2003 Peter Wemm
6 * Copyright (c) 2005-2008 Alan L. Cox <alc@cs.rice.edu>
7 * Copyright (c) 2008, 2009 The DragonFly Project.
8 * Copyright (c) 2008, 2009 Jordan Gordeev.
d7f50089 9 * All rights reserved.
c8fe38ae
MD
10 *
11 * This code is derived from software contributed to Berkeley by
12 * the Systems Programming Group of the University of Utah Computer
13 * Science Department and William Jolitz of UUNET Technologies Inc.
14 *
d7f50089
YY
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
17 * are met:
d7f50089
YY
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
c8fe38ae
MD
21 * notice, this list of conditions and the following disclaimer in the
22 * documentation and/or other materials provided with the distribution.
23 * 3. All advertising materials mentioning features or use of this software
24 * must display the following acknowledgement:
25 * This product includes software developed by the University of
26 * California, Berkeley and its contributors.
27 * 4. Neither the name of the University nor the names of its contributors
28 * may be used to endorse or promote products derived from this software
29 * without specific prior written permission.
30 *
31 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
32 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
33 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
35 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
36 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
37 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
38 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
39 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
40 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
d7f50089 41 * SUCH DAMAGE.
c8fe38ae
MD
42 *
43 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91
d7f50089 44 * $FreeBSD: src/sys/i386/i386/pmap.c,v 1.250.2.18 2002/03/06 22:48:53 silby Exp $
c8fe38ae 45 * $DragonFly: src/sys/platform/pc64/amd64/pmap.c,v 1.3 2008/08/29 17:07:10 dillon Exp $
d7f50089 46 */
c8fe38ae 47
d7f50089 48/*
c8fe38ae
MD
49 * Manages physical address maps.
50 *
51 * In addition to hardware address maps, this
52 * module is called upon to provide software-use-only
53 * maps which may or may not be stored in the same
54 * form as hardware maps. These pseudo-maps are
55 * used to store intermediate results from copy
56 * operations to and from address spaces.
57 *
58 * Since the information managed by this module is
59 * also stored by the logical address mapping module,
60 * this module may throw away valid virtual-to-physical
61 * mappings at almost any time. However, invalidations
62 * of virtual-to-physical mappings must be done as
63 * requested.
64 *
65 * In order to cope with hardware architectures which
66 * make virtual-to-physical map invalidates expensive,
67 * this module may delay invalidate or reduced protection
68 * operations until such time as they are actually
69 * necessary. This module is given full information as
70 * to which processors are currently using which maps,
71 * and to when physical maps must be made correct.
72 */
73
74#if JG
75#include "opt_disable_pse.h"
76#include "opt_pmap.h"
77#endif
78#include "opt_msgbuf.h"
d7f50089 79
c8fe38ae 80#include <sys/param.h>
d7f50089
YY
81#include <sys/systm.h>
82#include <sys/kernel.h>
d7f50089 83#include <sys/proc.h>
c8fe38ae
MD
84#include <sys/msgbuf.h>
85#include <sys/vmmeter.h>
86#include <sys/mman.h>
d7f50089 87
c8fe38ae
MD
88#include <vm/vm.h>
89#include <vm/vm_param.h>
90#include <sys/sysctl.h>
91#include <sys/lock.h>
d7f50089 92#include <vm/vm_kern.h>
c8fe38ae
MD
93#include <vm/vm_page.h>
94#include <vm/vm_map.h>
d7f50089 95#include <vm/vm_object.h>
c8fe38ae 96#include <vm/vm_extern.h>
d7f50089 97#include <vm/vm_pageout.h>
c8fe38ae
MD
98#include <vm/vm_pager.h>
99#include <vm/vm_zone.h>
100
101#include <sys/user.h>
102#include <sys/thread2.h>
103#include <sys/sysref2.h>
d7f50089 104
c8fe38ae 105#include <machine/cputypes.h>
d7f50089 106#include <machine/md_var.h>
c8fe38ae
MD
107#include <machine/specialreg.h>
108#include <machine/smp.h>
109#include <machine_base/apic/apicreg.h>
d7f50089 110#include <machine/globaldata.h>
c8fe38ae
MD
111#include <machine/pmap.h>
112#include <machine/pmap_inval.h>
113
48ffc236
JG
114#include <ddb/ddb.h>
115
c8fe38ae
MD
116#define PMAP_KEEP_PDIRS
117#ifndef PMAP_SHPGPERPROC
118#define PMAP_SHPGPERPROC 200
119#endif
120
121#if defined(DIAGNOSTIC)
122#define PMAP_DIAGNOSTIC
123#endif
124
125#define MINPV 2048
126
c8fe38ae
MD
127/*
128 * Get PDEs and PTEs for user/kernel address space
129 */
48ffc236 130static pd_entry_t *pmap_pde(pmap_t pmap, vm_offset_t va);
c8fe38ae
MD
131#define pdir_pde(m, v) (m[(vm_offset_t)(v) >> PDRSHIFT])
132
133#define pmap_pde_v(pte) ((*(pd_entry_t *)pte & PG_V) != 0)
134#define pmap_pte_w(pte) ((*(pt_entry_t *)pte & PG_W) != 0)
135#define pmap_pte_m(pte) ((*(pt_entry_t *)pte & PG_M) != 0)
136#define pmap_pte_u(pte) ((*(pt_entry_t *)pte & PG_A) != 0)
137#define pmap_pte_v(pte) ((*(pt_entry_t *)pte & PG_V) != 0)
138
139
140/*
141 * Given a map and a machine independent protection code,
142 * convert to a vax protection code.
143 */
144#define pte_prot(m, p) \
145 (protection_codes[p & (VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE)])
146static int protection_codes[8];
d7f50089
YY
147
148struct pmap kernel_pmap;
c8fe38ae 149static TAILQ_HEAD(,pmap) pmap_list = TAILQ_HEAD_INITIALIZER(pmap_list);
d7f50089 150
c8fe38ae
MD
151vm_paddr_t avail_start; /* PA of first available physical page */
152vm_paddr_t avail_end; /* PA of last available physical page */
153vm_offset_t virtual_start; /* VA of first avail page (after kernel bss) */
154vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */
155vm_offset_t KvaStart; /* VA start of KVA space */
156vm_offset_t KvaEnd; /* VA end of KVA space (non-inclusive) */
157vm_offset_t KvaSize; /* max size of kernel virtual address space */
158static boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */
159static int pgeflag; /* PG_G or-in */
160static int pseflag; /* PG_PS or-in */
d7f50089 161
c8fe38ae
MD
162static vm_object_t kptobj;
163
48ffc236
JG
164static int ndmpdp;
165static vm_paddr_t dmaplimit;
c8fe38ae
MD
166static int nkpt;
167vm_offset_t kernel_vm_end;
d7f50089 168
48ffc236
JG
169static uint64_t KPDphys; /* phys addr of kernel level 2 */
170uint64_t KPDPphys; /* phys addr of kernel level 3 */
171uint64_t KPML4phys; /* phys addr of kernel level 4 */
172
173static uint64_t DMPDphys; /* phys addr of direct mapped level 2 */
174static uint64_t DMPDPphys; /* phys addr of direct mapped level 3 */
175
d7f50089 176/*
c8fe38ae 177 * Data for the pv entry allocation mechanism
d7f50089 178 */
c8fe38ae
MD
179static vm_zone_t pvzone;
180static struct vm_zone pvzone_store;
181static struct vm_object pvzone_obj;
182static int pv_entry_count=0, pv_entry_max=0, pv_entry_high_water=0;
183static int pmap_pagedaemon_waken = 0;
184static struct pv_entry *pvinit;
d7f50089
YY
185
186/*
c8fe38ae 187 * All those kernel PT submaps that BSD is so fond of
d7f50089 188 */
c8fe38ae
MD
189pt_entry_t *CMAP1 = 0, *ptmmap;
190caddr_t CADDR1 = 0, ptvmmap = 0;
191static pt_entry_t *msgbufmap;
192struct msgbuf *msgbufp=0;
d7f50089 193
c8fe38ae
MD
194/*
195 * Crashdump maps.
d7f50089 196 */
c8fe38ae
MD
197static pt_entry_t *pt_crashdumpmap;
198static caddr_t crashdumpmap;
199
200extern uint64_t KPTphys;
201extern pt_entry_t *SMPpt;
202extern uint64_t SMPptpa;
203
204#define DISABLE_PSE
205
c8fe38ae 206static pv_entry_t get_pv_entry (void);
bfc09ba0
MD
207static void i386_protection_init (void);
208static void create_pagetables(vm_paddr_t *firstaddr);
209static void pmap_remove_all (vm_page_t m);
210static void pmap_enter_quick (pmap_t pmap, vm_offset_t va, vm_page_t m);
211static int pmap_remove_pte (struct pmap *pmap, pt_entry_t *ptq,
c8fe38ae
MD
212 vm_offset_t sva, pmap_inval_info_t info);
213static void pmap_remove_page (struct pmap *pmap,
214 vm_offset_t va, pmap_inval_info_t info);
bfc09ba0 215static int pmap_remove_entry (struct pmap *pmap, vm_page_t m,
c8fe38ae
MD
216 vm_offset_t va, pmap_inval_info_t info);
217static boolean_t pmap_testbit (vm_page_t m, int bit);
218static void pmap_insert_entry (pmap_t pmap, vm_offset_t va,
bfc09ba0 219 vm_page_t mpte, vm_page_t m);
c8fe38ae
MD
220
221static vm_page_t pmap_allocpte (pmap_t pmap, vm_offset_t va);
222
223static int pmap_release_free_page (pmap_t pmap, vm_page_t p);
224static vm_page_t _pmap_allocpte (pmap_t pmap, vm_pindex_t ptepindex);
225static pt_entry_t * pmap_pte_quick (pmap_t pmap, vm_offset_t va);
226static vm_page_t pmap_page_lookup (vm_object_t object, vm_pindex_t pindex);
bfc09ba0
MD
227static int _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m,
228 pmap_inval_info_t info);
c8fe38ae
MD
229static int pmap_unuse_pt (pmap_t, vm_offset_t, vm_page_t, pmap_inval_info_t);
230static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
231
232static unsigned pdir4mb;
d7f50089
YY
233
234/*
c8fe38ae 235 * Move the kernel virtual free pointer to the next
f9cc0f15
JG
236 * 2MB. This is used to help improve performance
237 * by using a large (2MB) page for much of the kernel
c8fe38ae 238 * (.text, .data, .bss)
d7f50089 239 */
bfc09ba0
MD
240static
241vm_offset_t
c8fe38ae 242pmap_kmem_choose(vm_offset_t addr)
d7f50089 243{
c8fe38ae 244 vm_offset_t newaddr = addr;
f9cc0f15
JG
245
246 newaddr = (addr + (NBPDR - 1)) & ~(NBPDR - 1);
c8fe38ae 247 return newaddr;
d7f50089
YY
248}
249
d7f50089 250/*
c8fe38ae 251 * pmap_pte_quick:
d7f50089 252 *
c8fe38ae
MD
253 * Super fast pmap_pte routine best used when scanning the pv lists.
254 * This eliminates many course-grained invltlb calls. Note that many of
255 * the pv list scans are across different pmaps and it is very wasteful
256 * to do an entire invltlb when checking a single mapping.
257 *
258 * Should only be called while in a critical section.
259 */
48ffc236
JG
260static __inline pt_entry_t *pmap_pte(pmap_t pmap, vm_offset_t va);
261
bfc09ba0
MD
262static
263pt_entry_t *
c8fe38ae
MD
264pmap_pte_quick(pmap_t pmap, vm_offset_t va)
265{
48ffc236
JG
266 return pmap_pte(pmap, va);
267}
268
269/* Return a non-clipped PD index for a given VA */
bfc09ba0
MD
270static __inline
271vm_pindex_t
48ffc236 272pmap_pde_pindex(vm_offset_t va)
48ffc236
JG
273{
274 return va >> PDRSHIFT;
275}
276
277/* Return various clipped indexes for a given VA */
bfc09ba0
MD
278static __inline
279vm_pindex_t
48ffc236 280pmap_pte_index(vm_offset_t va)
48ffc236
JG
281{
282
283 return ((va >> PAGE_SHIFT) & ((1ul << NPTEPGSHIFT) - 1));
284}
285
bfc09ba0
MD
286static __inline
287vm_pindex_t
48ffc236 288pmap_pde_index(vm_offset_t va)
48ffc236
JG
289{
290
291 return ((va >> PDRSHIFT) & ((1ul << NPDEPGSHIFT) - 1));
292}
293
bfc09ba0
MD
294static __inline
295vm_pindex_t
48ffc236 296pmap_pdpe_index(vm_offset_t va)
48ffc236
JG
297{
298
299 return ((va >> PDPSHIFT) & ((1ul << NPDPEPGSHIFT) - 1));
300}
301
bfc09ba0
MD
302static __inline
303vm_pindex_t
48ffc236 304pmap_pml4e_index(vm_offset_t va)
48ffc236
JG
305{
306
307 return ((va >> PML4SHIFT) & ((1ul << NPML4EPGSHIFT) - 1));
308}
309
310/* Return a pointer to the PML4 slot that corresponds to a VA */
bfc09ba0
MD
311static __inline
312pml4_entry_t *
48ffc236 313pmap_pml4e(pmap_t pmap, vm_offset_t va)
48ffc236
JG
314{
315
316 return (&pmap->pm_pml4[pmap_pml4e_index(va)]);
317}
318
319/* Return a pointer to the PDP slot that corresponds to a VA */
bfc09ba0
MD
320static __inline
321pdp_entry_t *
48ffc236 322pmap_pml4e_to_pdpe(pml4_entry_t *pml4e, vm_offset_t va)
48ffc236
JG
323{
324 pdp_entry_t *pdpe;
325
326 pdpe = (pdp_entry_t *)PHYS_TO_DMAP(*pml4e & PG_FRAME);
327 return (&pdpe[pmap_pdpe_index(va)]);
328}
329
330/* Return a pointer to the PDP slot that corresponds to a VA */
bfc09ba0
MD
331static __inline
332pdp_entry_t *
48ffc236 333pmap_pdpe(pmap_t pmap, vm_offset_t va)
48ffc236
JG
334{
335 pml4_entry_t *pml4e;
336
337 pml4e = pmap_pml4e(pmap, va);
338 if ((*pml4e & PG_V) == 0)
339 return NULL;
340 return (pmap_pml4e_to_pdpe(pml4e, va));
341}
342
343/* Return a pointer to the PD slot that corresponds to a VA */
bfc09ba0
MD
344static __inline
345pd_entry_t *
48ffc236 346pmap_pdpe_to_pde(pdp_entry_t *pdpe, vm_offset_t va)
48ffc236
JG
347{
348 pd_entry_t *pde;
349
350 pde = (pd_entry_t *)PHYS_TO_DMAP(*pdpe & PG_FRAME);
351 return (&pde[pmap_pde_index(va)]);
352}
353
354/* Return a pointer to the PD slot that corresponds to a VA */
bfc09ba0
MD
355static __inline
356pd_entry_t *
48ffc236 357pmap_pde(pmap_t pmap, vm_offset_t va)
48ffc236
JG
358{
359 pdp_entry_t *pdpe;
360
361 pdpe = pmap_pdpe(pmap, va);
362 if (pdpe == NULL || (*pdpe & PG_V) == 0)
363 return NULL;
364 return (pmap_pdpe_to_pde(pdpe, va));
365}
366
367/* Return a pointer to the PT slot that corresponds to a VA */
bfc09ba0
MD
368static __inline
369pt_entry_t *
48ffc236 370pmap_pde_to_pte(pd_entry_t *pde, vm_offset_t va)
48ffc236
JG
371{
372 pt_entry_t *pte;
373
374 pte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME);
375 return (&pte[pmap_pte_index(va)]);
376}
377
378/* Return a pointer to the PT slot that corresponds to a VA */
bfc09ba0
MD
379static __inline
380pt_entry_t *
48ffc236 381pmap_pte(pmap_t pmap, vm_offset_t va)
48ffc236
JG
382{
383 pd_entry_t *pde;
384
385 pde = pmap_pde(pmap, va);
386 if (pde == NULL || (*pde & PG_V) == 0)
387 return NULL;
388 if ((*pde & PG_PS) != 0) /* compat with i386 pmap_pte() */
389 return ((pt_entry_t *)pde);
390 return (pmap_pde_to_pte(pde, va));
391}
392
bfc09ba0
MD
393static __inline
394pt_entry_t *
48ffc236 395vtopte(vm_offset_t va)
48ffc236
JG
396{
397 uint64_t mask = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
398
399 return (PTmap + ((va >> PAGE_SHIFT) & mask));
c8fe38ae 400}
d7f50089 401
bfc09ba0
MD
402static __inline
403pd_entry_t *
48ffc236 404vtopde(vm_offset_t va)
48ffc236
JG
405{
406 uint64_t mask = ((1ul << (NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
407
408 return (PDmap + ((va >> PDRSHIFT) & mask));
409}
c8fe38ae 410
48ffc236 411static uint64_t
c8fe38ae 412allocpages(vm_paddr_t *firstaddr, int n)
d7f50089 413{
48ffc236 414 uint64_t ret;
c8fe38ae
MD
415
416 ret = *firstaddr;
417 bzero((void *)ret, n * PAGE_SIZE);
418 *firstaddr += n * PAGE_SIZE;
419 return (ret);
d7f50089
YY
420}
421
bfc09ba0 422static
c8fe38ae
MD
423void
424create_pagetables(vm_paddr_t *firstaddr)
425{
426 int i;
c8fe38ae
MD
427
428 /* we are running (mostly) V=P at this point */
429
48ffc236
JG
430 /* Allocate pages */
431 KPTphys = allocpages(firstaddr, NKPT);
432 KPML4phys = allocpages(firstaddr, 1);
433 KPDPphys = allocpages(firstaddr, NKPML4E);
434 KPDphys = allocpages(firstaddr, NKPDPE);
435
436 ndmpdp = (ptoa(Maxmem) + NBPDP - 1) >> PDPSHIFT;
437 if (ndmpdp < 4) /* Minimum 4GB of dirmap */
438 ndmpdp = 4;
439 DMPDPphys = allocpages(firstaddr, NDMPML4E);
440 if ((amd_feature & AMDID_PAGE1GB) == 0)
441 DMPDphys = allocpages(firstaddr, ndmpdp);
442 dmaplimit = (vm_paddr_t)ndmpdp << PDPSHIFT;
443
444 /* Fill in the underlying page table pages */
445 /* Read-only from zero to physfree */
446 /* XXX not fully used, underneath 2M pages */
447 for (i = 0; (i << PAGE_SHIFT) < *firstaddr; i++) {
448 ((pt_entry_t *)KPTphys)[i] = i << PAGE_SHIFT;
449 ((pt_entry_t *)KPTphys)[i] |= PG_RW | PG_V | PG_G;
450 }
451
452 /* Now map the page tables at their location within PTmap */
453 for (i = 0; i < NKPT; i++) {
454 ((pd_entry_t *)KPDphys)[i] = KPTphys + (i << PAGE_SHIFT);
455 ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V;
456 }
457
458 /* Map from zero to end of allocations under 2M pages */
459 /* This replaces some of the KPTphys entries above */
460 for (i = 0; (i << PDRSHIFT) < *firstaddr; i++) {
461 ((pd_entry_t *)KPDphys)[i] = i << PDRSHIFT;
462 ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V | PG_PS | PG_G;
463 }
464
465 /* And connect up the PD to the PDP */
466 for (i = 0; i < NKPDPE; i++) {
467 ((pdp_entry_t *)KPDPphys)[i + KPDPI] = KPDphys +
468 (i << PAGE_SHIFT);
469 ((pdp_entry_t *)KPDPphys)[i + KPDPI] |= PG_RW | PG_V | PG_U;
470 }
471
472 /* Now set up the direct map space using either 2MB or 1GB pages */
473 /* Preset PG_M and PG_A because demotion expects it */
474 if ((amd_feature & AMDID_PAGE1GB) == 0) {
475 for (i = 0; i < NPDEPG * ndmpdp; i++) {
476 ((pd_entry_t *)DMPDphys)[i] = (vm_paddr_t)i << PDRSHIFT;
477 ((pd_entry_t *)DMPDphys)[i] |= PG_RW | PG_V | PG_PS |
478 PG_G | PG_M | PG_A;
479 }
480 /* And the direct map space's PDP */
481 for (i = 0; i < ndmpdp; i++) {
482 ((pdp_entry_t *)DMPDPphys)[i] = DMPDphys +
483 (i << PAGE_SHIFT);
484 ((pdp_entry_t *)DMPDPphys)[i] |= PG_RW | PG_V | PG_U;
485 }
486 } else {
487 for (i = 0; i < ndmpdp; i++) {
488 ((pdp_entry_t *)DMPDPphys)[i] =
489 (vm_paddr_t)i << PDPSHIFT;
490 ((pdp_entry_t *)DMPDPphys)[i] |= PG_RW | PG_V | PG_PS |
491 PG_G | PG_M | PG_A;
492 }
493 }
494
495 /* And recursively map PML4 to itself in order to get PTmap */
496 ((pdp_entry_t *)KPML4phys)[PML4PML4I] = KPML4phys;
497 ((pdp_entry_t *)KPML4phys)[PML4PML4I] |= PG_RW | PG_V | PG_U;
498
499 /* Connect the Direct Map slot up to the PML4 */
500 ((pdp_entry_t *)KPML4phys)[DMPML4I] = DMPDPphys;
501 ((pdp_entry_t *)KPML4phys)[DMPML4I] |= PG_RW | PG_V | PG_U;
502
503 /* Connect the KVA slot up to the PML4 */
504 ((pdp_entry_t *)KPML4phys)[KPML4I] = KPDPphys;
505 ((pdp_entry_t *)KPML4phys)[KPML4I] |= PG_RW | PG_V | PG_U;
c8fe38ae
MD
506}
507
508void
f81851b8
MD
509init_paging(vm_paddr_t *firstaddr)
510{
c8fe38ae 511 create_pagetables(firstaddr);
d7f50089
YY
512}
513
514/*
c8fe38ae
MD
515 * Bootstrap the system enough to run with virtual memory.
516 *
517 * On the i386 this is called after mapping has already been enabled
518 * and just syncs the pmap module with what has already been done.
519 * [We can't call it easily with mapping off since the kernel is not
520 * mapped with PA == VA, hence we would have to relocate every address
521 * from the linked base (virtual) address "KERNBASE" to the actual
522 * (physical) address starting relative to 0]
d7f50089
YY
523 */
524void
48ffc236 525pmap_bootstrap(vm_paddr_t *firstaddr)
c8fe38ae
MD
526{
527 vm_offset_t va;
528 pt_entry_t *pte;
529 struct mdglobaldata *gd;
c8fe38ae
MD
530 int pg;
531
48ffc236
JG
532 KvaStart = VM_MIN_KERNEL_ADDRESS;
533 KvaEnd = VM_MAX_KERNEL_ADDRESS;
534 KvaSize = KvaEnd - KvaStart;
535
c8fe38ae
MD
536 avail_start = *firstaddr;
537
538 /*
48ffc236 539 * Create an initial set of page tables to run the kernel in.
c8fe38ae 540 */
48ffc236
JG
541 create_pagetables(firstaddr);
542
c8fe38ae
MD
543 virtual_start = (vm_offset_t) PTOV_OFFSET + *firstaddr;
544 virtual_start = pmap_kmem_choose(virtual_start);
48ffc236
JG
545
546 virtual_end = VM_MAX_KERNEL_ADDRESS;
547
548 /* XXX do %cr0 as well */
549 load_cr4(rcr4() | CR4_PGE | CR4_PSE);
550 load_cr3(KPML4phys);
c8fe38ae
MD
551
552 /*
553 * Initialize protection array.
554 */
555 i386_protection_init();
556
557 /*
558 * The kernel's pmap is statically allocated so we don't have to use
559 * pmap_create, which is unlikely to work correctly at this part of
560 * the boot sequence (XXX and which no longer exists).
561 */
48ffc236 562 kernel_pmap.pm_pml4 = (pdp_entry_t *) (PTOV_OFFSET + KPML4phys);
c8fe38ae
MD
563 kernel_pmap.pm_count = 1;
564 kernel_pmap.pm_active = (cpumask_t)-1; /* don't allow deactivation */
565 TAILQ_INIT(&kernel_pmap.pm_pvlist);
566 nkpt = NKPT;
567
568 /*
569 * Reserve some special page table entries/VA space for temporary
570 * mapping of pages.
571 */
572#define SYSMAP(c, p, v, n) \
573 v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n);
574
575 va = virtual_start;
48ffc236 576#ifdef JG
c8fe38ae 577 pte = (pt_entry_t *) pmap_pte(&kernel_pmap, va);
48ffc236
JG
578#else
579 pte = vtopte(va);
580#endif
c8fe38ae
MD
581
582 /*
583 * CMAP1/CMAP2 are used for zeroing and copying pages.
584 */
585 SYSMAP(caddr_t, CMAP1, CADDR1, 1)
586
587 /*
588 * Crashdump maps.
589 */
590 SYSMAP(caddr_t, pt_crashdumpmap, crashdumpmap, MAXDUMPPGS);
591
592 /*
593 * ptvmmap is used for reading arbitrary physical pages via
594 * /dev/mem.
595 */
596 SYSMAP(caddr_t, ptmmap, ptvmmap, 1)
597
598 /*
599 * msgbufp is used to map the system message buffer.
600 * XXX msgbufmap is not used.
601 */
602 SYSMAP(struct msgbuf *, msgbufmap, msgbufp,
603 atop(round_page(MSGBUF_SIZE)))
604
605 virtual_start = va;
606
607 *CMAP1 = 0;
c8fe38ae
MD
608
609 /*
610 * PG_G is terribly broken on SMP because we IPI invltlb's in some
611 * cases rather then invl1pg. Actually, I don't even know why it
612 * works under UP because self-referential page table mappings
613 */
614#ifdef SMP
615 pgeflag = 0;
616#else
617 if (cpu_feature & CPUID_PGE)
618 pgeflag = PG_G;
619#endif
620
621/*
622 * Initialize the 4MB page size flag
623 */
624 pseflag = 0;
625/*
626 * The 4MB page version of the initial
627 * kernel page mapping.
628 */
629 pdir4mb = 0;
630
631#if !defined(DISABLE_PSE)
632 if (cpu_feature & CPUID_PSE) {
633 pt_entry_t ptditmp;
634 /*
635 * Note that we have enabled PSE mode
636 */
637 pseflag = PG_PS;
638 ptditmp = *(PTmap + amd64_btop(KERNBASE));
639 ptditmp &= ~(NBPDR - 1);
640 ptditmp |= PG_V | PG_RW | PG_PS | PG_U | pgeflag;
641 pdir4mb = ptditmp;
642
643#ifndef SMP
644 /*
645 * Enable the PSE mode. If we are SMP we can't do this
646 * now because the APs will not be able to use it when
647 * they boot up.
648 */
649 load_cr4(rcr4() | CR4_PSE);
650
651 /*
652 * We can do the mapping here for the single processor
653 * case. We simply ignore the old page table page from
654 * now on.
655 */
656 /*
657 * For SMP, we still need 4K pages to bootstrap APs,
658 * PSE will be enabled as soon as all APs are up.
659 */
660 PTD[KPTDI] = (pd_entry_t)ptditmp;
c8fe38ae
MD
661 cpu_invltlb();
662#endif
663 }
664#endif
665#ifdef SMP
666 if (cpu_apic_address == 0)
667 panic("pmap_bootstrap: no local apic!");
c8fe38ae
MD
668#endif
669
670 /*
671 * We need to finish setting up the globaldata page for the BSP.
672 * locore has already populated the page table for the mdglobaldata
673 * portion.
674 */
675 pg = MDGLOBALDATA_BASEALLOC_PAGES;
676 gd = &CPU_prvspace[0].mdglobaldata;
677 gd->gd_CMAP1 = &SMPpt[pg + 0];
678 gd->gd_CMAP2 = &SMPpt[pg + 1];
679 gd->gd_CMAP3 = &SMPpt[pg + 2];
680 gd->gd_PMAP1 = &SMPpt[pg + 3];
681 gd->gd_CADDR1 = CPU_prvspace[0].CPAGE1;
682 gd->gd_CADDR2 = CPU_prvspace[0].CPAGE2;
683 gd->gd_CADDR3 = CPU_prvspace[0].CPAGE3;
684 gd->gd_PADDR1 = (pt_entry_t *)CPU_prvspace[0].PPAGE1;
685
686 cpu_invltlb();
d7f50089
YY
687}
688
c8fe38ae 689#ifdef SMP
d7f50089 690/*
c8fe38ae 691 * Set 4mb pdir for mp startup
d7f50089
YY
692 */
693void
c8fe38ae
MD
694pmap_set_opt(void)
695{
696 if (pseflag && (cpu_feature & CPUID_PSE)) {
697 load_cr4(rcr4() | CR4_PSE);
698 if (pdir4mb && mycpu->gd_cpuid == 0) { /* only on BSP */
c8fe38ae
MD
699 cpu_invltlb();
700 }
701 }
d7f50089 702}
c8fe38ae 703#endif
d7f50089 704
c8fe38ae
MD
705/*
706 * Initialize the pmap module.
707 * Called by vm_init, to initialize any structures that the pmap
708 * system needs to map virtual memory.
709 * pmap_init has been enhanced to support in a fairly consistant
710 * way, discontiguous physical memory.
d7f50089
YY
711 */
712void
c8fe38ae 713pmap_init(void)
d7f50089 714{
c8fe38ae
MD
715 int i;
716 int initial_pvs;
717
718 /*
719 * object for kernel page table pages
720 */
48ffc236
JG
721 /* JG I think the number can be arbitrary */
722 kptobj = vm_object_allocate(OBJT_DEFAULT, 5);
c8fe38ae
MD
723
724 /*
725 * Allocate memory for random pmap data structures. Includes the
726 * pv_head_table.
727 */
728
729 for(i = 0; i < vm_page_array_size; i++) {
730 vm_page_t m;
731
732 m = &vm_page_array[i];
733 TAILQ_INIT(&m->md.pv_list);
734 m->md.pv_list_count = 0;
735 }
736
737 /*
738 * init the pv free list
739 */
740 initial_pvs = vm_page_array_size;
741 if (initial_pvs < MINPV)
742 initial_pvs = MINPV;
743 pvzone = &pvzone_store;
744 pvinit = (struct pv_entry *) kmem_alloc(&kernel_map,
745 initial_pvs * sizeof (struct pv_entry));
746 zbootinit(pvzone, "PV ENTRY", sizeof (struct pv_entry), pvinit,
747 initial_pvs);
748
749 /*
750 * Now it is safe to enable pv_table recording.
751 */
752 pmap_initialized = TRUE;
d887674b 753#ifdef SMP
057877ac 754 lapic = pmap_mapdev_uncacheable(cpu_apic_address, sizeof(struct LAPIC));
d887674b 755#endif
d7f50089
YY
756}
757
c8fe38ae
MD
758/*
759 * Initialize the address space (zone) for the pv_entries. Set a
760 * high water mark so that the system can recover from excessive
761 * numbers of pv entries.
762 */
d7f50089 763void
c8fe38ae 764pmap_init2(void)
d7f50089 765{
c8fe38ae
MD
766 int shpgperproc = PMAP_SHPGPERPROC;
767
768 TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
769 pv_entry_max = shpgperproc * maxproc + vm_page_array_size;
770 TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
771 pv_entry_high_water = 9 * (pv_entry_max / 10);
772 zinitna(pvzone, &pvzone_obj, NULL, 0, pv_entry_max, ZONE_INTERRUPT, 1);
d7f50089
YY
773}
774
c8fe38ae
MD
775
776/***************************************************
777 * Low level helper routines.....
778 ***************************************************/
779
780#if defined(PMAP_DIAGNOSTIC)
d7f50089
YY
781
782/*
c8fe38ae
MD
783 * This code checks for non-writeable/modified pages.
784 * This should be an invalid condition.
d7f50089 785 */
bfc09ba0
MD
786static
787int
48ffc236 788pmap_nw_modified(pt_entry_t pte)
d7f50089 789{
c8fe38ae
MD
790 if ((pte & (PG_M|PG_RW)) == PG_M)
791 return 1;
792 else
793 return 0;
d7f50089 794}
c8fe38ae
MD
795#endif
796
d7f50089 797
c8fe38ae
MD
798/*
799 * this routine defines the region(s) of memory that should
800 * not be tested for the modified bit.
801 */
bfc09ba0
MD
802static __inline
803int
c8fe38ae 804pmap_track_modified(vm_offset_t va)
d7f50089 805{
c8fe38ae
MD
806 if ((va < clean_sva) || (va >= clean_eva))
807 return 1;
808 else
809 return 0;
d7f50089
YY
810}
811
d7f50089 812/*
c8fe38ae
MD
813 * pmap_extract:
814 *
815 * Extract the physical page address associated with the map/VA pair.
816 *
817 * This function may not be called from an interrupt if the pmap is
818 * not kernel_pmap.
d7f50089 819 */
c8fe38ae
MD
820vm_paddr_t
821pmap_extract(pmap_t pmap, vm_offset_t va)
d7f50089 822{
48ffc236
JG
823 vm_paddr_t rtval;
824 pt_entry_t *pte;
825 pd_entry_t pde, *pdep;
c8fe38ae 826
48ffc236
JG
827 rtval = 0;
828 pdep = pmap_pde(pmap, va);
829 if (pdep != NULL) {
830 pde = *pdep;
831 if (pde) {
832 if ((pde & PG_PS) != 0) {
833 rtval = (pde & PG_PS_FRAME) | (va & PDRMASK);
834 } else {
835 pte = pmap_pde_to_pte(pdep, va);
836 rtval = (*pte & PG_FRAME) | (va & PAGE_MASK);
837 }
c8fe38ae 838 }
c8fe38ae 839 }
48ffc236
JG
840 return rtval;
841}
842
843/*
844 * Routine: pmap_kextract
845 * Function:
846 * Extract the physical page address associated
847 * kernel virtual address.
848 */
849vm_paddr_t
850pmap_kextract(vm_offset_t va)
48ffc236
JG
851{
852 pd_entry_t pde;
853 vm_paddr_t pa;
854
855 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
856 pa = DMAP_TO_PHYS(va);
857 } else {
858 pde = *vtopde(va);
859 if (pde & PG_PS) {
860 pa = (pde & PG_PS_FRAME) | (va & PDRMASK);
861 } else {
862 /*
863 * Beware of a concurrent promotion that changes the
864 * PDE at this point! For example, vtopte() must not
865 * be used to access the PTE because it would use the
866 * new PDE. It is, however, safe to use the old PDE
867 * because the page table page is preserved by the
868 * promotion.
869 */
870 pa = *pmap_pde_to_pte(&pde, va);
871 pa = (pa & PG_FRAME) | (va & PAGE_MASK);
872 }
873 }
874 return pa;
d7f50089
YY
875}
876
c8fe38ae
MD
877/***************************************************
878 * Low level mapping routines.....
879 ***************************************************/
880
d7f50089 881/*
c8fe38ae
MD
882 * Routine: pmap_kenter
883 * Function:
884 * Add a wired page to the KVA
885 * NOTE! note that in order for the mapping to take effect -- you
886 * should do an invltlb after doing the pmap_kenter().
d7f50089 887 */
c8fe38ae 888void
d7f50089
YY
889pmap_kenter(vm_offset_t va, vm_paddr_t pa)
890{
c8fe38ae
MD
891 pt_entry_t *pte;
892 pt_entry_t npte;
893 pmap_inval_info info;
894
895 pmap_inval_init(&info);
896 npte = pa | PG_RW | PG_V | pgeflag;
897 pte = vtopte(va);
898 pmap_inval_add(&info, &kernel_pmap, va);
899 *pte = npte;
900 pmap_inval_flush(&info);
d7f50089
YY
901}
902
903/*
c8fe38ae
MD
904 * Routine: pmap_kenter_quick
905 * Function:
906 * Similar to pmap_kenter(), except we only invalidate the
907 * mapping on the current CPU.
d7f50089 908 */
c8fe38ae
MD
909void
910pmap_kenter_quick(vm_offset_t va, vm_paddr_t pa)
911{
912 pt_entry_t *pte;
913 pt_entry_t npte;
914
915 npte = pa | PG_RW | PG_V | pgeflag;
916 pte = vtopte(va);
917 *pte = npte;
918 cpu_invlpg((void *)va);
919}
920
d7f50089
YY
921void
922pmap_kenter_sync(vm_offset_t va)
923{
c8fe38ae
MD
924 pmap_inval_info info;
925
926 pmap_inval_init(&info);
927 pmap_inval_add(&info, &kernel_pmap, va);
928 pmap_inval_flush(&info);
d7f50089
YY
929}
930
d7f50089
YY
931void
932pmap_kenter_sync_quick(vm_offset_t va)
933{
c8fe38ae 934 cpu_invlpg((void *)va);
d7f50089
YY
935}
936
d7f50089 937/*
c8fe38ae 938 * remove a page from the kernel pagetables
d7f50089
YY
939 */
940void
c8fe38ae 941pmap_kremove(vm_offset_t va)
d7f50089 942{
c8fe38ae
MD
943 pt_entry_t *pte;
944 pmap_inval_info info;
945
946 pmap_inval_init(&info);
947 pte = vtopte(va);
948 pmap_inval_add(&info, &kernel_pmap, va);
949 *pte = 0;
950 pmap_inval_flush(&info);
951}
952
953void
954pmap_kremove_quick(vm_offset_t va)
955{
956 pt_entry_t *pte;
957 pte = vtopte(va);
958 *pte = 0;
959 cpu_invlpg((void *)va);
d7f50089
YY
960}
961
962/*
c8fe38ae 963 * XXX these need to be recoded. They are not used in any critical path.
d7f50089
YY
964 */
965void
c8fe38ae 966pmap_kmodify_rw(vm_offset_t va)
d7f50089 967{
c8fe38ae
MD
968 *vtopte(va) |= PG_RW;
969 cpu_invlpg((void *)va);
d7f50089
YY
970}
971
c8fe38ae
MD
972void
973pmap_kmodify_nc(vm_offset_t va)
974{
975 *vtopte(va) |= PG_N;
976 cpu_invlpg((void *)va);
977}
d7f50089
YY
978
979/*
c8fe38ae
MD
980 * Used to map a range of physical addresses into kernel
981 * virtual address space.
982 *
983 * For now, VM is already on, we only need to map the
984 * specified memory.
d7f50089
YY
985 */
986vm_offset_t
987pmap_map(vm_offset_t virt, vm_paddr_t start, vm_paddr_t end, int prot)
988{
8fdd3267 989 return PHYS_TO_DMAP(start);
d7f50089
YY
990}
991
c8fe38ae 992
d7f50089 993/*
c8fe38ae
MD
994 * Add a list of wired pages to the kva
995 * this routine is only used for temporary
996 * kernel mappings that do not need to have
997 * page modification or references recorded.
998 * Note that old mappings are simply written
999 * over. The page *must* be wired.
d7f50089
YY
1000 */
1001void
c8fe38ae 1002pmap_qenter(vm_offset_t va, vm_page_t *m, int count)
d7f50089 1003{
c8fe38ae
MD
1004 vm_offset_t end_va;
1005
1006 end_va = va + count * PAGE_SIZE;
1007
1008 while (va < end_va) {
1009 pt_entry_t *pte;
1010
1011 pte = vtopte(va);
1012 *pte = VM_PAGE_TO_PHYS(*m) | PG_RW | PG_V | pgeflag;
1013 cpu_invlpg((void *)va);
1014 va += PAGE_SIZE;
1015 m++;
1016 }
1017#ifdef SMP
1018 smp_invltlb(); /* XXX */
1019#endif
1020}
1021
1022void
1023pmap_qenter2(vm_offset_t va, vm_page_t *m, int count, cpumask_t *mask)
1024{
1025 vm_offset_t end_va;
1026 cpumask_t cmask = mycpu->gd_cpumask;
1027
1028 end_va = va + count * PAGE_SIZE;
1029
1030 while (va < end_va) {
1031 pt_entry_t *pte;
1032 pt_entry_t pteval;
1033
1034 /*
1035 * Install the new PTE. If the pte changed from the prior
1036 * mapping we must reset the cpu mask and invalidate the page.
1037 * If the pte is the same but we have not seen it on the
1038 * current cpu, invlpg the existing mapping. Otherwise the
1039 * entry is optimal and no invalidation is required.
1040 */
1041 pte = vtopte(va);
1042 pteval = VM_PAGE_TO_PHYS(*m) | PG_A | PG_RW | PG_V | pgeflag;
1043 if (*pte != pteval) {
1044 *mask = 0;
1045 *pte = pteval;
1046 cpu_invlpg((void *)va);
1047 } else if ((*mask & cmask) == 0) {
1048 cpu_invlpg((void *)va);
1049 }
1050 va += PAGE_SIZE;
1051 m++;
1052 }
1053 *mask |= cmask;
d7f50089
YY
1054}
1055
1056/*
7155fc7d 1057 * This routine jerks page mappings from the
c8fe38ae 1058 * kernel -- it is meant only for temporary mappings.
7155fc7d
MD
1059 *
1060 * MPSAFE, INTERRUPT SAFE (cluster callback)
d7f50089 1061 */
c8fe38ae
MD
1062void
1063pmap_qremove(vm_offset_t va, int count)
d7f50089 1064{
c8fe38ae
MD
1065 vm_offset_t end_va;
1066
48ffc236 1067 end_va = va + count * PAGE_SIZE;
c8fe38ae
MD
1068
1069 while (va < end_va) {
1070 pt_entry_t *pte;
1071
1072 pte = vtopte(va);
1073 *pte = 0;
1074 cpu_invlpg((void *)va);
1075 va += PAGE_SIZE;
1076 }
1077#ifdef SMP
1078 smp_invltlb();
1079#endif
d7f50089
YY
1080}
1081
1082/*
c8fe38ae
MD
1083 * This routine works like vm_page_lookup() but also blocks as long as the
1084 * page is busy. This routine does not busy the page it returns.
1085 *
1086 * Unless the caller is managing objects whos pages are in a known state,
1087 * the call should be made with a critical section held so the page's object
1088 * association remains valid on return.
d7f50089 1089 */
bfc09ba0
MD
1090static
1091vm_page_t
c8fe38ae 1092pmap_page_lookup(vm_object_t object, vm_pindex_t pindex)
d7f50089 1093{
c8fe38ae
MD
1094 vm_page_t m;
1095
1096 do {
1097 m = vm_page_lookup(object, pindex);
1098 } while (m && vm_page_sleep_busy(m, FALSE, "pplookp"));
1099
1100 return(m);
d7f50089
YY
1101}
1102
1103/*
c8fe38ae
MD
1104 * Create a new thread and optionally associate it with a (new) process.
1105 * NOTE! the new thread's cpu may not equal the current cpu.
d7f50089
YY
1106 */
1107void
c8fe38ae 1108pmap_init_thread(thread_t td)
d7f50089 1109{
c8fe38ae
MD
1110 /* enforce pcb placement */
1111 td->td_pcb = (struct pcb *)(td->td_kstack + td->td_kstack_size) - 1;
1112 td->td_savefpu = &td->td_pcb->pcb_save;
48ffc236 1113 td->td_sp = (char *)td->td_pcb - 16; /* JG is -16 needed on amd64? */
d7f50089
YY
1114}
1115
1116/*
c8fe38ae 1117 * This routine directly affects the fork perf for a process.
d7f50089
YY
1118 */
1119void
c8fe38ae 1120pmap_init_proc(struct proc *p)
d7f50089
YY
1121{
1122}
1123
1124/*
c8fe38ae
MD
1125 * Dispose the UPAGES for a process that has exited.
1126 * This routine directly impacts the exit perf of a process.
d7f50089
YY
1127 */
1128void
c8fe38ae 1129pmap_dispose_proc(struct proc *p)
d7f50089 1130{
c8fe38ae 1131 KASSERT(p->p_lock == 0, ("attempt to dispose referenced proc! %p", p));
d7f50089
YY
1132}
1133
c8fe38ae
MD
1134/***************************************************
1135 * Page table page management routines.....
1136 ***************************************************/
1137
d7f50089 1138/*
c8fe38ae
MD
1139 * This routine unholds page table pages, and if the hold count
1140 * drops to zero, then it decrements the wire count.
d7f50089 1141 */
bfc09ba0
MD
1142static __inline
1143int
1144pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m,
1145 pmap_inval_info_t info)
1146{
1147 KKASSERT(m->hold_count > 0);
1148 if (m->hold_count > 1) {
1149 vm_page_unhold(m);
1150 return 0;
1151 } else {
1152 return _pmap_unwire_pte_hold(pmap, va, m, info);
1153 }
1154}
1155
1156static
1157int
1158_pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m,
1159 pmap_inval_info_t info)
c8fe38ae
MD
1160{
1161 /*
1162 * Wait until we can busy the page ourselves. We cannot have
1163 * any active flushes if we block.
1164 */
1165 if (m->flags & PG_BUSY) {
1166 pmap_inval_flush(info);
1167 while (vm_page_sleep_busy(m, FALSE, "pmuwpt"))
1168 ;
1169 }
1170 KASSERT(m->queue == PQ_NONE,
1171 ("_pmap_unwire_pte_hold: %p->queue != PQ_NONE", m));
1172
1173 if (m->hold_count == 1) {
1174 /*
1175 * Unmap the page table page
1176 */
1177 vm_page_busy(m);
1178 pmap_inval_add(info, pmap, -1);
48ffc236
JG
1179
1180 if (m->pindex >= (NUPDE + NUPDPE)) {
1181 /* PDP page */
1182 pml4_entry_t *pml4;
1183 pml4 = pmap_pml4e(pmap, va);
1184 *pml4 = 0;
1185 } else if (m->pindex >= NUPDE) {
1186 /* PD page */
1187 pdp_entry_t *pdp;
1188 pdp = pmap_pdpe(pmap, va);
1189 *pdp = 0;
1190 } else {
3535204a 1191 /* PT page */
48ffc236
JG
1192 pd_entry_t *pd;
1193 pd = pmap_pde(pmap, va);
1194 *pd = 0;
1195 }
c8fe38ae
MD
1196
1197 KKASSERT(pmap->pm_stats.resident_count > 0);
1198 --pmap->pm_stats.resident_count;
1199
1200 if (pmap->pm_ptphint == m)
1201 pmap->pm_ptphint = NULL;
1202
48ffc236
JG
1203 if (m->pindex < NUPDE) {
1204 /* We just released a PT, unhold the matching PD */
1205 vm_page_t pdpg;
1206
1207 pdpg = PHYS_TO_VM_PAGE(*pmap_pdpe(pmap, va) & PG_FRAME);
1208 pmap_unwire_pte_hold(pmap, va, pdpg, info);
1209 }
1210 if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) {
1211 /* We just released a PD, unhold the matching PDP */
1212 vm_page_t pdppg;
1213
1214 pdppg = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, va) & PG_FRAME);
1215 pmap_unwire_pte_hold(pmap, va, pdppg, info);
1216 }
48ffc236 1217
c8fe38ae
MD
1218 /*
1219 * This was our last hold, the page had better be unwired
1220 * after we decrement wire_count.
1221 *
1222 * FUTURE NOTE: shared page directory page could result in
1223 * multiple wire counts.
1224 */
1225 vm_page_unhold(m);
1226 --m->wire_count;
1227 KKASSERT(m->wire_count == 0);
1228 --vmstats.v_wire_count;
1229 vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE);
1230 vm_page_flash(m);
1231 vm_page_free_zero(m);
1232 return 1;
1233 } else {
1b2e0b92 1234 /* JG Can we get here? */
c8fe38ae
MD
1235 KKASSERT(m->hold_count > 1);
1236 vm_page_unhold(m);
1237 return 0;
1238 }
1239}
1240
c8fe38ae
MD
1241/*
1242 * After removing a page table entry, this routine is used to
1243 * conditionally free the page, and manage the hold/wire counts.
d7f50089 1244 */
bfc09ba0
MD
1245static
1246int
c8fe38ae
MD
1247pmap_unuse_pt(pmap_t pmap, vm_offset_t va, vm_page_t mpte,
1248 pmap_inval_info_t info)
1249{
48ffc236 1250 /* JG Use FreeBSD/amd64 or FreeBSD/i386 ptepde approaches? */
c8fe38ae 1251 vm_pindex_t ptepindex;
48ffc236 1252 if (va >= VM_MAX_USER_ADDRESS)
c8fe38ae
MD
1253 return 0;
1254
1255 if (mpte == NULL) {
48ffc236
JG
1256 ptepindex = pmap_pde_pindex(va);
1257#if JGHINT
c8fe38ae
MD
1258 if (pmap->pm_ptphint &&
1259 (pmap->pm_ptphint->pindex == ptepindex)) {
1260 mpte = pmap->pm_ptphint;
1261 } else {
48ffc236 1262#endif
c8fe38ae 1263 pmap_inval_flush(info);
48ffc236 1264 mpte = pmap_page_lookup(pmap->pm_pteobj, ptepindex);
c8fe38ae 1265 pmap->pm_ptphint = mpte;
48ffc236 1266#if JGHINT
c8fe38ae 1267 }
48ffc236 1268#endif
c8fe38ae 1269 }
48ffc236 1270 return pmap_unwire_pte_hold(pmap, va, mpte, info);
c8fe38ae 1271}
d7f50089
YY
1272
1273/*
c8fe38ae
MD
1274 * Initialize pmap0/vmspace0. This pmap is not added to pmap_list because
1275 * it, and IdlePTD, represents the template used to update all other pmaps.
1276 *
1277 * On architectures where the kernel pmap is not integrated into the user
1278 * process pmap, this pmap represents the process pmap, not the kernel pmap.
1279 * kernel_pmap should be used to directly access the kernel_pmap.
d7f50089
YY
1280 */
1281void
c8fe38ae 1282pmap_pinit0(struct pmap *pmap)
d7f50089 1283{
48ffc236 1284 pmap->pm_pml4 = (pml4_entry_t *)(PTOV_OFFSET + KPML4phys);
c8fe38ae
MD
1285 pmap->pm_count = 1;
1286 pmap->pm_active = 0;
1287 pmap->pm_ptphint = NULL;
1288 TAILQ_INIT(&pmap->pm_pvlist);
1289 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
d7f50089
YY
1290}
1291
1292/*
c8fe38ae
MD
1293 * Initialize a preallocated and zeroed pmap structure,
1294 * such as one in a vmspace structure.
d7f50089
YY
1295 */
1296void
c8fe38ae 1297pmap_pinit(struct pmap *pmap)
d7f50089 1298{
c8fe38ae
MD
1299 vm_page_t ptdpg;
1300
1301 /*
1302 * No need to allocate page table space yet but we do need a valid
1303 * page directory table.
1304 */
48ffc236
JG
1305 if (pmap->pm_pml4 == NULL) {
1306 pmap->pm_pml4 =
1307 (pml4_entry_t *)kmem_alloc_pageable(&kernel_map, PAGE_SIZE);
c8fe38ae
MD
1308 }
1309
1310 /*
1311 * Allocate an object for the ptes
1312 */
1313 if (pmap->pm_pteobj == NULL)
0a5c555b 1314 pmap->pm_pteobj = vm_object_allocate(OBJT_DEFAULT, NUPDE + NUPDPE + PML4PML4I + 1);
c8fe38ae
MD
1315
1316 /*
1317 * Allocate the page directory page, unless we already have
1318 * one cached. If we used the cached page the wire_count will
1319 * already be set appropriately.
1320 */
1321 if ((ptdpg = pmap->pm_pdirm) == NULL) {
0a5c555b 1322 ptdpg = vm_page_grab(pmap->pm_pteobj, NUPDE + NUPDPE + PML4PML4I,
c8fe38ae
MD
1323 VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
1324 pmap->pm_pdirm = ptdpg;
1325 vm_page_flag_clear(ptdpg, PG_MAPPED | PG_BUSY);
1326 ptdpg->valid = VM_PAGE_BITS_ALL;
1327 ptdpg->wire_count = 1;
1328 ++vmstats.v_wire_count;
48ffc236 1329 pmap_kenter((vm_offset_t)pmap->pm_pml4, VM_PAGE_TO_PHYS(ptdpg));
c8fe38ae
MD
1330 }
1331 if ((ptdpg->flags & PG_ZERO) == 0)
48ffc236 1332 bzero(pmap->pm_pml4, PAGE_SIZE);
c8fe38ae 1333
48ffc236
JG
1334 pmap->pm_pml4[KPML4I] = KPDPphys | PG_RW | PG_V | PG_U;
1335 pmap->pm_pml4[DMPML4I] = DMPDPphys | PG_RW | PG_V | PG_U;
c8fe38ae
MD
1336
1337 /* install self-referential address mapping entry */
48ffc236 1338 pmap->pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(ptdpg) | PG_V | PG_RW | PG_A | PG_M;
c8fe38ae
MD
1339
1340 pmap->pm_count = 1;
1341 pmap->pm_active = 0;
1342 pmap->pm_ptphint = NULL;
1343 TAILQ_INIT(&pmap->pm_pvlist);
1344 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
1345 pmap->pm_stats.resident_count = 1;
d7f50089
YY
1346}
1347
1348/*
c8fe38ae
MD
1349 * Clean up a pmap structure so it can be physically freed. This routine
1350 * is called by the vmspace dtor function. A great deal of pmap data is
1351 * left passively mapped to improve vmspace management so we have a bit
1352 * of cleanup work to do here.
d7f50089
YY
1353 */
1354void
c8fe38ae 1355pmap_puninit(pmap_t pmap)
d7f50089 1356{
c8fe38ae
MD
1357 vm_page_t p;
1358
1359 KKASSERT(pmap->pm_active == 0);
1360 if ((p = pmap->pm_pdirm) != NULL) {
48ffc236 1361 KKASSERT(pmap->pm_pml4 != NULL);
bfc09ba0 1362 KKASSERT(pmap->pm_pml4 != (void *)(PTOV_OFFSET + KPML4phys));
48ffc236 1363 pmap_kremove((vm_offset_t)pmap->pm_pml4);
c8fe38ae
MD
1364 p->wire_count--;
1365 vmstats.v_wire_count--;
1366 KKASSERT((p->flags & PG_BUSY) == 0);
1367 vm_page_busy(p);
1368 vm_page_free_zero(p);
1369 pmap->pm_pdirm = NULL;
1370 }
48ffc236 1371 if (pmap->pm_pml4) {
bfc09ba0 1372 KKASSERT(pmap->pm_pml4 != (void *)(PTOV_OFFSET + KPML4phys));
48ffc236
JG
1373 kmem_free(&kernel_map, (vm_offset_t)pmap->pm_pml4, PAGE_SIZE);
1374 pmap->pm_pml4 = NULL;
c8fe38ae
MD
1375 }
1376 if (pmap->pm_pteobj) {
1377 vm_object_deallocate(pmap->pm_pteobj);
1378 pmap->pm_pteobj = NULL;
1379 }
d7f50089
YY
1380}
1381
1382/*
c8fe38ae
MD
1383 * Wire in kernel global address entries. To avoid a race condition
1384 * between pmap initialization and pmap_growkernel, this procedure
1385 * adds the pmap to the master list (which growkernel scans to update),
1386 * then copies the template.
d7f50089
YY
1387 */
1388void
c8fe38ae 1389pmap_pinit2(struct pmap *pmap)
d7f50089 1390{
c8fe38ae
MD
1391 crit_enter();
1392 TAILQ_INSERT_TAIL(&pmap_list, pmap, pm_pmnode);
1393 /* XXX copies current process, does not fill in MPPTDI */
c8fe38ae 1394 crit_exit();
d7f50089
YY
1395}
1396
1397/*
c8fe38ae
MD
1398 * Attempt to release and free a vm_page in a pmap. Returns 1 on success,
1399 * 0 on failure (if the procedure had to sleep).
d7f50089 1400 *
c8fe38ae
MD
1401 * When asked to remove the page directory page itself, we actually just
1402 * leave it cached so we do not have to incur the SMP inval overhead of
1403 * removing the kernel mapping. pmap_puninit() will take care of it.
d7f50089 1404 */
bfc09ba0
MD
1405static
1406int
c8fe38ae 1407pmap_release_free_page(struct pmap *pmap, vm_page_t p)
d7f50089 1408{
c8fe38ae
MD
1409 /*
1410 * This code optimizes the case of freeing non-busy
1411 * page-table pages. Those pages are zero now, and
1412 * might as well be placed directly into the zero queue.
1413 */
1414 if (vm_page_sleep_busy(p, FALSE, "pmaprl"))
d7f50089 1415 return 0;
d7f50089 1416
c8fe38ae
MD
1417 vm_page_busy(p);
1418
1419 /*
1420 * Remove the page table page from the processes address space.
1421 */
4a4ea614
MD
1422 if (p->pindex == NUPDE + NUPDPE + PML4PML4I) {
1423 /*
1424 * We are the pml4 table itself.
1425 */
1426 /* XXX anything to do here? */
1427 } else if (p->pindex >= (NUPDE + NUPDPE)) {
1b2e0b92
JG
1428 /*
1429 * We are a PDP page.
1430 * We look for the PML4 entry that points to us.
1431 */
1432 vm_page_t m4 = vm_page_lookup(pmap->pm_pteobj, NUPDE + NUPDPE + PML4PML4I);
1433 KKASSERT(m4 != NULL);
bfc09ba0 1434 pml4_entry_t *pml4 = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m4));
1b2e0b92
JG
1435 int idx = (p->pindex - (NUPDE + NUPDPE)) % NPML4EPG;
1436 KKASSERT(pml4[idx] != 0);
1437 pml4[idx] = 0;
1438 m4->hold_count--;
1439 /* JG What about wire_count? */
1440 } else if (p->pindex >= NUPDE) {
1441 /*
1442 * We are a PD page.
1443 * We look for the PDP entry that points to us.
1444 */
1445 vm_page_t m3 = vm_page_lookup(pmap->pm_pteobj, NUPDE + NUPDPE + (p->pindex - NUPDE) / NPDPEPG);
1446 KKASSERT(m3 != NULL);
bfc09ba0 1447 pdp_entry_t *pdp = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m3));
1b2e0b92
JG
1448 int idx = (p->pindex - NUPDE) % NPDPEPG;
1449 KKASSERT(pdp[idx] != 0);
1450 pdp[idx] = 0;
1451 m3->hold_count--;
1452 /* JG What about wire_count? */
1453 } else {
1454 /* We are a PT page.
1455 * We look for the PD entry that points to us.
1456 */
1457 vm_page_t m2 = vm_page_lookup(pmap->pm_pteobj, NUPDE + p->pindex / NPDEPG);
1458 KKASSERT(m2 != NULL);
bfc09ba0 1459 pd_entry_t *pd = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m2));
1b2e0b92
JG
1460 int idx = p->pindex % NPDEPG;
1461 pd[idx] = 0;
1462 m2->hold_count--;
1463 /* JG What about wire_count? */
1464 }
c8fe38ae
MD
1465 KKASSERT(pmap->pm_stats.resident_count > 0);
1466 --pmap->pm_stats.resident_count;
1467
1468 if (p->hold_count) {
1469 panic("pmap_release: freeing held page table page");
1470 }
1471 if (pmap->pm_ptphint && (pmap->pm_ptphint->pindex == p->pindex))
1472 pmap->pm_ptphint = NULL;
1473
1b2e0b92
JG
1474 /*
1475 * We leave the top-level page table page cached, wired, and mapped in
1476 * the pmap until the dtor function (pmap_puninit()) gets called.
1477 * However, still clean it up so we can set PG_ZERO.
1478 */
1479 if (p->pindex == NUPDE + NUPDPE + PML4PML4I) {
1480 bzero(pmap->pm_pml4, PAGE_SIZE);
1481 vm_page_flag_set(p, PG_ZERO);
1482 vm_page_wakeup(p);
1483 } else {
1484 p->wire_count--;
1485 vmstats.v_wire_count--;
1486 /* JG eventually revert to using vm_page_free_zero() */
1487 vm_page_free(p);
1488 }
c8fe38ae
MD
1489 return 1;
1490}
d7f50089
YY
1491
1492/*
e8510e54
MD
1493 * This routine is called when various levels in the page table need to
1494 * be populated. This routine cannot fail.
d7f50089 1495 */
bfc09ba0
MD
1496static
1497vm_page_t
c8fe38ae
MD
1498_pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex)
1499{
e8510e54 1500 vm_page_t m;
c8fe38ae
MD
1501
1502 /*
e8510e54 1503 * Find or fabricate a new pagetable page. This will busy the page.
c8fe38ae
MD
1504 */
1505 m = vm_page_grab(pmap->pm_pteobj, ptepindex,
1506 VM_ALLOC_NORMAL | VM_ALLOC_ZERO | VM_ALLOC_RETRY);
1507
48ffc236
JG
1508
1509 if ((m->flags & PG_ZERO) == 0) {
1510 pmap_zero_page(VM_PAGE_TO_PHYS(m));
1511 }
1512
c8fe38ae
MD
1513 KASSERT(m->queue == PQ_NONE,
1514 ("_pmap_allocpte: %p->queue != PQ_NONE", m));
1515
1516 /*
1517 * Increment the hold count for the page we will be returning to
1518 * the caller.
1519 */
1520 m->hold_count++;
c8fe38ae
MD
1521 if (m->wire_count == 0)
1522 vmstats.v_wire_count++;
1523 m->wire_count++;
1524
c8fe38ae
MD
1525 /*
1526 * Map the pagetable page into the process address space, if
1527 * it isn't already there.
e8510e54
MD
1528 *
1529 * It is possible that someone else got in and mapped the page
1530 * directory page while we were blocked, if so just unbusy and
1531 * return the held page.
c8fe38ae 1532 */
c8fe38ae
MD
1533 ++pmap->pm_stats.resident_count;
1534
48ffc236 1535 if (ptepindex >= (NUPDE + NUPDPE)) {
e8510e54
MD
1536 /*
1537 * Wire up a new PDP page in the PML4
1538 */
48ffc236
JG
1539 pml4_entry_t *pml4;
1540 vm_pindex_t pml4index;
1541
48ffc236
JG
1542 pml4index = ptepindex - (NUPDE + NUPDPE);
1543 pml4 = &pmap->pm_pml4[pml4index];
e8510e54
MD
1544 if (*pml4 & PG_V) {
1545 --m->wire_count;
1546 vm_page_wakeup(m);
1547 return(m);
1548 }
48ffc236 1549 *pml4 = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
48ffc236 1550 } else if (ptepindex >= NUPDE) {
e8510e54
MD
1551 /*
1552 * Wire up a new PD page in the PDP
1553 */
48ffc236
JG
1554 vm_pindex_t pml4index;
1555 vm_pindex_t pdpindex;
e8510e54 1556 vm_page_t pdppg;
48ffc236
JG
1557 pml4_entry_t *pml4;
1558 pdp_entry_t *pdp;
1559
48ffc236
JG
1560 pdpindex = ptepindex - NUPDE;
1561 pml4index = pdpindex >> NPML4EPGSHIFT;
1562
1563 pml4 = &pmap->pm_pml4[pml4index];
1564 if ((*pml4 & PG_V) == 0) {
e8510e54
MD
1565 /*
1566 * Have to allocate a new PDP page, recurse.
1567 * This always succeeds. Returned page will
1568 * be held.
1569 */
1570 pdppg = _pmap_allocpte(pmap,
1571 NUPDE + NUPDPE + pml4index);
48ffc236 1572 } else {
e8510e54
MD
1573 /*
1574 * Add a held reference to the PDP page.
1575 */
48ffc236 1576 pdppg = PHYS_TO_VM_PAGE(*pml4 & PG_FRAME);
1b2e0b92 1577 pdppg->hold_count++;
48ffc236 1578 }
c8fe38ae 1579
e8510e54
MD
1580 /*
1581 * Now find the pdp_entry and map the PDP. If the PDP
1582 * has already been mapped unwind and return the
1583 * already-mapped PDP held.
1584 */
1585 pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
48ffc236 1586 pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
e8510e54
MD
1587 if (*pdp & PG_V) {
1588 vm_page_unhold(pdppg);
1589 --m->wire_count;
1590 vm_page_wakeup(m);
1591 return(m);
1592 }
48ffc236 1593 *pdp = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
48ffc236 1594 } else {
e8510e54
MD
1595 /*
1596 * Wire up the new PT page in the PD
1597 */
48ffc236
JG
1598 vm_pindex_t pml4index;
1599 vm_pindex_t pdpindex;
1600 pml4_entry_t *pml4;
1601 pdp_entry_t *pdp;
1602 pd_entry_t *pd;
e8510e54 1603 vm_page_t pdpg;
48ffc236 1604
48ffc236
JG
1605 pdpindex = ptepindex >> NPDPEPGSHIFT;
1606 pml4index = pdpindex >> NPML4EPGSHIFT;
1607
e8510e54
MD
1608 /*
1609 * Locate the PDP page in the PML4, then the PD page in
1610 * the PDP. If either does not exist we simply recurse
1611 * to allocate them.
1612 *
1613 * We can just recurse on the PD page as it will recurse
1614 * on the PDP if necessary.
1615 */
48ffc236
JG
1616 pml4 = &pmap->pm_pml4[pml4index];
1617 if ((*pml4 & PG_V) == 0) {
e8510e54 1618 pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex);
48ffc236
JG
1619 pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
1620 pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
c8fe38ae 1621 } else {
48ffc236
JG
1622 pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
1623 pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
1624 if ((*pdp & PG_V) == 0) {
e8510e54 1625 pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex);
48ffc236 1626 } else {
48ffc236 1627 pdpg = PHYS_TO_VM_PAGE(*pdp & PG_FRAME);
1b2e0b92 1628 pdpg->hold_count++;
48ffc236 1629 }
c8fe38ae 1630 }
48ffc236 1631
e8510e54
MD
1632 /*
1633 * Now fill in the pte in the PD. If the pte already exists
1634 * (again, if we raced the grab), unhold pdpg and unwire
1635 * m, returning a held m.
1636 */
1637 pd = (pd_entry_t *)PHYS_TO_DMAP(*pdp & PG_FRAME);
48ffc236 1638 pd = &pd[ptepindex & ((1ul << NPDEPGSHIFT) - 1)];
e8510e54
MD
1639 if (*pd == 0) {
1640 *pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW |
1641 PG_V | PG_A | PG_M;
1642 } else {
1643 vm_page_unhold(pdpg);
1644 --m->wire_count;
1645 vm_page_wakeup(m);
1646 return(m);
1647 }
c8fe38ae
MD
1648 }
1649
48ffc236 1650 /*
e8510e54
MD
1651 * We successfully loaded a PDP, PD, or PTE. Set the page table hint,
1652 * valid bits, mapped flag, unbusy, and we're done.
48ffc236
JG
1653 */
1654 pmap->pm_ptphint = m;
1655
c8fe38ae
MD
1656 m->valid = VM_PAGE_BITS_ALL;
1657 vm_page_flag_clear(m, PG_ZERO);
1658 vm_page_flag_set(m, PG_MAPPED);
1659 vm_page_wakeup(m);
1660
e8510e54 1661 return (m);
c8fe38ae
MD
1662}
1663
bfc09ba0
MD
1664static
1665vm_page_t
c8fe38ae 1666pmap_allocpte(pmap_t pmap, vm_offset_t va)
d7f50089 1667{
c8fe38ae 1668 vm_pindex_t ptepindex;
48ffc236 1669 pd_entry_t *pd;
c8fe38ae
MD
1670 vm_page_t m;
1671
1672 /*
1673 * Calculate pagetable page index
1674 */
48ffc236 1675 ptepindex = pmap_pde_pindex(va);
c8fe38ae
MD
1676
1677 /*
1678 * Get the page directory entry
1679 */
48ffc236 1680 pd = pmap_pde(pmap, va);
c8fe38ae
MD
1681
1682 /*
48ffc236 1683 * This supports switching from a 2MB page to a
c8fe38ae
MD
1684 * normal 4K page.
1685 */
48ffc236 1686 if (pd != NULL && (*pd & (PG_PS | PG_V)) == (PG_PS | PG_V)) {
1b2e0b92 1687 panic("no promotion/demotion yet");
48ffc236
JG
1688 *pd = 0;
1689 pd = NULL;
c8fe38ae
MD
1690 cpu_invltlb();
1691 smp_invltlb();
1692 }
1693
1694 /*
1695 * If the page table page is mapped, we just increment the
1696 * hold count, and activate it.
1697 */
48ffc236
JG
1698 if (pd != NULL && (*pd & PG_V) != 0) {
1699 /* YYY hint is used here on i386 */
1700 m = pmap_page_lookup( pmap->pm_pteobj, ptepindex);
1701 pmap->pm_ptphint = m;
c8fe38ae
MD
1702 m->hold_count++;
1703 return m;
1704 }
1705 /*
1706 * Here if the pte page isn't mapped, or if it has been deallocated.
1707 */
1708 return _pmap_allocpte(pmap, ptepindex);
d7f50089
YY
1709}
1710
c8fe38ae
MD
1711
1712/***************************************************
1713 * Pmap allocation/deallocation routines.
1714 ***************************************************/
1715
d7f50089 1716/*
c8fe38ae
MD
1717 * Release any resources held by the given physical map.
1718 * Called when a pmap initialized by pmap_pinit is being released.
1719 * Should only be called if the map contains no valid mappings.
d7f50089 1720 */
c8fe38ae 1721static int pmap_release_callback(struct vm_page *p, void *data);
d7f50089 1722
c8fe38ae
MD
1723void
1724pmap_release(struct pmap *pmap)
d7f50089 1725{
c8fe38ae
MD
1726 vm_object_t object = pmap->pm_pteobj;
1727 struct rb_vm_page_scan_info info;
1728
1729 KASSERT(pmap->pm_active == 0, ("pmap still active! %08x", pmap->pm_active));
1730#if defined(DIAGNOSTIC)
1731 if (object->ref_count != 1)
1732 panic("pmap_release: pteobj reference count != 1");
1733#endif
1734
1735 info.pmap = pmap;
1736 info.object = object;
1737 crit_enter();
1738 TAILQ_REMOVE(&pmap_list, pmap, pm_pmnode);
1739 crit_exit();
1740
1741 do {
1742 crit_enter();
1743 info.error = 0;
1744 info.mpte = NULL;
1745 info.limit = object->generation;
1746
1747 vm_page_rb_tree_RB_SCAN(&object->rb_memq, NULL,
1748 pmap_release_callback, &info);
1749 if (info.error == 0 && info.mpte) {
1750 if (!pmap_release_free_page(pmap, info.mpte))
1751 info.error = 1;
1752 }
1753 crit_exit();
1754 } while (info.error);
d7f50089
YY
1755}
1756
bfc09ba0
MD
1757static
1758int
c8fe38ae 1759pmap_release_callback(struct vm_page *p, void *data)
d7f50089 1760{
c8fe38ae
MD
1761 struct rb_vm_page_scan_info *info = data;
1762
0a5c555b 1763 if (p->pindex == NUPDE + NUPDPE + PML4PML4I) {
c8fe38ae
MD
1764 info->mpte = p;
1765 return(0);
1766 }
1767 if (!pmap_release_free_page(info->pmap, p)) {
1768 info->error = 1;
1769 return(-1);
1770 }
1771 if (info->object->generation != info->limit) {
1772 info->error = 1;
1773 return(-1);
1774 }
1775 return(0);
d7f50089
YY
1776}
1777
1778/*
c8fe38ae 1779 * Grow the number of kernel page table entries, if needed.
d7f50089 1780 */
c8fe38ae
MD
1781void
1782pmap_growkernel(vm_offset_t addr)
d7f50089 1783{
48ffc236 1784 vm_paddr_t paddr;
c8fe38ae
MD
1785 vm_offset_t ptppaddr;
1786 vm_page_t nkpg;
48ffc236
JG
1787 pd_entry_t *pde, newpdir;
1788 pdp_entry_t newpdp;
c8fe38ae
MD
1789
1790 crit_enter();
1791 if (kernel_vm_end == 0) {
1792 kernel_vm_end = KERNBASE;
1793 nkpt = 0;
48ffc236 1794 while ((*pmap_pde(&kernel_pmap, kernel_vm_end) & PG_V) != 0) {
c8fe38ae
MD
1795 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
1796 nkpt++;
48ffc236
JG
1797 if (kernel_vm_end - 1 >= kernel_map.max_offset) {
1798 kernel_vm_end = kernel_map.max_offset;
1799 break;
1800 }
c8fe38ae
MD
1801 }
1802 }
48ffc236
JG
1803 addr = roundup2(addr, PAGE_SIZE * NPTEPG);
1804 if (addr - 1 >= kernel_map.max_offset)
1805 addr = kernel_map.max_offset;
c8fe38ae 1806 while (kernel_vm_end < addr) {
48ffc236
JG
1807 pde = pmap_pde(&kernel_pmap, kernel_vm_end);
1808 if (pde == NULL) {
1809 /* We need a new PDP entry */
1810 nkpg = vm_page_alloc(kptobj, nkpt,
1811 VM_ALLOC_NORMAL | VM_ALLOC_SYSTEM
1812 | VM_ALLOC_INTERRUPT);
1813 if (nkpg == NULL)
1814 panic("pmap_growkernel: no memory to grow kernel");
48ffc236 1815 paddr = VM_PAGE_TO_PHYS(nkpg);
7f2a2740
MD
1816 if ((nkpg->flags & PG_ZERO) == 0)
1817 pmap_zero_page(paddr);
1818 vm_page_flag_clear(nkpg, PG_ZERO);
48ffc236
JG
1819 newpdp = (pdp_entry_t)
1820 (paddr | PG_V | PG_RW | PG_A | PG_M);
1821 *pmap_pdpe(&kernel_pmap, kernel_vm_end) = newpdp;
7f2a2740 1822 nkpt++;
48ffc236
JG
1823 continue; /* try again */
1824 }
1825 if ((*pde & PG_V) != 0) {
c8fe38ae 1826 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
48ffc236
JG
1827 if (kernel_vm_end - 1 >= kernel_map.max_offset) {
1828 kernel_vm_end = kernel_map.max_offset;
1829 break;
1830 }
c8fe38ae
MD
1831 continue;
1832 }
1833
1834 /*
1835 * This index is bogus, but out of the way
1836 */
48ffc236 1837 nkpg = vm_page_alloc(kptobj, nkpt,
c8fe38ae
MD
1838 VM_ALLOC_NORMAL | VM_ALLOC_SYSTEM | VM_ALLOC_INTERRUPT);
1839 if (nkpg == NULL)
1840 panic("pmap_growkernel: no memory to grow kernel");
1841
1842 vm_page_wire(nkpg);
1843 ptppaddr = VM_PAGE_TO_PHYS(nkpg);
1844 pmap_zero_page(ptppaddr);
7f2a2740 1845 vm_page_flag_clear(nkpg, PG_ZERO);
c8fe38ae 1846 newpdir = (pd_entry_t) (ptppaddr | PG_V | PG_RW | PG_A | PG_M);
c8fe38ae
MD
1847 *pmap_pde(&kernel_pmap, kernel_vm_end) = newpdir;
1848 nkpt++;
1849
48ffc236
JG
1850 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
1851 if (kernel_vm_end - 1 >= kernel_map.max_offset) {
1852 kernel_vm_end = kernel_map.max_offset;
1853 break;
c8fe38ae 1854 }
c8fe38ae
MD
1855 }
1856 crit_exit();
d7f50089
YY
1857}
1858
1859/*
c8fe38ae
MD
1860 * Retire the given physical map from service.
1861 * Should only be called if the map contains
1862 * no valid mappings.
d7f50089 1863 */
c8fe38ae
MD
1864void
1865pmap_destroy(pmap_t pmap)
d7f50089 1866{
c8fe38ae
MD
1867 int count;
1868
1869 if (pmap == NULL)
1870 return;
1871
1872 count = --pmap->pm_count;
1873 if (count == 0) {
1874 pmap_release(pmap);
1875 panic("destroying a pmap is not yet implemented");
1876 }
d7f50089
YY
1877}
1878
1879/*
c8fe38ae 1880 * Add a reference to the specified pmap.
d7f50089 1881 */
c8fe38ae
MD
1882void
1883pmap_reference(pmap_t pmap)
d7f50089 1884{
c8fe38ae
MD
1885 if (pmap != NULL) {
1886 pmap->pm_count++;
1887 }
d7f50089
YY
1888}
1889
c8fe38ae
MD
1890/***************************************************
1891* page management routines.
1892 ***************************************************/
d7f50089
YY
1893
1894/*
1895 * free the pv_entry back to the free list. This function may be
1896 * called from an interrupt.
1897 */
bfc09ba0
MD
1898static __inline
1899void
d7f50089
YY
1900free_pv_entry(pv_entry_t pv)
1901{
c8fe38ae 1902 pv_entry_count--;
48ffc236 1903 KKASSERT(pv_entry_count >= 0);
c8fe38ae 1904 zfree(pvzone, pv);
d7f50089
YY
1905}
1906
1907/*
1908 * get a new pv_entry, allocating a block from the system
1909 * when needed. This function may be called from an interrupt.
1910 */
bfc09ba0
MD
1911static
1912pv_entry_t
d7f50089
YY
1913get_pv_entry(void)
1914{
c8fe38ae
MD
1915 pv_entry_count++;
1916 if (pv_entry_high_water &&
48ffc236
JG
1917 (pv_entry_count > pv_entry_high_water) &&
1918 (pmap_pagedaemon_waken == 0)) {
c8fe38ae 1919 pmap_pagedaemon_waken = 1;
48ffc236 1920 wakeup(&vm_pages_needed);
c8fe38ae
MD
1921 }
1922 return zalloc(pvzone);
d7f50089
YY
1923}
1924
1925/*
1926 * This routine is very drastic, but can save the system
1927 * in a pinch.
1928 */
1929void
1930pmap_collect(void)
1931{
c8fe38ae
MD
1932 int i;
1933 vm_page_t m;
1934 static int warningdone=0;
1935
1936 if (pmap_pagedaemon_waken == 0)
1937 return;
1938
1939 if (warningdone < 5) {
1940 kprintf("pmap_collect: collecting pv entries -- suggest increasing PMAP_SHPGPERPROC\n");
1941 warningdone++;
1942 }
1943
1944 for(i = 0; i < vm_page_array_size; i++) {
1945 m = &vm_page_array[i];
1946 if (m->wire_count || m->hold_count || m->busy ||
1947 (m->flags & PG_BUSY))
1948 continue;
1949 pmap_remove_all(m);
1950 }
48ffc236 1951 pmap_pagedaemon_waken = 0;
d7f50089
YY
1952}
1953
c8fe38ae 1954
d7f50089
YY
1955/*
1956 * If it is the first entry on the list, it is actually
1957 * in the header and we must copy the following entry up
1958 * to the header. Otherwise we must search the list for
1959 * the entry. In either case we free the now unused entry.
1960 */
bfc09ba0
MD
1961static
1962int
c8fe38ae
MD
1963pmap_remove_entry(struct pmap *pmap, vm_page_t m,
1964 vm_offset_t va, pmap_inval_info_t info)
1965{
1966 pv_entry_t pv;
1967 int rtval;
1968
1969 crit_enter();
1970 if (m->md.pv_list_count < pmap->pm_stats.resident_count) {
1971 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
1972 if (pmap == pv->pv_pmap && va == pv->pv_va)
1973 break;
1974 }
1975 } else {
1976 TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) {
1977 if (va == pv->pv_va)
1978 break;
1979 }
1980 }
1981
1982 rtval = 0;
48ffc236 1983 /* JGXXX When can 'pv' be NULL? */
c8fe38ae
MD
1984 if (pv) {
1985 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
1986 m->md.pv_list_count--;
48ffc236 1987 KKASSERT(m->md.pv_list_count >= 0);
c8fe38ae
MD
1988 if (TAILQ_EMPTY(&m->md.pv_list))
1989 vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE);
1990 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
1991 ++pmap->pm_generation;
1992 rtval = pmap_unuse_pt(pmap, va, pv->pv_ptem, info);
1993 free_pv_entry(pv);
1994 }
1995 crit_exit();
1996 return rtval;
d7f50089
YY
1997}
1998
1999/*
c8fe38ae
MD
2000 * Create a pv entry for page at pa for
2001 * (pmap, va).
d7f50089 2002 */
bfc09ba0
MD
2003static
2004void
d7f50089
YY
2005pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t mpte, vm_page_t m)
2006{
c8fe38ae
MD
2007 pv_entry_t pv;
2008
2009 crit_enter();
2010 pv = get_pv_entry();
2011 pv->pv_va = va;
2012 pv->pv_pmap = pmap;
2013 pv->pv_ptem = mpte;
2014
2015 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist);
2016 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
2017 m->md.pv_list_count++;
2018
2019 crit_exit();
d7f50089
YY
2020}
2021
2022/*
2023 * pmap_remove_pte: do the things to unmap a page in a process
2024 */
bfc09ba0
MD
2025static
2026int
c8fe38ae
MD
2027pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va,
2028 pmap_inval_info_t info)
2029{
2030 pt_entry_t oldpte;
2031 vm_page_t m;
2032
2033 pmap_inval_add(info, pmap, va);
2034 oldpte = pte_load_clear(ptq);
2035 if (oldpte & PG_W)
2036 pmap->pm_stats.wired_count -= 1;
2037 /*
2038 * Machines that don't support invlpg, also don't support
2039 * PG_G. XXX PG_G is disabled for SMP so don't worry about
2040 * the SMP case.
2041 */
2042 if (oldpte & PG_G)
2043 cpu_invlpg((void *)va);
2044 KKASSERT(pmap->pm_stats.resident_count > 0);
2045 --pmap->pm_stats.resident_count;
2046 if (oldpte & PG_MANAGED) {
2047 m = PHYS_TO_VM_PAGE(oldpte);
2048 if (oldpte & PG_M) {
2049#if defined(PMAP_DIAGNOSTIC)
2050 if (pmap_nw_modified((pt_entry_t) oldpte)) {
2051 kprintf(
48ffc236 2052 "pmap_remove: modified page not writable: va: 0x%lx, pte: 0x%lx\n",
c8fe38ae
MD
2053 va, oldpte);
2054 }
2055#endif
2056 if (pmap_track_modified(va))
2057 vm_page_dirty(m);
2058 }
2059 if (oldpte & PG_A)
2060 vm_page_flag_set(m, PG_REFERENCED);
2061 return pmap_remove_entry(pmap, m, va, info);
2062 } else {
2063 return pmap_unuse_pt(pmap, va, NULL, info);
2064 }
2065
d7f50089
YY
2066 return 0;
2067}
2068
2069/*
2070 * pmap_remove_page:
2071 *
2072 * Remove a single page from a process address space.
2073 *
2074 * This function may not be called from an interrupt if the pmap is
2075 * not kernel_pmap.
2076 */
bfc09ba0
MD
2077static
2078void
c8fe38ae
MD
2079pmap_remove_page(struct pmap *pmap, vm_offset_t va, pmap_inval_info_t info)
2080{
48ffc236 2081 pt_entry_t *pte;
c8fe38ae 2082
48ffc236
JG
2083 pte = pmap_pte(pmap, va);
2084 if (pte == NULL)
2085 return;
2086 if ((*pte & PG_V) == 0)
2087 return;
2088 pmap_remove_pte(pmap, pte, va, info);
d7f50089
YY
2089}
2090
2091/*
2092 * pmap_remove:
2093 *
2094 * Remove the given range of addresses from the specified map.
2095 *
2096 * It is assumed that the start and end are properly
2097 * rounded to the page size.
2098 *
2099 * This function may not be called from an interrupt if the pmap is
2100 * not kernel_pmap.
2101 */
2102void
2103pmap_remove(struct pmap *pmap, vm_offset_t sva, vm_offset_t eva)
2104{
48ffc236
JG
2105 vm_offset_t va_next;
2106 pml4_entry_t *pml4e;
2107 pdp_entry_t *pdpe;
2108 pd_entry_t ptpaddr, *pde;
2109 pt_entry_t *pte;
c8fe38ae
MD
2110 struct pmap_inval_info info;
2111
2112 if (pmap == NULL)
2113 return;
2114
2115 if (pmap->pm_stats.resident_count == 0)
2116 return;
2117
2118 pmap_inval_init(&info);
2119
2120 /*
2121 * special handling of removing one page. a very
2122 * common operation and easy to short circuit some
2123 * code.
2124 */
48ffc236
JG
2125 if (sva + PAGE_SIZE == eva) {
2126 pde = pmap_pde(pmap, sva);
2127 if (pde && (*pde & PG_PS) == 0) {
2128 pmap_remove_page(pmap, sva, &info);
2129 pmap_inval_flush(&info);
2130 return;
2131 }
c8fe38ae
MD
2132 }
2133
48ffc236
JG
2134 for (; sva < eva; sva = va_next) {
2135 pml4e = pmap_pml4e(pmap, sva);
2136 if ((*pml4e & PG_V) == 0) {
2137 va_next = (sva + NBPML4) & ~PML4MASK;
2138 if (va_next < sva)
2139 va_next = eva;
2140 continue;
2141 }
c8fe38ae 2142
48ffc236
JG
2143 pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
2144 if ((*pdpe & PG_V) == 0) {
2145 va_next = (sva + NBPDP) & ~PDPMASK;
2146 if (va_next < sva)
2147 va_next = eva;
2148 continue;
2149 }
c8fe38ae
MD
2150
2151 /*
2152 * Calculate index for next page table.
2153 */
48ffc236
JG
2154 va_next = (sva + NBPDR) & ~PDRMASK;
2155 if (va_next < sva)
2156 va_next = eva;
c8fe38ae 2157
48ffc236
JG
2158 pde = pmap_pdpe_to_pde(pdpe, sva);
2159 ptpaddr = *pde;
c8fe38ae
MD
2160
2161 /*
48ffc236 2162 * Weed out invalid mappings.
c8fe38ae
MD
2163 */
2164 if (ptpaddr == 0)
2165 continue;
2166
48ffc236
JG
2167 /*
2168 * Check for large page.
2169 */
2170 if ((ptpaddr & PG_PS) != 0) {
2171 /* JG FreeBSD has more complex treatment here */
2172 pmap_inval_add(&info, pmap, -1);
2173 *pde = 0;
2174 pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
2175 continue;
2176 }
2177
c8fe38ae
MD
2178 /*
2179 * Limit our scan to either the end of the va represented
2180 * by the current page table page, or to the end of the
2181 * range being removed.
2182 */
48ffc236
JG
2183 if (va_next > eva)
2184 va_next = eva;
c8fe38ae
MD
2185
2186 /*
2187 * NOTE: pmap_remove_pte() can block.
2188 */
48ffc236
JG
2189 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
2190 sva += PAGE_SIZE) {
2191 if (*pte == 0)
c8fe38ae 2192 continue;
48ffc236 2193 if (pmap_remove_pte(pmap, pte, sva, &info))
c8fe38ae
MD
2194 break;
2195 }
2196 }
2197 pmap_inval_flush(&info);
d7f50089
YY
2198}
2199
2200/*
2201 * pmap_remove_all:
2202 *
c8fe38ae
MD
2203 * Removes this physical page from all physical maps in which it resides.
2204 * Reflects back modify bits to the pager.
d7f50089 2205 *
c8fe38ae 2206 * This routine may not be called from an interrupt.
d7f50089 2207 */
c8fe38ae 2208
bfc09ba0
MD
2209static
2210void
d7f50089
YY
2211pmap_remove_all(vm_page_t m)
2212{
c8fe38ae
MD
2213 struct pmap_inval_info info;
2214 pt_entry_t *pte, tpte;
2215 pv_entry_t pv;
2216
2217 if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
2218 return;
2219
2220 pmap_inval_init(&info);
2221 crit_enter();
2222 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
2223 KKASSERT(pv->pv_pmap->pm_stats.resident_count > 0);
2224 --pv->pv_pmap->pm_stats.resident_count;
2225
2226 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
2227 pmap_inval_add(&info, pv->pv_pmap, pv->pv_va);
2228 tpte = pte_load_clear(pte);
2229
2230 if (tpte & PG_W)
2231 pv->pv_pmap->pm_stats.wired_count--;
2232
2233 if (tpte & PG_A)
2234 vm_page_flag_set(m, PG_REFERENCED);
2235
2236 /*
2237 * Update the vm_page_t clean and reference bits.
2238 */
2239 if (tpte & PG_M) {
2240#if defined(PMAP_DIAGNOSTIC)
48ffc236 2241 if (pmap_nw_modified(tpte)) {
c8fe38ae 2242 kprintf(
48ffc236 2243 "pmap_remove_all: modified page not writable: va: 0x%lx, pte: 0x%lx\n",
c8fe38ae
MD
2244 pv->pv_va, tpte);
2245 }
2246#endif
2247 if (pmap_track_modified(pv->pv_va))
2248 vm_page_dirty(m);
2249 }
2250 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
2251 TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist);
2252 ++pv->pv_pmap->pm_generation;
2253 m->md.pv_list_count--;
48ffc236 2254 KKASSERT(m->md.pv_list_count >= 0);
c8fe38ae
MD
2255 if (TAILQ_EMPTY(&m->md.pv_list))
2256 vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE);
2257 pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem, &info);
2258 free_pv_entry(pv);
2259 }
2260 crit_exit();
2261 KKASSERT((m->flags & (PG_MAPPED|PG_WRITEABLE)) == 0);
2262 pmap_inval_flush(&info);
d7f50089
YY
2263}
2264
2265/*
2266 * pmap_protect:
2267 *
2268 * Set the physical protection on the specified range of this map
2269 * as requested.
2270 *
2271 * This function may not be called from an interrupt if the map is
2272 * not the kernel_pmap.
2273 */
2274void
2275pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
2276{
48ffc236
JG
2277 vm_offset_t va_next;
2278 pml4_entry_t *pml4e;
2279 pdp_entry_t *pdpe;
2280 pd_entry_t ptpaddr, *pde;
2281 pt_entry_t *pte;
c8fe38ae
MD
2282 pmap_inval_info info;
2283
48ffc236
JG
2284 /* JG review for NX */
2285
c8fe38ae
MD
2286 if (pmap == NULL)
2287 return;
2288
2289 if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
2290 pmap_remove(pmap, sva, eva);
2291 return;
2292 }
2293
2294 if (prot & VM_PROT_WRITE)
2295 return;
2296
2297 pmap_inval_init(&info);
2298
48ffc236 2299 for (; sva < eva; sva = va_next) {
c8fe38ae 2300
48ffc236
JG
2301 pml4e = pmap_pml4e(pmap, sva);
2302 if ((*pml4e & PG_V) == 0) {
2303 va_next = (sva + NBPML4) & ~PML4MASK;
2304 if (va_next < sva)
2305 va_next = eva;
2306 continue;
2307 }
c8fe38ae 2308
48ffc236
JG
2309 pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
2310 if ((*pdpe & PG_V) == 0) {
2311 va_next = (sva + NBPDP) & ~PDPMASK;
2312 if (va_next < sva)
2313 va_next = eva;
2314 continue;
2315 }
c8fe38ae 2316
48ffc236
JG
2317 va_next = (sva + NBPDR) & ~PDRMASK;
2318 if (va_next < sva)
2319 va_next = eva;
c8fe38ae 2320
48ffc236
JG
2321 pde = pmap_pdpe_to_pde(pdpe, sva);
2322 ptpaddr = *pde;
c8fe38ae 2323
48ffc236
JG
2324 /*
2325 * Check for large page.
2326 */
2327 if ((ptpaddr & PG_PS) != 0) {
c8fe38ae 2328 pmap_inval_add(&info, pmap, -1);
48ffc236 2329 *pde &= ~(PG_M|PG_RW);
c8fe38ae
MD
2330 pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
2331 continue;
2332 }
2333
2334 /*
2335 * Weed out invalid mappings. Note: we assume that the page
2336 * directory table is always allocated, and in kernel virtual.
2337 */
2338 if (ptpaddr == 0)
2339 continue;
2340
48ffc236
JG
2341 if (va_next > eva)
2342 va_next = eva;
c8fe38ae 2343
48ffc236
JG
2344 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
2345 sva += PAGE_SIZE) {
2346 pt_entry_t obits, pbits;
c8fe38ae
MD
2347 vm_page_t m;
2348
2349 /*
2350 * XXX non-optimal. Note also that there can be
2351 * no pmap_inval_flush() calls until after we modify
2352 * ptbase[sindex] (or otherwise we have to do another
2353 * pmap_inval_add() call).
2354 */
48ffc236
JG
2355 pmap_inval_add(&info, pmap, sva);
2356 obits = pbits = *pte;
2357 if ((pbits & PG_V) == 0)
2358 continue;
c8fe38ae
MD
2359 if (pbits & PG_MANAGED) {
2360 m = NULL;
2361 if (pbits & PG_A) {
48ffc236 2362 m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
c8fe38ae
MD
2363 vm_page_flag_set(m, PG_REFERENCED);
2364 pbits &= ~PG_A;
2365 }
2366 if (pbits & PG_M) {
48ffc236 2367 if (pmap_track_modified(sva)) {
c8fe38ae 2368 if (m == NULL)
3cfe1a9f 2369 m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
c8fe38ae
MD
2370 vm_page_dirty(m);
2371 pbits &= ~PG_M;
2372 }
2373 }
2374 }
2375
2376 pbits &= ~PG_RW;
2377
48ffc236
JG
2378 if (pbits != obits) {
2379 *pte = pbits;
c8fe38ae
MD
2380 }
2381 }
2382 }
2383 pmap_inval_flush(&info);
d7f50089
YY
2384}
2385
2386/*
c8fe38ae
MD
2387 * Insert the given physical page (p) at
2388 * the specified virtual address (v) in the
2389 * target physical map with the protection requested.
d7f50089 2390 *
c8fe38ae
MD
2391 * If specified, the page will be wired down, meaning
2392 * that the related pte can not be reclaimed.
d7f50089 2393 *
c8fe38ae
MD
2394 * NB: This is the only routine which MAY NOT lazy-evaluate
2395 * or lose information. That is, this routine must actually
2396 * insert this page into the given map NOW.
d7f50089
YY
2397 */
2398void
2399pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
2400 boolean_t wired)
2401{
c8fe38ae 2402 vm_paddr_t pa;
48ffc236 2403 pd_entry_t *pde;
c8fe38ae
MD
2404 pt_entry_t *pte;
2405 vm_paddr_t opa;
48ffc236 2406 pt_entry_t origpte, newpte;
c8fe38ae
MD
2407 vm_page_t mpte;
2408 pmap_inval_info info;
2409
2410 if (pmap == NULL)
2411 return;
2412
48ffc236 2413 va = trunc_page(va);
c8fe38ae
MD
2414#ifdef PMAP_DIAGNOSTIC
2415 if (va >= KvaEnd)
2416 panic("pmap_enter: toobig");
2417 if ((va >= UPT_MIN_ADDRESS) && (va < UPT_MAX_ADDRESS))
48ffc236 2418 panic("pmap_enter: invalid to pmap_enter page table pages (va: 0x%lx)", va);
c8fe38ae
MD
2419#endif
2420 if (va < UPT_MAX_ADDRESS && pmap == &kernel_pmap) {
2421 kprintf("Warning: pmap_enter called on UVA with kernel_pmap\n");
48ffc236
JG
2422#ifdef DDB
2423 db_print_backtrace();
2424#endif
c8fe38ae
MD
2425 }
2426 if (va >= UPT_MAX_ADDRESS && pmap != &kernel_pmap) {
2427 kprintf("Warning: pmap_enter called on KVA without kernel_pmap\n");
48ffc236
JG
2428#ifdef DDB
2429 db_print_backtrace();
2430#endif
c8fe38ae
MD
2431 }
2432
2433 /*
2434 * In the case that a page table page is not
2435 * resident, we are creating it here.
2436 */
48ffc236 2437 if (va < VM_MAX_USER_ADDRESS)
c8fe38ae
MD
2438 mpte = pmap_allocpte(pmap, va);
2439 else
2440 mpte = NULL;
2441
2442 pmap_inval_init(&info);
48ffc236
JG
2443 pde = pmap_pde(pmap, va);
2444 if (pde != NULL && (*pde & PG_V) != 0) {
2445 if ((*pde & PG_PS) != 0)
2446 panic("pmap_enter: attempted pmap_enter on 2MB page");
2447 pte = pmap_pde_to_pte(pde, va);
2448 } else
2449 panic("pmap_enter: invalid page directory va=%#lx", va);
2450
2451 KKASSERT(pte != NULL);
2452 pa = VM_PAGE_TO_PHYS(m);
48ffc236 2453 origpte = *pte;
c8fe38ae
MD
2454 opa = origpte & PG_FRAME;
2455
c8fe38ae
MD
2456 /*
2457 * Mapping has not changed, must be protection or wiring change.
2458 */
2459 if (origpte && (opa == pa)) {
2460 /*
2461 * Wiring change, just update stats. We don't worry about
2462 * wiring PT pages as they remain resident as long as there
2463 * are valid mappings in them. Hence, if a user page is wired,
2464 * the PT page will be also.
2465 */
2466 if (wired && ((origpte & PG_W) == 0))
2467 pmap->pm_stats.wired_count++;
2468 else if (!wired && (origpte & PG_W))
2469 pmap->pm_stats.wired_count--;
2470
2471#if defined(PMAP_DIAGNOSTIC)
48ffc236 2472 if (pmap_nw_modified(origpte)) {
c8fe38ae 2473 kprintf(
48ffc236 2474 "pmap_enter: modified page not writable: va: 0x%lx, pte: 0x%lx\n",
c8fe38ae
MD
2475 va, origpte);
2476 }
2477#endif
2478
2479 /*
2480 * Remove the extra pte reference. Note that we cannot
2481 * optimize the RO->RW case because we have adjusted the
2482 * wiring count above and may need to adjust the wiring
2483 * bits below.
2484 */
2485 if (mpte)
2486 mpte->hold_count--;
2487
2488 /*
2489 * We might be turning off write access to the page,
2490 * so we go ahead and sense modify status.
2491 */
2492 if (origpte & PG_MANAGED) {
2493 if ((origpte & PG_M) && pmap_track_modified(va)) {
2494 vm_page_t om;
2495 om = PHYS_TO_VM_PAGE(opa);
2496 vm_page_dirty(om);
2497 }
2498 pa |= PG_MANAGED;
2499 KKASSERT(m->flags & PG_MAPPED);
2500 }
2501 goto validate;
2502 }
2503 /*
2504 * Mapping has changed, invalidate old range and fall through to
2505 * handle validating new mapping.
2506 */
2507 if (opa) {
2508 int err;
2509 err = pmap_remove_pte(pmap, pte, va, &info);
2510 if (err)
48ffc236 2511 panic("pmap_enter: pte vanished, va: 0x%lx", va);
c8fe38ae
MD
2512 }
2513
2514 /*
2515 * Enter on the PV list if part of our managed memory. Note that we
2516 * raise IPL while manipulating pv_table since pmap_enter can be
2517 * called at interrupt time.
2518 */
2519 if (pmap_initialized &&
2520 (m->flags & (PG_FICTITIOUS|PG_UNMANAGED)) == 0) {
2521 pmap_insert_entry(pmap, va, mpte, m);
2522 pa |= PG_MANAGED;
2523 vm_page_flag_set(m, PG_MAPPED);
2524 }
2525
2526 /*
2527 * Increment counters
2528 */
2529 ++pmap->pm_stats.resident_count;
2530 if (wired)
2531 pmap->pm_stats.wired_count++;
2532
2533validate:
2534 /*
2535 * Now validate mapping with desired protection/wiring.
2536 */
48ffc236 2537 newpte = (pt_entry_t) (pa | pte_prot(pmap, prot) | PG_V);
c8fe38ae
MD
2538
2539 if (wired)
2540 newpte |= PG_W;
48ffc236 2541 if (va < VM_MAX_USER_ADDRESS)
c8fe38ae
MD
2542 newpte |= PG_U;
2543 if (pmap == &kernel_pmap)
2544 newpte |= pgeflag;
2545
2546 /*
2547 * if the mapping or permission bits are different, we need
2548 * to update the pte.
2549 */
2550 if ((origpte & ~(PG_M|PG_A)) != newpte) {
2551 pmap_inval_add(&info, pmap, va);
2552 *pte = newpte | PG_A;
2553 if (newpte & PG_RW)
2554 vm_page_flag_set(m, PG_WRITEABLE);
2555 }
2556 KKASSERT((newpte & PG_MANAGED) == 0 || (m->flags & PG_MAPPED));
2557 pmap_inval_flush(&info);
d7f50089
YY
2558}
2559
2560/*
c8fe38ae
MD
2561 * This code works like pmap_enter() but assumes VM_PROT_READ and not-wired.
2562 * This code also assumes that the pmap has no pre-existing entry for this
2563 * VA.
d7f50089 2564 *
c8fe38ae 2565 * This code currently may only be used on user pmaps, not kernel_pmap.
d7f50089 2566 */
bfc09ba0
MD
2567static
2568void
c8fe38ae 2569pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m)
d7f50089 2570{
c8fe38ae
MD
2571 pt_entry_t *pte;
2572 vm_paddr_t pa;
2573 vm_page_t mpte;
2574 vm_pindex_t ptepindex;
48ffc236 2575 pd_entry_t *ptepa;
c8fe38ae
MD
2576 pmap_inval_info info;
2577
2578 pmap_inval_init(&info);
2579
2580 if (va < UPT_MAX_ADDRESS && pmap == &kernel_pmap) {
2581 kprintf("Warning: pmap_enter_quick called on UVA with kernel_pmap\n");
48ffc236
JG
2582#ifdef DDB
2583 db_print_backtrace();
2584#endif
c8fe38ae
MD
2585 }
2586 if (va >= UPT_MAX_ADDRESS && pmap != &kernel_pmap) {
2587 kprintf("Warning: pmap_enter_quick called on KVA without kernel_pmap\n");
48ffc236
JG
2588#ifdef DDB
2589 db_print_backtrace();
2590#endif
c8fe38ae
MD
2591 }
2592
2593 KKASSERT(va < UPT_MIN_ADDRESS); /* assert used on user pmaps only */
2594
2595 /*
2596 * Calculate the page table page (mpte), allocating it if necessary.
2597 *
2598 * A held page table page (mpte), or NULL, is passed onto the
2599 * section following.
2600 */
48ffc236 2601 if (va < VM_MAX_USER_ADDRESS) {
c8fe38ae
MD
2602 /*
2603 * Calculate pagetable page index
2604 */
48ffc236 2605 ptepindex = pmap_pde_pindex(va);
c8fe38ae
MD
2606
2607 do {
2608 /*
2609 * Get the page directory entry
2610 */
48ffc236 2611 ptepa = pmap_pde(pmap, va);
c8fe38ae
MD
2612
2613 /*
2614 * If the page table page is mapped, we just increment
2615 * the hold count, and activate it.
2616 */
48ffc236
JG
2617 if (ptepa && (*ptepa & PG_V) != 0) {
2618 if (*ptepa & PG_PS)
2619 panic("pmap_enter_quick: unexpected mapping into 2MB page");
2620// if (pmap->pm_ptphint &&
2621// (pmap->pm_ptphint->pindex == ptepindex)) {
2622// mpte = pmap->pm_ptphint;
2623// } else {
c8fe38ae
MD
2624 mpte = pmap_page_lookup( pmap->pm_pteobj, ptepindex);
2625 pmap->pm_ptphint = mpte;
48ffc236 2626// }
c8fe38ae
MD
2627 if (mpte)
2628 mpte->hold_count++;
2629 } else {
2630 mpte = _pmap_allocpte(pmap, ptepindex);
2631 }
2632 } while (mpte == NULL);
2633 } else {
2634 mpte = NULL;
2635 /* this code path is not yet used */
2636 }
2637
2638 /*
2639 * With a valid (and held) page directory page, we can just use
2640 * vtopte() to get to the pte. If the pte is already present
2641 * we do not disturb it.
2642 */
2643 pte = vtopte(va);
2644 if (*pte & PG_V) {
2645 if (mpte)
48ffc236 2646 pmap_unwire_pte_hold(pmap, va, mpte, &info);
c8fe38ae
MD
2647 pa = VM_PAGE_TO_PHYS(m);
2648 KKASSERT(((*pte ^ pa) & PG_FRAME) == 0);
2649 return;
2650 }
2651
2652 /*
2653 * Enter on the PV list if part of our managed memory
2654 */
2655 if ((m->flags & (PG_FICTITIOUS|PG_UNMANAGED)) == 0) {
2656 pmap_insert_entry(pmap, va, mpte, m);
2657 vm_page_flag_set(m, PG_MAPPED);
2658 }
2659
2660 /*
2661 * Increment counters
2662 */
2663 ++pmap->pm_stats.resident_count;
2664
2665 pa = VM_PAGE_TO_PHYS(m);
2666
2667 /*
2668 * Now validate mapping with RO protection
2669 */
2670 if (m->flags & (PG_FICTITIOUS|PG_UNMANAGED))
2671 *pte = pa | PG_V | PG_U;
2672 else
2673 *pte = pa | PG_V | PG_U | PG_MANAGED;
2674/* pmap_inval_add(&info, pmap, va); shouldn't be needed inval->valid */
2675 pmap_inval_flush(&info);
d7f50089
YY
2676}
2677
2678/*
c8fe38ae
MD
2679 * Make a temporary mapping for a physical address. This is only intended
2680 * to be used for panic dumps.
d7f50089 2681 */
48ffc236 2682/* JG Needed on amd64? */
c8fe38ae
MD
2683void *
2684pmap_kenter_temporary(vm_paddr_t pa, int i)
d7f50089 2685{
c8fe38ae
MD
2686 pmap_kenter((vm_offset_t)crashdumpmap + (i * PAGE_SIZE), pa);
2687 return ((void *)crashdumpmap);
d7f50089
YY
2688}
2689
c8fe38ae
MD
2690#define MAX_INIT_PT (96)
2691
d7f50089
YY
2692/*
2693 * This routine preloads the ptes for a given object into the specified pmap.
2694 * This eliminates the blast of soft faults on process startup and
2695 * immediately after an mmap.
2696 */
2697static int pmap_object_init_pt_callback(vm_page_t p, void *data);
2698
2699void
2700pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_prot_t prot,
2701 vm_object_t object, vm_pindex_t pindex,
2702 vm_size_t size, int limit)
2703{
c8fe38ae
MD
2704 struct rb_vm_page_scan_info info;
2705 struct lwp *lp;
48ffc236 2706 vm_size_t psize;
c8fe38ae
MD
2707
2708 /*
2709 * We can't preinit if read access isn't set or there is no pmap
2710 * or object.
2711 */
2712 if ((prot & VM_PROT_READ) == 0 || pmap == NULL || object == NULL)
2713 return;
2714
2715 /*
2716 * We can't preinit if the pmap is not the current pmap
2717 */
2718 lp = curthread->td_lwp;
2719 if (lp == NULL || pmap != vmspace_pmap(lp->lwp_vmspace))
2720 return;
2721
2722 psize = amd64_btop(size);
2723
2724 if ((object->type != OBJT_VNODE) ||
2725 ((limit & MAP_PREFAULT_PARTIAL) && (psize > MAX_INIT_PT) &&
2726 (object->resident_page_count > MAX_INIT_PT))) {
2727 return;
2728 }
2729
2730 if (psize + pindex > object->size) {
2731 if (object->size < pindex)
2732 return;
2733 psize = object->size - pindex;
2734 }
2735
2736 if (psize == 0)
2737 return;
2738
2739 /*
2740 * Use a red-black scan to traverse the requested range and load
2741 * any valid pages found into the pmap.
2742 *
2743 * We cannot safely scan the object's memq unless we are in a
2744 * critical section since interrupts can remove pages from objects.
2745 */
2746 info.start_pindex = pindex;
2747 info.end_pindex = pindex + psize - 1;
2748 info.limit = limit;
2749 info.mpte = NULL;
2750 info.addr = addr;
2751 info.pmap = pmap;
2752
2753 crit_enter();
2754 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp,
2755 pmap_object_init_pt_callback, &info);
2756 crit_exit();
d7f50089
YY
2757}
2758
2759static
2760int
2761pmap_object_init_pt_callback(vm_page_t p, void *data)
2762{
c8fe38ae
MD
2763 struct rb_vm_page_scan_info *info = data;
2764 vm_pindex_t rel_index;
2765 /*
2766 * don't allow an madvise to blow away our really
2767 * free pages allocating pv entries.
2768 */
2769 if ((info->limit & MAP_PREFAULT_MADVISE) &&
2770 vmstats.v_free_count < vmstats.v_free_reserved) {
2771 return(-1);
2772 }
2773 if (((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) &&
2774 (p->busy == 0) && (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) {
2775 if ((p->queue - p->pc) == PQ_CACHE)
2776 vm_page_deactivate(p);
2777 vm_page_busy(p);
2778 rel_index = p->pindex - info->start_pindex;
2779 pmap_enter_quick(info->pmap,
2780 info->addr + amd64_ptob(rel_index), p);
2781 vm_page_wakeup(p);
2782 }
d7f50089
YY
2783 return(0);
2784}
2785
2786/*
2787 * pmap_prefault provides a quick way of clustering pagefaults into a
2788 * processes address space. It is a "cousin" of pmap_object_init_pt,
2789 * except it runs at page fault time instead of mmap time.
2790 */
2791#define PFBAK 4
2792#define PFFOR 4
2793#define PAGEORDER_SIZE (PFBAK+PFFOR)
2794
2795static int pmap_prefault_pageorder[] = {
2796 -PAGE_SIZE, PAGE_SIZE,
2797 -2 * PAGE_SIZE, 2 * PAGE_SIZE,
2798 -3 * PAGE_SIZE, 3 * PAGE_SIZE,
2799 -4 * PAGE_SIZE, 4 * PAGE_SIZE
2800};
2801
2802void
2803pmap_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry)
2804{
c8fe38ae
MD
2805 int i;
2806 vm_offset_t starta;
2807 vm_offset_t addr;
2808 vm_pindex_t pindex;
2809 vm_page_t m;
2810 vm_object_t object;
2811 struct lwp *lp;
2812
2813 /*
2814 * We do not currently prefault mappings that use virtual page
2815 * tables. We do not prefault foreign pmaps.
2816 */
2817 if (entry->maptype == VM_MAPTYPE_VPAGETABLE)
2818 return;
2819 lp = curthread->td_lwp;
2820 if (lp == NULL || (pmap != vmspace_pmap(lp->lwp_vmspace)))
2821 return;
2822
2823 object = entry->object.vm_object;
2824
2825 starta = addra - PFBAK * PAGE_SIZE;
2826 if (starta < entry->start)
2827 starta = entry->start;
2828 else if (starta > addra)
2829 starta = 0;
2830
2831 /*
2832 * critical section protection is required to maintain the
2833 * page/object association, interrupts can free pages and remove
2834 * them from their objects.
2835 */
2836 crit_enter();
2837 for (i = 0; i < PAGEORDER_SIZE; i++) {
2838 vm_object_t lobject;
2839 pt_entry_t *pte;
665d7d25 2840 pd_entry_t *pde;
c8fe38ae
MD
2841
2842 addr = addra + pmap_prefault_pageorder[i];
2843 if (addr > addra + (PFFOR * PAGE_SIZE))
2844 addr = 0;
2845
2846 if (addr < starta || addr >= entry->end)
2847 continue;
2848
665d7d25
MD
2849 pde = pmap_pde(pmap, addr);
2850 if (pde == NULL || *pde == 0)
c8fe38ae
MD
2851 continue;
2852
2853 pte = vtopte(addr);
2854 if (*pte)
2855 continue;
2856
2857 pindex = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT;
2858 lobject = object;
2859
2860 for (m = vm_page_lookup(lobject, pindex);
2861 (!m && (lobject->type == OBJT_DEFAULT) &&
2862 (lobject->backing_object));
2863 lobject = lobject->backing_object
2864 ) {
2865 if (lobject->backing_object_offset & PAGE_MASK)
2866 break;
2867 pindex += (lobject->backing_object_offset >> PAGE_SHIFT);
2868 m = vm_page_lookup(lobject->backing_object, pindex);
2869 }
2870
2871 /*
2872 * give-up when a page is not in memory
2873 */
2874 if (m == NULL)
2875 break;
2876
2877 if (((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) &&
2878 (m->busy == 0) &&
2879 (m->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) {
2880
2881 if ((m->queue - m->pc) == PQ_CACHE) {
2882 vm_page_deactivate(m);
2883 }
2884 vm_page_busy(m);
2885 pmap_enter_quick(pmap, addr, m);
2886 vm_page_wakeup(m);
2887 }
2888 }
2889 crit_exit();
d7f50089
YY
2890}
2891
2892/*
2893 * Routine: pmap_change_wiring
2894 * Function: Change the wiring attribute for a map/virtual-address
2895 * pair.
2896 * In/out conditions:
2897 * The mapping must already exist in the pmap.
2898 */
2899void
2900pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired)
2901{
c8fe38ae
MD
2902 pt_entry_t *pte;
2903
2904 if (pmap == NULL)
2905 return;
2906
2907 pte = pmap_pte(pmap, va);
2908
2909 if (wired && !pmap_pte_w(pte))
2910 pmap->pm_stats.wired_count++;
2911 else if (!wired && pmap_pte_w(pte))
2912 pmap->pm_stats.wired_count--;
2913
2914 /*
2915 * Wiring is not a hardware characteristic so there is no need to
2916 * invalidate TLB. However, in an SMP environment we must use
2917 * a locked bus cycle to update the pte (if we are not using
2918 * the pmap_inval_*() API that is)... it's ok to do this for simple
2919 * wiring changes.
2920 */
2921#ifdef SMP
2922 if (wired)
71577ce5 2923 atomic_set_long(pte, PG_W);
c8fe38ae 2924 else
71577ce5 2925 atomic_clear_long(pte, PG_W);
c8fe38ae
MD
2926#else
2927 if (wired)
71577ce5 2928 atomic_set_long_nonlocked(pte, PG_W);
c8fe38ae 2929 else
71577ce5 2930 atomic_clear_long_nonlocked(pte, PG_W);
c8fe38ae 2931#endif
d7f50089
YY
2932}
2933
c8fe38ae
MD
2934
2935
d7f50089
YY
2936/*
2937 * Copy the range specified by src_addr/len
2938 * from the source map to the range dst_addr/len
2939 * in the destination map.
2940 *
2941 * This routine is only advisory and need not do anything.
2942 */
2943void
2944pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
bfc09ba0 2945 vm_size_t len, vm_offset_t src_addr)
d7f50089 2946{
bfc09ba0
MD
2947 return;
2948#if 0
c8fe38ae
MD
2949 pmap_inval_info info;
2950 vm_offset_t addr;
2951 vm_offset_t end_addr = src_addr + len;
2952 vm_offset_t pdnxt;
2953 pd_entry_t src_frame, dst_frame;
2954 vm_page_t m;
2955
2956 if (dst_addr != src_addr)
2957 return;
48ffc236 2958#if JGPMAP32
c8fe38ae
MD
2959 src_frame = src_pmap->pm_pdir[PTDPTDI] & PG_FRAME;
2960 if (src_frame != (PTDpde & PG_FRAME)) {
2961 return;
2962 }
2963
2964 dst_frame = dst_pmap->pm_pdir[PTDPTDI] & PG_FRAME;
2965 if (dst_frame != (APTDpde & PG_FRAME)) {
2966 APTDpde = (pd_entry_t) (dst_frame | PG_RW | PG_V);
2967 /* The page directory is not shared between CPUs */
2968 cpu_invltlb();
2969 }
48ffc236 2970#endif
c8fe38ae
MD
2971 pmap_inval_init(&info);
2972 pmap_inval_add(&info, dst_pmap, -1);
2973 pmap_inval_add(&info, src_pmap, -1);
2974
2975 /*
2976 * critical section protection is required to maintain the page/object
2977 * association, interrupts can free pages and remove them from
2978 * their objects.
2979 */
2980 crit_enter();
2981 for (addr = src_addr; addr < end_addr; addr = pdnxt) {
2982 pt_entry_t *src_pte, *dst_pte;
2983 vm_page_t dstmpte, srcmpte;
2984 vm_offset_t srcptepaddr;
2985 vm_pindex_t ptepindex;
2986
2987 if (addr >= UPT_MIN_ADDRESS)
2988 panic("pmap_copy: invalid to pmap_copy page tables\n");
2989
2990 /*
2991 * Don't let optional prefaulting of pages make us go
2992 * way below the low water mark of free pages or way
2993 * above high water mark of used pv entries.
2994 */
2995 if (vmstats.v_free_count < vmstats.v_free_reserved ||
2996 pv_entry_count > pv_entry_high_water)
2997 break;
2998
2999 pdnxt = ((addr + PAGE_SIZE*NPTEPG) & ~(PAGE_SIZE*NPTEPG - 1));
3000 ptepindex = addr >> PDRSHIFT;
3001
48ffc236 3002#if JGPMAP32
c8fe38ae 3003 srcptepaddr = (vm_offset_t) src_pmap->pm_pdir[ptepindex];
48ffc236 3004#endif
c8fe38ae
MD
3005 if (srcptepaddr == 0)
3006 continue;
3007
3008 if (srcptepaddr & PG_PS) {
48ffc236 3009#if JGPMAP32
c8fe38ae
MD
3010 if (dst_pmap->pm_pdir[ptepindex] == 0) {
3011 dst_pmap->pm_pdir[ptepindex] = (pd_entry_t) srcptepaddr;
3012 dst_pmap->pm_stats.resident_count += NBPDR / PAGE_SIZE;
3013 }
48ffc236 3014#endif
c8fe38ae
MD
3015 continue;
3016 }
3017
3018 srcmpte = vm_page_lookup(src_pmap->pm_pteobj, ptepindex);
3019 if ((srcmpte == NULL) || (srcmpte->hold_count == 0) ||
3020 (srcmpte->flags & PG_BUSY)) {
3021 continue;
3022 }
3023
3024 if (pdnxt > end_addr)
3025 pdnxt = end_addr;
3026
3027 src_pte = vtopte(addr);
48ffc236 3028#if JGPMAP32
c8fe38ae 3029 dst_pte = avtopte(addr);
48ffc236 3030#endif
c8fe38ae
MD
3031 while (addr < pdnxt) {
3032 pt_entry_t ptetemp;
3033
3034 ptetemp = *src_pte;
3035 /*
3036 * we only virtual copy managed pages
3037 */
3038 if ((ptetemp & PG_MANAGED) != 0) {
3039 /*
3040 * We have to check after allocpte for the
3041 * pte still being around... allocpte can
3042 * block.
3043 *
3044 * pmap_allocpte() can block. If we lose
3045 * our page directory mappings we stop.
3046 */
3047 dstmpte = pmap_allocpte(dst_pmap, addr);
3048
48ffc236 3049#if JGPMAP32
c8fe38ae
MD
3050 if (src_frame != (PTDpde & PG_FRAME) ||
3051 dst_frame != (APTDpde & PG_FRAME)
3052 ) {
3053 kprintf("WARNING: pmap_copy: detected and corrected race\n");
3054 pmap_unwire_pte_hold(dst_pmap, dstmpte, &info);
3055 goto failed;
3056 } else if ((*dst_pte == 0) &&
3057 (ptetemp = *src_pte) != 0 &&
3058 (ptetemp & PG_MANAGED)) {
3059 /*
3060 * Clear the modified and
3061 * accessed (referenced) bits
3062 * during the copy.
3063 */
3064 m = PHYS_TO_VM_PAGE(ptetemp);
3065 *dst_pte = ptetemp & ~(PG_M | PG_A);
3066 ++dst_pmap->pm_stats.resident_count;
3067 pmap_insert_entry(dst_pmap, addr,
3068 dstmpte, m);
3069 KKASSERT(m->flags & PG_MAPPED);
3070 } else {
3071 kprintf("WARNING: pmap_copy: dst_pte race detected and corrected\n");
3072 pmap_unwire_pte_hold(dst_pmap, dstmpte, &info);
3073 goto failed;
3074 }
48ffc236 3075#endif
c8fe38ae
MD
3076 if (dstmpte->hold_count >= srcmpte->hold_count)
3077 break;
3078 }
3079 addr += PAGE_SIZE;
3080 src_pte++;
3081 dst_pte++;
3082 }
3083 }
3084failed:
3085 crit_exit();
3086 pmap_inval_flush(&info);
f81851b8 3087#endif
d7f50089
YY
3088}
3089
3090/*
3091 * pmap_zero_page:
3092 *
48ffc236 3093 * Zero the specified physical page.
d7f50089
YY
3094 *
3095 * This function may be called from an interrupt and no locking is
3096 * required.
3097 */
3098void
3099pmap_zero_page(vm_paddr_t phys)
3100{
48ffc236 3101 vm_offset_t va = PHYS_TO_DMAP(phys);
c8fe38ae 3102
48ffc236 3103 pagezero((void *)va);
d7f50089
YY
3104}
3105
3106/*
3107 * pmap_page_assertzero:
3108 *
3109 * Assert that a page is empty, panic if it isn't.
3110 */
3111void
3112pmap_page_assertzero(vm_paddr_t phys)
3113{
48ffc236 3114 vm_offset_t virt = PHYS_TO_DMAP(phys);
bfc09ba0 3115 int i;
48ffc236 3116
bfc09ba0
MD
3117 for (i = 0; i < PAGE_SIZE; i += sizeof(long)) {
3118 if (*(long *)((char *)virt + i) != 0) {
3119 panic("pmap_page_assertzero() @ %p not zero!\n", (void *)virt);
c8fe38ae
MD
3120 }
3121 }
d7f50089
YY
3122}
3123
3124/*
3125 * pmap_zero_page:
3126 *
3127 * Zero part of a physical page by mapping it into memory and clearing
3128 * its contents with bzero.
3129 *
3130 * off and size may not cover an area beyond a single hardware page.
3131 */
3132void
3133pmap_zero_page_area(vm_paddr_t phys, int off, int size)
3134{
48ffc236 3135 vm_offset_t virt = PHYS_TO_DMAP(phys);
bfc09ba0 3136
48ffc236 3137 bzero((char *)virt + off, size);
d7f50089
YY
3138}
3139
3140/*
3141 * pmap_copy_page:
3142 *
3143 * Copy the physical page from the source PA to the target PA.
3144 * This function may be called from an interrupt. No locking
3145 * is required.
3146 */
3147void
3148pmap_copy_page(vm_paddr_t src, vm_paddr_t dst)
3149{
48ffc236 3150 vm_offset_t src_virt, dst_virt;
c8fe38ae 3151
48ffc236
JG
3152 src_virt = PHYS_TO_DMAP(src);
3153 dst_virt = PHYS_TO_DMAP(dst);
bfc09ba0 3154 bcopy((void *)src_virt, (void *)dst_virt, PAGE_SIZE);
d7f50089
YY
3155}
3156
3157/*
3158 * pmap_copy_page_frag:
3159 *
3160 * Copy the physical page from the source PA to the target PA.
3161 * This function may be called from an interrupt. No locking
3162 * is required.
3163 */
3164void
3165pmap_copy_page_frag(vm_paddr_t src, vm_paddr_t dst, size_t bytes)
3166{
48ffc236 3167 vm_offset_t src_virt, dst_virt;
c8fe38ae 3168
48ffc236
JG
3169 src_virt = PHYS_TO_DMAP(src);
3170 dst_virt = PHYS_TO_DMAP(dst);
bfc09ba0 3171
48ffc236
JG
3172 bcopy((char *)src_virt + (src & PAGE_MASK),
3173 (char *)dst_virt + (dst & PAGE_MASK),
c8fe38ae 3174 bytes);
d7f50089
YY
3175}
3176
3177/*
3178 * Returns true if the pmap's pv is one of the first
3179 * 16 pvs linked to from this page. This count may
3180 * be changed upwards or downwards in the future; it
3181 * is only necessary that true be returned for a small
3182 * subset of pmaps for proper page aging.
3183 */
3184boolean_t
3185pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
3186{
c8fe38ae
MD
3187 pv_entry_t pv;
3188 int loops = 0;
3189
3190 if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
3191 return FALSE;
3192
3193 crit_enter();
3194
3195 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
3196 if (pv->pv_pmap == pmap) {
3197 crit_exit();
3198 return TRUE;
3199 }
3200 loops++;
3201 if (loops >= 16)
3202 break;
3203 }
3204 crit_exit();
d7f50089
YY
3205 return (FALSE);
3206}
3207
3208/*
3209 * Remove all pages from specified address space
3210 * this aids process exit speeds. Also, this code
3211 * is special cased for current process only, but
3212 * can have the more generic (and slightly slower)
3213 * mode enabled. This is much faster than pmap_remove
3214 * in the case of running down an entire address space.
3215 */
3216void
3217pmap_remove_pages(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
3218{
c8fe38ae
MD
3219 struct lwp *lp;
3220 pt_entry_t *pte, tpte;
3221 pv_entry_t pv, npv;
3222 vm_page_t m;
3223 pmap_inval_info info;
3224 int iscurrentpmap;
48ffc236 3225 int save_generation;
c8fe38ae
MD
3226
3227 lp = curthread->td_lwp;
3228 if (lp && pmap == vmspace_pmap(lp->lwp_vmspace))
3229 iscurrentpmap = 1;
3230 else
3231 iscurrentpmap = 0;
3232
3233 pmap_inval_init(&info);
3234 crit_enter();
3235 for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) {
3236 if (pv->pv_va >= eva || pv->pv_va < sva) {
3237 npv = TAILQ_NEXT(pv, pv_plist);
3238 continue;
3239 }
3240
3241 KKASSERT(pmap == pv->pv_pmap);
3242
3243 if (iscurrentpmap)
3244 pte = vtopte(pv->pv_va);
3245 else
3246 pte = pmap_pte_quick(pmap, pv->pv_va);
3247 if (pmap->pm_active)
3248 pmap_inval_add(&info, pmap, pv->pv_va);
3249
3250 /*
3251 * We cannot remove wired pages from a process' mapping
3252 * at this time
3253 */
3254 if (*pte & PG_W) {
3255 npv = TAILQ_NEXT(pv, pv_plist);
3256 continue;
3257 }
3258 tpte = pte_load_clear(pte);
3259
48ffc236 3260 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
c8fe38ae
MD
3261
3262 KASSERT(m < &vm_page_array[vm_page_array_size],
48ffc236 3263 ("pmap_remove_pages: bad tpte %lx", tpte));
c8fe38ae
MD
3264
3265 KKASSERT(pmap->pm_stats.resident_count > 0);
3266 --pmap->pm_stats.resident_count;
3267
3268 /*
3269 * Update the vm_page_t clean and reference bits.
3270 */
3271 if (tpte & PG_M) {
3272 vm_page_dirty(m);
3273 }
3274
3275 npv = TAILQ_NEXT(pv, pv_plist);
3276 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
3277 save_generation = ++pmap->pm_generation;
3278
3279 m->md.pv_list_count--;
3280 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
3281 if (TAILQ_EMPTY(&m->md.pv_list))
3282 vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE);
3283
3284 pmap_unuse_pt(pmap, pv->pv_va, pv->pv_ptem, &info);
3285 free_pv_entry(pv);
3286
3287 /*
3288 * Restart the scan if we blocked during the unuse or free
3289 * calls and other removals were made.
3290 */
3291 if (save_generation != pmap->pm_generation) {
3292 kprintf("Warning: pmap_remove_pages race-A avoided\n");
3293 pv = TAILQ_FIRST(&pmap->pm_pvlist);
3294 }
3295 }
3296 pmap_inval_flush(&info);
3297 crit_exit();
d7f50089
YY
3298}
3299
3300/*
c8fe38ae
MD
3301 * pmap_testbit tests bits in pte's
3302 * note that the testbit/clearbit routines are inline,
3303 * and a lot of things compile-time evaluate.
d7f50089 3304 */
bfc09ba0
MD
3305static
3306boolean_t
d7f50089
YY
3307pmap_testbit(vm_page_t m, int bit)
3308{
c8fe38ae
MD
3309 pv_entry_t pv;
3310 pt_entry_t *pte;
3311
3312 if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
3313 return FALSE;
3314
3315 if (TAILQ_FIRST(&m->md.pv_list) == NULL)
3316 return FALSE;
3317
3318 crit_enter();
3319
3320 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
3321 /*
3322 * if the bit being tested is the modified bit, then
3323 * mark clean_map and ptes as never
3324 * modified.
3325 */
3326 if (bit & (PG_A|PG_M)) {
3327 if (!pmap_track_modified(pv->pv_va))
3328 continue;
3329 }
3330
3331#if defined(PMAP_DIAGNOSTIC)
48ffc236
JG
3332 if (pv->pv_pmap == NULL) {
3333 kprintf("Null pmap (tb) at va: 0x%lx\n", pv->pv_va);
c8fe38ae
MD
3334 continue;
3335 }
3336#endif
3337 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
3338 if (*pte & bit) {
3339 crit_exit();
3340 return TRUE;
3341 }
3342 }
3343 crit_exit();
d7f50089
YY
3344 return (FALSE);
3345}
3346
3347/*
c8fe38ae 3348 * this routine is used to modify bits in ptes
d7f50089 3349 */
bfc09ba0
MD
3350static __inline
3351void
d7f50089
YY
3352pmap_clearbit(vm_page_t m, int bit)
3353{
c8fe38ae
MD
3354 struct pmap_inval_info info;
3355 pv_entry_t pv;
3356 pt_entry_t *pte;
3357 pt_entry_t pbits;
3358
3359 if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
3360 return;
3361
3362 pmap_inval_init(&info);
3363 crit_enter();
3364
3365 /*
3366 * Loop over all current mappings setting/clearing as appropos If
3367 * setting RO do we need to clear the VAC?
3368 */
3369 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
3370 /*
3371 * don't write protect pager mappings
3372 */
3373 if (bit == PG_RW) {
3374 if (!pmap_track_modified(pv->pv_va))
3375 continue;
3376 }
3377
3378#if defined(PMAP_DIAGNOSTIC)
48ffc236
JG
3379 if (pv->pv_pmap == NULL) {
3380 kprintf("Null pmap (cb) at va: 0x%lx\n", pv->pv_va);
c8fe38ae
MD
3381 continue;
3382 }
3383#endif
3384
3385 /*
3386 * Careful here. We can use a locked bus instruction to
3387 * clear PG_A or PG_M safely but we need to synchronize
3388 * with the target cpus when we mess with PG_RW.
3389 *
3390 * We do not have to force synchronization when clearing
3391 * PG_M even for PTEs generated via virtual memory maps,
3392 * because the virtual kernel will invalidate the pmap
3393 * entry when/if it needs to resynchronize the Modify bit.
3394 */
3395 if (bit & PG_RW)
3396 pmap_inval_add(&info, pv->pv_pmap, pv->pv_va);
3397 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
3398again:
3399 pbits = *pte;
3400 if (pbits & bit) {
3401 if (bit == PG_RW) {
3402 if (pbits & PG_M) {
3403 vm_page_dirty(m);
48ffc236 3404 atomic_clear_long(pte, PG_M|PG_RW);
c8fe38ae
MD
3405 } else {
3406 /*
3407 * The cpu may be trying to set PG_M
3408 * simultaniously with our clearing
3409 * of PG_RW.
3410 */
48ffc236 3411 if (!atomic_cmpset_long(pte, pbits,
c8fe38ae
MD
3412 pbits & ~PG_RW))
3413 goto again;
3414 }
3415 } else if (bit == PG_M) {
3416 /*
3417 * We could also clear PG_RW here to force
3418 * a fault on write to redetect PG_M for
3419 * virtual kernels, but it isn't necessary
3420 * since virtual kernels invalidate the pte
3421 * when they clear the VPTE_M bit in their
3422 * virtual page tables.
3423 */
48ffc236 3424 atomic_clear_long(pte, PG_M);
c8fe38ae 3425 } else {
48ffc236 3426 atomic_clear_long(pte, bit);
c8fe38ae
MD
3427 }
3428 }
3429 }
3430 pmap_inval_flush(&info);
3431 crit_exit();
d7f50089
YY
3432}
3433
3434/*
3435 * pmap_page_protect:
3436 *
3437 * Lower the permission for all mappings to a given page.
3438 */
3439void
3440pmap_page_protect(vm_page_t m, vm_prot_t prot)
3441{
48ffc236 3442 /* JG NX support? */
c8fe38ae
MD
3443 if ((prot & VM_PROT_WRITE) == 0) {
3444 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) {
3445 pmap_clearbit(m, PG_RW);
3446 vm_page_flag_clear(m, PG_WRITEABLE);
3447 } else {
3448 pmap_remove_all(m);
3449 }
3450 }
d7f50089
YY
3451}
3452
3453vm_paddr_t
c8fe38ae 3454pmap_phys_address(vm_pindex_t ppn)
d7f50089 3455{
c8fe38ae 3456 return (amd64_ptob(ppn));
d7f50089
YY
3457}
3458
3459/*
3460 * pmap_ts_referenced:
3461 *
3462 * Return a count of reference bits for a page, clearing those bits.
3463 * It is not necessary for every reference bit to be cleared, but it
3464 * is necessary that 0 only be returned when there are truly no
3465 * reference bits set.
3466 *
3467 * XXX: The exact number of bits to check and clear is a matter that
3468 * should be tested and standardized at some point in the future for
3469 * optimal aging of shared pages.
3470 */
3471int
3472pmap_ts_referenced(vm_page_t m)
3473{
c8fe38ae
MD
3474 pv_entry_t pv, pvf, pvn;
3475 pt_entry_t *pte;
3476 int rtval = 0;
3477
3478 if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
3479 return (rtval);
3480
3481 crit_enter();
3482
3483 if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
3484
3485 pvf = pv;
3486
3487 do {
3488 pvn = TAILQ_NEXT(pv, pv_list);
3489
3490 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
3491
3492 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
3493
3494 if (!pmap_track_modified(pv->pv_va))
3495 continue;
3496
3497 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
3498
3499 if (pte && (*pte & PG_A)) {
3500#ifdef SMP
48ffc236 3501 atomic_clear_long(pte, PG_A);
c8fe38ae 3502#else
48ffc236 3503 atomic_clear_long_nonlocked(pte, PG_A);
c8fe38ae
MD
3504#endif
3505 rtval++;
3506 if (rtval > 4) {
3507 break;
3508 }
3509 }
3510 } while ((pv = pvn) != NULL && pv != pvf);
3511 }
3512 crit_exit();
3513
3514 return (rtval);
d7f50089
YY
3515}
3516
3517/*
3518 * pmap_is_modified:
3519 *
3520 * Return whether or not the specified physical page was modified
3521 * in any physical maps.
3522 */
3523boolean_t
3524pmap_is_modified(vm_page_t m)
3525{
c8fe38ae 3526 return pmap_testbit(m, PG_M);
d7f50089
YY
3527}
3528
3529/*
3530 * Clear the modify bits on the specified physical page.
3531 */
3532void
3533pmap_clear_modify(vm_page_t m)
3534{
c8fe38ae 3535 pmap_clearbit(m, PG_M);
d7f50089
YY
3536}
3537
3538/*
3539 * pmap_clear_reference:
3540 *
3541 * Clear the reference bit on the specified physical page.
3542 */
3543void
3544pmap_clear_reference(vm_page_t m)
3545{
c8fe38ae 3546 pmap_clearbit(m, PG_A);
d7f50089
YY
3547}
3548
d7f50089
YY
3549/*
3550 * Miscellaneous support routines follow
3551 */
3552
bfc09ba0
MD
3553static
3554void
d7f50089
YY
3555i386_protection_init(void)
3556{
3557 int *kp, prot;
3558
48ffc236 3559 /* JG NX support may go here; No VM_PROT_EXECUTE ==> set NX bit */
d7f50089
YY
3560 kp = protection_codes;
3561 for (prot = 0; prot < 8; prot++) {
c8fe38ae
MD
3562 switch (prot) {
3563 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE:
3564 /*
3565 * Read access is also 0. There isn't any execute bit,
3566 * so just make it readable.
3567 */
3568 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_NONE:
3569 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_EXECUTE:
3570 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_EXECUTE:
3571 *kp++ = 0;
3572 break;
3573 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE:
3574 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_EXECUTE:
3575 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_NONE:
3576 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE:
3577 *kp++ = PG_RW;
3578 break;
3579 }
d7f50089
YY
3580 }
3581}
3582
3583/*
3584 * Map a set of physical memory pages into the kernel virtual
3585 * address space. Return a pointer to where it is mapped. This
3586 * routine is intended to be used for mapping device memory,
3587 * NOT real memory.
3588 *
3589 * NOTE: we can't use pgeflag unless we invalidate the pages one at
3590 * a time.
3591 */
3592void *
3593pmap_mapdev(vm_paddr_t pa, vm_size_t size)
3594{
3595 vm_offset_t va, tmpva, offset;
c8fe38ae 3596 pt_entry_t *pte;
d7f50089
YY
3597
3598 offset = pa & PAGE_MASK;
3599 size = roundup(offset + size, PAGE_SIZE);
3600
3601 va = kmem_alloc_nofault(&kernel_map, size);
48ffc236 3602 if (va == 0)
d7f50089
YY