amd64: first steps towards 64-bit pmap
authorJordan Gordeev <jgordeev@dir.bg>
Sun, 26 Apr 2009 18:58:06 +0000 (21:58 +0300)
committerJordan Gordeev <jgordeev@dir.bg>
Sun, 26 Apr 2009 18:58:06 +0000 (21:58 +0300)
remove 32-bit amd64 pmap
replace with work-in-progress 64-bit pmap

12 files changed:
sys/platform/pc64/amd64/exception.S
sys/platform/pc64/amd64/genassym.c
sys/platform/pc64/amd64/global.s
sys/platform/pc64/amd64/locore.s
sys/platform/pc64/amd64/machdep.c
sys/platform/pc64/amd64/pmap.c
sys/platform/pc64/amd64/support.s
sys/platform/pc64/amd64/swtch.s
sys/platform/pc64/amd64/vm_machdep.c
sys/platform/pc64/conf/kern.mk
sys/platform/pc64/include/pmap.h
sys/platform/pc64/include/vmparam.h

index e469fec..f1a262b 100644 (file)
@@ -376,7 +376,7 @@ IDTVEC(nmi)
        jnz     nmi_needswapgs          /* we came from userland */
        movl    $MSR_GSBASE,%ecx
        rdmsr
-       cmpl    $VM_MAXUSER_ADDRESS >> 32,%edx
+       cmpl    $VM_MAX_USER_ADDRESS >> 32,%edx
        jae     nmi_calltrap            /* GS.base holds a kernel VA */
 nmi_needswapgs:
        incl    %ebx
index 64cb85e..d4c073c 100644 (file)
@@ -101,7 +101,7 @@ ASSYM(V_FORWARDED_MISSES, offsetof(struct vmmeter, v_forwarded_misses));
 ASSYM(MAXCOMLEN, MAXCOMLEN);
 ASSYM(EFAULT, EFAULT);
 ASSYM(ENAMETOOLONG, ENAMETOOLONG);
-ASSYM(VM_MAXUSER_ADDRESS, VM_MAXUSER_ADDRESS);
+ASSYM(VM_MAX_USER_ADDRESS, VM_MAX_USER_ADDRESS);
 
 ASSYM(GD_CURTHREAD, offsetof(struct mdglobaldata, mi.gd_curthread));
 ASSYM(GD_CNT, offsetof(struct mdglobaldata, mi.gd_cnt));
index a6d594e..27327b7 100644 (file)
         * segment.
         */
        .data
+#if JG
        .globl  CPU_prvspace, lapic
        .set    CPU_prvspace,(MPPTDI << PDRSHIFT)
+       /* JG TODO check for correctness */
        .set    lapic,CPU_prvspace + (NPTEPG-1) * PAGE_SIZE
+#endif
 
        .globl  globaldata
        .set    globaldata,0
index d38027c..81237ab 100644 (file)
@@ -1,6 +1,5 @@
 /*-
  * Copyright (c) 2003 Peter Wemm <peter@FreeBSD.org>
- * Copyright (c) 2008 The DragonFly Project.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
 #include "assym.s"
 
 /*
- * PTmap is recursive pagemap at top of virtual address space.
- * Within PTmap, the page directory can be found (third indirection).
- */
-       .globl  PTmap,PTD,PTDpde
-       .set    PTmap,(PTDPTDI << PDRSHIFT)
-       .set    PTD,PTmap + (PTDPTDI * PAGE_SIZE)
-       .set    PTDpde,PTD + (PTDPTDI * PDESIZE)
-
-/*
- * APTmap, APTD is the alternate recursive pagemap.
- * It's used when modifying another process's page tables.
- */
-       .globl  APTmap,APTD,APTDpde
-       .set    APTmap,APTDPTDI << PDRSHIFT
-       .set    APTD,APTmap + (APTDPTDI * PAGE_SIZE)
-       .set    APTDpde,PTD + (APTDPTDI * PDESIZE)
-
-/*
  * Compiled KERNBASE location
  */
        .globl  kernbase,loc_PTmap,loc_PDmap,loc_PDPmap,loc_PML4map,loc_PML4pml4e,dmapbase,dmapend
@@ -67,7 +48,6 @@
        .set    dmapend,DMAP_MAX_ADDRESS
 
        .text
-
 /**********************************************************************
  *
  * This is where the loader trampoline start us, set the ball rolling...
index b3fa41a..d8eb2fc 100644 (file)
@@ -152,15 +152,14 @@ SYSINIT(cpu, SI_BOOT2_SMP, SI_ORDER_FIRST, cpu_startup, NULL)
 extern vm_offset_t ksym_start, ksym_end;
 #endif
 
-uint64_t common_lvl4_phys;
-uint64_t common_lvl3_phys;
-uint64_t IdlePTD;
 uint64_t KPTphys;
 uint64_t SMPptpa;
 pt_entry_t *SMPpt;
-pdp_entry_t *link_pdpe;
 
 
+/* JG SMP */
+struct privatespace CPU_prvspace[1];
+
 int    _udatasel, _ucodesel, _ucode32sel;
 u_long atdevbase;
 #ifdef SMP
@@ -299,8 +298,7 @@ cpu_startup(void *dummy)
 #ifdef PERFMON
        perfmon_init();
 #endif
-       kprintf("real memory  = %llu (%lluK bytes)\n",
-               (long long)ptoa(Maxmem), (long long)ptoa(Maxmem) / 1024);
+       kprintf("real memory  = %llu (%lluK bytes)\n", ptoa(Maxmem), ptoa(Maxmem) / 1024);
        /*
         * Display any holes after the first chunk of extended memory.
         */
@@ -312,10 +310,8 @@ cpu_startup(void *dummy)
                        vm_paddr_t size1 = phys_avail[indx + 1] - phys_avail[indx];
 
                        kprintf("0x%08llx - 0x%08llx, %llu bytes (%llu pages)\n",
-                           (long long)phys_avail[indx],
-                           (long long)phys_avail[indx + 1] - 1,
-                           (long long)size1,
-                           (long long)(size1 / PAGE_SIZE));
+                           phys_avail[indx], phys_avail[indx + 1] - 1, size1,
+                           size1 / PAGE_SIZE);
                }
        }
 
@@ -1482,7 +1478,7 @@ getmemsize(caddr_t kmdp, u_int64_t first)
                kprintf("Physical memory use set to %ldK\n", Maxmem * 4);
 
        /* call pmap initialization to make new kernel address space */
-       pmap_bootstrap(&first, 0);
+       pmap_bootstrap(&first);
 
        /*
         * Size up each available chunk of physical memory.
@@ -1681,11 +1677,13 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
        u_int64_t msr;
        char *env;
 
+#if JG
        /*
         * This must be done before the first references
         * to CPU_prvspace[0] are made.
         */
        init_paging(&physfree);
+#endif
 
        /*
         * Prevent lowering of the ipl if we call tsleep() early.
@@ -1752,6 +1750,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
        }
        ssdtosyssd(&gdt_segs[GPROC0_SEL],
            (struct system_segment_descriptor *)&gdt[GPROC0_SEL]);
+
        r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1;
        r_gdt.rd_base =  (long) gdt;
        lgdt(&r_gdt);
@@ -1880,11 +1879,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 
        /* setup proc 0's pcb */
        thread0.td_pcb->pcb_flags = 0;
-#if JG
        thread0.td_pcb->pcb_cr3 = KPML4phys;
-#else
-       thread0.td_pcb->pcb_cr3 = IdlePTD;
-#endif
        thread0.td_pcb->pcb_ext = 0;
        lwp0.lwp_md.md_regs = &proc0_tf;
         env = kgetenv("kernelname");
index f5691bd..5708666 100644 (file)
@@ -2,8 +2,10 @@
  * Copyright (c) 1991 Regents of the University of California.
  * Copyright (c) 1994 John S. Dyson
  * Copyright (c) 1994 David Greenman
- * Copyright (c) 2008 The DragonFly Project.
- * Copyright (c) 2008 Jordan Gordeev.
+ * Copyright (c) 2003 Peter Wemm
+ * Copyright (c) 2005-2008 Alan L. Cox <alc@cs.rice.edu>
+ * Copyright (c) 2008, 2009 The DragonFly Project.
+ * Copyright (c) 2008, 2009 Jordan Gordeev.
  * All rights reserved.
  *
  * This code is derived from software contributed to Berkeley by
 #include <machine/pmap.h>
 #include <machine/pmap_inval.h>
 
+#include <ddb/ddb.h>
+
 #define PMAP_KEEP_PDIRS
 #ifndef PMAP_SHPGPERPROC
 #define PMAP_SHPGPERPROC 200
 #define PMAP_INLINE
 #endif
 
+/* JGPMAP32 */
+#define PTDPTDI 0
+
+#define        READY0
+#define        READY1
+#define        READY2
+#define        READY3
+#define        READY4
+#define        READY5
+
 /*
  * Get PDEs and PTEs for user/kernel address space
  */
+#if JGPMAP32
 #define        pmap_pde(m, v)  (&((m)->pm_pdir[(vm_offset_t)(v) >> PDRSHIFT]))
+#endif
+static pd_entry_t *pmap_pde(pmap_t pmap, vm_offset_t va);
 #define pdir_pde(m, v) (m[(vm_offset_t)(v) >> PDRSHIFT])
 
 #define pmap_pde_v(pte)                ((*(pd_entry_t *)pte & PG_V) != 0)
@@ -163,9 +180,18 @@ static int pseflag;                /* PG_PS or-in */
 
 static vm_object_t kptobj;
 
+static int ndmpdp;
+static vm_paddr_t dmaplimit;
 static int nkpt;
 vm_offset_t kernel_vm_end;
 
+static uint64_t        KPDphys;        /* phys addr of kernel level 2 */
+uint64_t               KPDPphys;       /* phys addr of kernel level 3 */
+uint64_t               KPML4phys;      /* phys addr of kernel level 4 */
+
+static uint64_t        DMPDphys;       /* phys addr of direct mapped level 2 */
+static uint64_t        DMPDPphys;      /* phys addr of direct mapped level 3 */
+
 /*
  * Data for the pv entry allocation mechanism
  */
@@ -197,7 +223,6 @@ extern uint64_t SMPptpa;
 #define DISABLE_PSE
 
 static PMAP_INLINE void        free_pv_entry (pv_entry_t pv);
-static pt_entry_t * get_ptbase (pmap_t pmap);
 static pv_entry_t get_pv_entry (void);
 static void    i386_protection_init (void);
 static __inline void   pmap_clearbit (vm_page_t m, int bit);
@@ -220,6 +245,8 @@ static int pmap_release_free_page (pmap_t pmap, vm_page_t p);
 static vm_page_t _pmap_allocpte (pmap_t pmap, vm_pindex_t ptepindex);
 static pt_entry_t * pmap_pte_quick (pmap_t pmap, vm_offset_t va);
 static vm_page_t pmap_page_lookup (vm_object_t object, vm_pindex_t pindex);
+static int pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m,
+                pmap_inval_info_t info);
 static int pmap_unuse_pt (pmap_t, vm_offset_t, vm_page_t, pmap_inval_info_t);
 static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
 
@@ -233,6 +260,7 @@ static unsigned pdir4mb;
  */
 static vm_offset_t
 pmap_kmem_choose(vm_offset_t addr)
+READY0
 {
        vm_offset_t newaddr = addr;
 #ifndef DISABLE_PSE
@@ -244,30 +272,6 @@ pmap_kmem_choose(vm_offset_t addr)
 }
 
 /*
- * pmap_pte:
- *
- *     Extract the page table entry associated with the given map/virtual
- *     pair.
- *
- *     This function may NOT be called from an interrupt.
- */
-PMAP_INLINE pt_entry_t *
-pmap_pte(pmap_t pmap, vm_offset_t va)
-{
-       pd_entry_t *pdeaddr;
-
-       if (pmap) {
-               pdeaddr = pmap_pde(pmap, va);
-               if (*pdeaddr & PG_PS)
-                       return pdeaddr;
-               if (*pdeaddr) {
-                       return get_ptbase(pmap) + amd64_btop(va);
-               }
-       }
-       return (0);
-}
-
-/*
  * pmap_pte_quick:
  *
  *     Super fast pmap_pte routine best used when scanning the pv lists.
@@ -277,35 +281,163 @@ pmap_pte(pmap_t pmap, vm_offset_t va)
  *
  *     Should only be called while in a critical section.
  */
+static __inline pt_entry_t *pmap_pte(pmap_t pmap, vm_offset_t va);
+
 static pt_entry_t * 
 pmap_pte_quick(pmap_t pmap, vm_offset_t va)
+READY0
 {
-       struct mdglobaldata *gd = mdcpu;
-       pd_entry_t pde, newpf;
-
-       if ((pde = pmap->pm_pdir[va >> PDRSHIFT]) != 0) {
-               pd_entry_t frame = pmap->pm_pdir[PTDPTDI] & PG_FRAME;
-               vm_pindex_t index = amd64_btop(va);
-               /* are we current address space or kernel? */
-               if ((pmap == &kernel_pmap) ||
-                       (frame == (PTDpde & PG_FRAME))) {
-                       return (pt_entry_t *) PTmap + index;
-               }
-               newpf = pde & PG_FRAME;
-               if ( ((* (pt_entry_t *) gd->gd_PMAP1) & PG_FRAME) != newpf) {
-                       * (pt_entry_t *) gd->gd_PMAP1 = newpf | PG_RW | PG_V;
-                       cpu_invlpg(gd->gd_PADDR1);
-               }
-               return gd->gd_PADDR1 + (index & (NPTEPG - 1));
-       }
-       return (0);
+       return pmap_pte(pmap, va);
+}
+
+/* Return a non-clipped PD index for a given VA */
+static __inline vm_pindex_t
+pmap_pde_pindex(vm_offset_t va)
+READY1
+{
+       return va >> PDRSHIFT;
+}
+
+/* Return various clipped indexes for a given VA */
+static __inline vm_pindex_t
+pmap_pte_index(vm_offset_t va)
+READY1
+{
+
+       return ((va >> PAGE_SHIFT) & ((1ul << NPTEPGSHIFT) - 1));
+}
+
+static __inline vm_pindex_t
+pmap_pde_index(vm_offset_t va)
+READY1
+{
+
+       return ((va >> PDRSHIFT) & ((1ul << NPDEPGSHIFT) - 1));
+}
+
+static __inline vm_pindex_t
+pmap_pdpe_index(vm_offset_t va)
+READY1
+{
+
+       return ((va >> PDPSHIFT) & ((1ul << NPDPEPGSHIFT) - 1));
+}
+
+static __inline vm_pindex_t
+pmap_pml4e_index(vm_offset_t va)
+READY1
+{
+
+       return ((va >> PML4SHIFT) & ((1ul << NPML4EPGSHIFT) - 1));
+}
+
+/* Return a pointer to the PML4 slot that corresponds to a VA */
+static __inline pml4_entry_t *
+pmap_pml4e(pmap_t pmap, vm_offset_t va)
+READY1
+{
+
+       return (&pmap->pm_pml4[pmap_pml4e_index(va)]);
+}
+
+/* Return a pointer to the PDP slot that corresponds to a VA */
+static __inline pdp_entry_t *
+pmap_pml4e_to_pdpe(pml4_entry_t *pml4e, vm_offset_t va)
+READY1
+{
+       pdp_entry_t *pdpe;
+
+       pdpe = (pdp_entry_t *)PHYS_TO_DMAP(*pml4e & PG_FRAME);
+       return (&pdpe[pmap_pdpe_index(va)]);
+}
+
+/* Return a pointer to the PDP slot that corresponds to a VA */
+static __inline pdp_entry_t *
+pmap_pdpe(pmap_t pmap, vm_offset_t va)
+READY1
+{
+       pml4_entry_t *pml4e;
+
+       pml4e = pmap_pml4e(pmap, va);
+       if ((*pml4e & PG_V) == 0)
+               return NULL;
+       return (pmap_pml4e_to_pdpe(pml4e, va));
+}
+
+/* Return a pointer to the PD slot that corresponds to a VA */
+static __inline pd_entry_t *
+pmap_pdpe_to_pde(pdp_entry_t *pdpe, vm_offset_t va)
+READY1
+{
+       pd_entry_t *pde;
+
+       pde = (pd_entry_t *)PHYS_TO_DMAP(*pdpe & PG_FRAME);
+       return (&pde[pmap_pde_index(va)]);
+}
+
+/* Return a pointer to the PD slot that corresponds to a VA */
+static __inline pd_entry_t *
+pmap_pde(pmap_t pmap, vm_offset_t va)
+READY1
+{
+       pdp_entry_t *pdpe;
+
+       pdpe = pmap_pdpe(pmap, va);
+       if (pdpe == NULL || (*pdpe & PG_V) == 0)
+                return NULL;
+       return (pmap_pdpe_to_pde(pdpe, va));
+}
+
+/* Return a pointer to the PT slot that corresponds to a VA */
+static __inline pt_entry_t *
+pmap_pde_to_pte(pd_entry_t *pde, vm_offset_t va)
+READY1
+{
+       pt_entry_t *pte;
+
+       pte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME);
+       return (&pte[pmap_pte_index(va)]);
+}
+
+/* Return a pointer to the PT slot that corresponds to a VA */
+static __inline pt_entry_t *
+pmap_pte(pmap_t pmap, vm_offset_t va)
+READY1
+{
+       pd_entry_t *pde;
+
+       pde = pmap_pde(pmap, va);
+       if (pde == NULL || (*pde & PG_V) == 0)
+               return NULL;
+       if ((*pde & PG_PS) != 0)        /* compat with i386 pmap_pte() */
+               return ((pt_entry_t *)pde);
+       return (pmap_pde_to_pte(pde, va));
+}
+
+
+PMAP_INLINE pt_entry_t *
+vtopte(vm_offset_t va)
+READY1
+{
+       uint64_t mask = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
+
+       return (PTmap + ((va >> PAGE_SHIFT) & mask));
 }
 
+static __inline pd_entry_t *
+vtopde(vm_offset_t va)
+READY1
+{
+       uint64_t mask = ((1ul << (NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
+
+       return (PDmap + ((va >> PDRSHIFT) & mask));
+}
 
-static u_int64_t
+static uint64_t
 allocpages(vm_paddr_t *firstaddr, int n)
+READY1
 {
-       u_int64_t ret;
+       uint64_t ret;
 
        ret = *firstaddr;
        bzero((void *)ret, n * PAGE_SIZE);
@@ -315,6 +447,7 @@ allocpages(vm_paddr_t *firstaddr, int n)
 
 void
 create_pagetables(vm_paddr_t *firstaddr)
+READY0
 {
        int i;
        int count;
@@ -323,6 +456,83 @@ create_pagetables(vm_paddr_t *firstaddr)
 
        /* we are running (mostly) V=P at this point */
 
+       /* Allocate pages */
+       KPTphys = allocpages(firstaddr, NKPT);
+       KPML4phys = allocpages(firstaddr, 1);
+       KPDPphys = allocpages(firstaddr, NKPML4E);
+       KPDphys = allocpages(firstaddr, NKPDPE);
+
+       ndmpdp = (ptoa(Maxmem) + NBPDP - 1) >> PDPSHIFT;
+       if (ndmpdp < 4)         /* Minimum 4GB of dirmap */
+               ndmpdp = 4;
+       DMPDPphys = allocpages(firstaddr, NDMPML4E);
+       if ((amd_feature & AMDID_PAGE1GB) == 0)
+               DMPDphys = allocpages(firstaddr, ndmpdp);
+       dmaplimit = (vm_paddr_t)ndmpdp << PDPSHIFT;
+
+       /* Fill in the underlying page table pages */
+       /* Read-only from zero to physfree */
+       /* XXX not fully used, underneath 2M pages */
+       for (i = 0; (i << PAGE_SHIFT) < *firstaddr; i++) {
+               ((pt_entry_t *)KPTphys)[i] = i << PAGE_SHIFT;
+               ((pt_entry_t *)KPTphys)[i] |= PG_RW | PG_V | PG_G;
+       }
+
+       /* Now map the page tables at their location within PTmap */
+       for (i = 0; i < NKPT; i++) {
+               ((pd_entry_t *)KPDphys)[i] = KPTphys + (i << PAGE_SHIFT);
+               ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V;
+       }
+
+       /* Map from zero to end of allocations under 2M pages */
+       /* This replaces some of the KPTphys entries above */
+       for (i = 0; (i << PDRSHIFT) < *firstaddr; i++) {
+               ((pd_entry_t *)KPDphys)[i] = i << PDRSHIFT;
+               ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V | PG_PS | PG_G;
+       }
+
+       /* And connect up the PD to the PDP */
+       for (i = 0; i < NKPDPE; i++) {
+               ((pdp_entry_t *)KPDPphys)[i + KPDPI] = KPDphys +
+                   (i << PAGE_SHIFT);
+               ((pdp_entry_t *)KPDPphys)[i + KPDPI] |= PG_RW | PG_V | PG_U;
+       }
+
+       /* Now set up the direct map space using either 2MB or 1GB pages */
+       /* Preset PG_M and PG_A because demotion expects it */
+       if ((amd_feature & AMDID_PAGE1GB) == 0) {
+               for (i = 0; i < NPDEPG * ndmpdp; i++) {
+                       ((pd_entry_t *)DMPDphys)[i] = (vm_paddr_t)i << PDRSHIFT;
+                       ((pd_entry_t *)DMPDphys)[i] |= PG_RW | PG_V | PG_PS |
+                           PG_G | PG_M | PG_A;
+               }
+               /* And the direct map space's PDP */
+               for (i = 0; i < ndmpdp; i++) {
+                       ((pdp_entry_t *)DMPDPphys)[i] = DMPDphys +
+                           (i << PAGE_SHIFT);
+                       ((pdp_entry_t *)DMPDPphys)[i] |= PG_RW | PG_V | PG_U;
+               }
+       } else {
+               for (i = 0; i < ndmpdp; i++) {
+                       ((pdp_entry_t *)DMPDPphys)[i] =
+                           (vm_paddr_t)i << PDPSHIFT;
+                       ((pdp_entry_t *)DMPDPphys)[i] |= PG_RW | PG_V | PG_PS |
+                           PG_G | PG_M | PG_A;
+               }
+       }
+
+       /* And recursively map PML4 to itself in order to get PTmap */
+       ((pdp_entry_t *)KPML4phys)[PML4PML4I] = KPML4phys;
+       ((pdp_entry_t *)KPML4phys)[PML4PML4I] |= PG_RW | PG_V | PG_U;
+
+       /* Connect the Direct Map slot up to the PML4 */
+       ((pdp_entry_t *)KPML4phys)[DMPML4I] = DMPDPphys;
+       ((pdp_entry_t *)KPML4phys)[DMPML4I] |= PG_RW | PG_V | PG_U;
+
+       /* Connect the KVA slot up to the PML4 */
+       ((pdp_entry_t *)KPML4phys)[KPML4I] = KPDPphys;
+       ((pdp_entry_t *)KPML4phys)[KPML4I] |= PG_RW | PG_V | PG_U;
+#if JGPMAP32
        common_lvl4_phys = allocpages(firstaddr, 1);    /* 512 512G mappings */
        common_lvl3_phys = allocpages(firstaddr, 1);    /* 512 1G mappings */
        KPTphys = allocpages(firstaddr, NKPT);          /* kernel page table */
@@ -401,12 +611,15 @@ create_pagetables(vm_paddr_t *firstaddr)
         * Matt: location of user page directory entry (representing 1G)
         */
        link_pdpe = &((pdp_entry_t *)common_lvl3_phys)[LINKPDPI];
+#endif /* JGPMAP32 */
 }
 
+READY0
 void
 init_paging(vm_paddr_t *firstaddr) {
        create_pagetables(firstaddr);
 
+#if JGPMAP32
        /* switch to the newly created page table */
        *link_pdpe = IdlePTD | PG_RW | PG_V | PG_U;
        load_cr3(common_lvl4_phys);
@@ -415,6 +628,7 @@ init_paging(vm_paddr_t *firstaddr) {
        KvaStart = (vm_offset_t)VADDR(PTDPTDI, 0);
        KvaEnd = (vm_offset_t)VADDR(APTDPTDI, 0);
        KvaSize = KvaEnd - KvaStart;
+#endif
 }
 
 /*
@@ -428,7 +642,8 @@ init_paging(vm_paddr_t *firstaddr) {
  *     (physical) address starting relative to 0]
  */
 void
-pmap_bootstrap(vm_paddr_t *firstaddr, vm_paddr_t loadaddr)
+pmap_bootstrap(vm_paddr_t *firstaddr)
+READY0
 {
        vm_offset_t va;
        pt_entry_t *pte;
@@ -436,19 +651,25 @@ pmap_bootstrap(vm_paddr_t *firstaddr, vm_paddr_t loadaddr)
        int i;
        int pg;
 
+       KvaStart = VM_MIN_KERNEL_ADDRESS;
+       KvaEnd = VM_MAX_KERNEL_ADDRESS;
+       KvaSize = KvaEnd - KvaStart;
+
        avail_start = *firstaddr;
 
        /*
-        * XXX The calculation of virtual_start is wrong. It's NKPT*PAGE_SIZE
-        * too large. It should instead be correctly calculated in locore.s and
-        * not based on 'first' (which is a physical address, not a virtual
-        * address, for the start of unused physical memory). The kernel
-        * page tables are NOT double mapped and thus should not be included
-        * in this calculation.
+        * Create an initial set of page tables to run the kernel in.
         */
+       create_pagetables(firstaddr);
+
        virtual_start = (vm_offset_t) PTOV_OFFSET + *firstaddr;
        virtual_start = pmap_kmem_choose(virtual_start);
-       virtual_end = VADDR(KPTDI+NKPDE-1, NPTEPG-1);
+
+       virtual_end = VM_MAX_KERNEL_ADDRESS;
+
+       /* XXX do %cr0 as well */
+       load_cr4(rcr4() | CR4_PGE | CR4_PSE);
+       load_cr3(KPML4phys);
 
        /*
         * Initialize protection array.
@@ -460,7 +681,10 @@ pmap_bootstrap(vm_paddr_t *firstaddr, vm_paddr_t loadaddr)
         * pmap_create, which is unlikely to work correctly at this part of
         * the boot sequence (XXX and which no longer exists).
         */
+#if JGPMAP32
        kernel_pmap.pm_pdir = (pd_entry_t *)(PTOV_OFFSET + (uint64_t)IdlePTD);
+#endif
+       kernel_pmap.pm_pml4 = (pdp_entry_t *) (PTOV_OFFSET + KPML4phys);
        kernel_pmap.pm_count = 1;
        kernel_pmap.pm_active = (cpumask_t)-1;  /* don't allow deactivation */
        TAILQ_INIT(&kernel_pmap.pm_pvlist);
@@ -474,7 +698,11 @@ pmap_bootstrap(vm_paddr_t *firstaddr, vm_paddr_t loadaddr)
        v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n);
 
        va = virtual_start;
+#ifdef JG
        pte = (pt_entry_t *) pmap_pte(&kernel_pmap, va);
+#else
+       pte = vtopte(va);
+#endif
 
        /*
         * CMAP1/CMAP2 are used for zeroing and copying pages.
@@ -502,8 +730,10 @@ pmap_bootstrap(vm_paddr_t *firstaddr, vm_paddr_t loadaddr)
        virtual_start = va;
 
        *CMAP1 = 0;
+#if JGPMAP32
        for (i = 0; i < NKPT; i++)
                PTD[i] = 0;
+#endif
 
        /*
         * PG_G is terribly broken on SMP because we IPI invltlb's in some
@@ -557,7 +787,9 @@ pmap_bootstrap(vm_paddr_t *firstaddr, vm_paddr_t loadaddr)
                 * PSE will be enabled as soon as all APs are up.
                 */
                PTD[KPTDI] = (pd_entry_t)ptditmp;
+#if JGPMAP32
                kernel_pmap.pm_pdir[KPTDI] = (pd_entry_t)ptditmp;
+#endif
                cpu_invltlb();
 #endif
        }
@@ -596,12 +828,15 @@ pmap_bootstrap(vm_paddr_t *firstaddr, vm_paddr_t loadaddr)
  */
 void
 pmap_set_opt(void)
+READY0
 {
        if (pseflag && (cpu_feature & CPUID_PSE)) {
                load_cr4(rcr4() | CR4_PSE);
                if (pdir4mb && mycpu->gd_cpuid == 0) {  /* only on BSP */
+#if JGPMAP32
                        kernel_pmap.pm_pdir[KPTDI] =
                            PTD[KPTDI] = (pd_entry_t)pdir4mb;
+#endif
                        cpu_invltlb();
                }
        }
@@ -617,6 +852,7 @@ pmap_set_opt(void)
  */
 void
 pmap_init(void)
+READY0
 {
        int i;
        int initial_pvs;
@@ -624,7 +860,8 @@ pmap_init(void)
        /*
         * object for kernel page table pages
         */
-       kptobj = vm_object_allocate(OBJT_DEFAULT, NKPDE);
+       /* JG I think the number can be arbitrary */
+       kptobj = vm_object_allocate(OBJT_DEFAULT, 5);
 
        /*
         * Allocate memory for random pmap data structures.  Includes the
@@ -664,6 +901,7 @@ pmap_init(void)
  */
 void
 pmap_init2(void)
+READY0
 {
        int shpgperproc = PMAP_SHPGPERPROC;
 
@@ -686,12 +924,9 @@ pmap_init2(void)
  * This should be an invalid condition.
  */
 static int
-pmap_nw_modified(pt_entry_t ptea)
+pmap_nw_modified(pt_entry_t pte)
+READY1
 {
-       int pte;
-
-       pte = (int) ptea;
-
        if ((pte & (PG_M|PG_RW)) == PG_M)
                return 1;
        else
@@ -706,6 +941,7 @@ pmap_nw_modified(pt_entry_t ptea)
  */
 static PMAP_INLINE int
 pmap_track_modified(vm_offset_t va)
+READY0
 {
        if ((va < clean_sva) || (va >= clean_eva)) 
                return 1;
@@ -713,29 +949,6 @@ pmap_track_modified(vm_offset_t va)
                return 0;
 }
 
-static pt_entry_t *
-get_ptbase(pmap_t pmap)
-{
-       pd_entry_t frame = pmap->pm_pdir[PTDPTDI] & PG_FRAME;
-       struct globaldata *gd = mycpu;
-
-       /* are we current address space or kernel? */
-       if (pmap == &kernel_pmap || frame == (PTDpde & PG_FRAME)) {
-               return (pt_entry_t *) PTmap;
-       }
-
-       /* otherwise, we are alternate address space */
-       KKASSERT(gd->gd_intr_nesting_level == 0 &&
-                (gd->gd_curthread->td_flags & TDF_INTTHREAD) == 0);
-
-       if (frame != (((pd_entry_t) APTDpde) & PG_FRAME)) {
-               APTDpde = (pd_entry_t)(frame | PG_RW | PG_V);
-               /* The page directory is not shared between CPUs */
-               cpu_invltlb();
-       }
-       return (pt_entry_t *) APTmap;
-}
-
 /*
  * pmap_extract:
  *
@@ -746,23 +959,61 @@ get_ptbase(pmap_t pmap)
  */
 vm_paddr_t 
 pmap_extract(pmap_t pmap, vm_offset_t va)
+READY1
 {
-       vm_offset_t rtval;
-       vm_offset_t pdirindex;
+       vm_paddr_t rtval;
+       pt_entry_t *pte;
+       pd_entry_t pde, *pdep;
 
-       pdirindex = va >> PDRSHIFT;
-       if (pmap && (rtval = pmap->pm_pdir[pdirindex])) {
-               pt_entry_t *pte;
-               if ((rtval & PG_PS) != 0) {
-                       rtval &= ~(NBPDR - 1);
-                       rtval |= va & (NBPDR - 1);
-                       return rtval;
+       rtval = 0;
+       pdep = pmap_pde(pmap, va);
+       if (pdep != NULL) {
+               pde = *pdep;
+               if (pde) {
+                       if ((pde & PG_PS) != 0) {
+                               rtval = (pde & PG_PS_FRAME) | (va & PDRMASK);
+                       } else {
+                               pte = pmap_pde_to_pte(pdep, va);
+                               rtval = (*pte & PG_FRAME) | (va & PAGE_MASK);
+                       }
                }
-               pte = get_ptbase(pmap) + amd64_btop(va);
-               rtval = ((*pte & PG_FRAME) | (va & PAGE_MASK));
-               return rtval;
        }
-       return 0;
+       return rtval;
+}
+
+/*
+ *     Routine:        pmap_kextract
+ *     Function:
+ *             Extract the physical page address associated
+ *             kernel virtual address.
+ */
+vm_paddr_t
+pmap_kextract(vm_offset_t va)
+READY1
+{
+       pd_entry_t pde;
+       vm_paddr_t pa;
+
+       if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
+               pa = DMAP_TO_PHYS(va);
+       } else {
+               pde = *vtopde(va);
+               if (pde & PG_PS) {
+                       pa = (pde & PG_PS_FRAME) | (va & PDRMASK);
+               } else {
+                       /*
+                        * Beware of a concurrent promotion that changes the
+                        * PDE at this point!  For example, vtopte() must not
+                        * be used to access the PTE because it would use the
+                        * new PDE.  It is, however, safe to use the old PDE
+                        * because the page table page is preserved by the
+                        * promotion.
+                        */
+                       pa = *pmap_pde_to_pte(&pde, va);
+                       pa = (pa & PG_FRAME) | (va & PAGE_MASK);
+               }
+       }
+       return pa;
 }
 
 /***************************************************
@@ -778,6 +1029,7 @@ pmap_extract(pmap_t pmap, vm_offset_t va)
  */
 void 
 pmap_kenter(vm_offset_t va, vm_paddr_t pa)
+READY1
 {
        pt_entry_t *pte;
        pt_entry_t npte;
@@ -799,6 +1051,7 @@ pmap_kenter(vm_offset_t va, vm_paddr_t pa)
  */
 void
 pmap_kenter_quick(vm_offset_t va, vm_paddr_t pa)
+READY1
 {
        pt_entry_t *pte;
        pt_entry_t npte;
@@ -811,6 +1064,7 @@ pmap_kenter_quick(vm_offset_t va, vm_paddr_t pa)
 
 void
 pmap_kenter_sync(vm_offset_t va)
+READY1
 {
        pmap_inval_info info;
 
@@ -821,6 +1075,7 @@ pmap_kenter_sync(vm_offset_t va)
 
 void
 pmap_kenter_sync_quick(vm_offset_t va)
+READY1
 {
        cpu_invlpg((void *)va);
 }
@@ -830,6 +1085,7 @@ pmap_kenter_sync_quick(vm_offset_t va)
  */
 void
 pmap_kremove(vm_offset_t va)
+READY1
 {
        pt_entry_t *pte;
        pmap_inval_info info;
@@ -843,6 +1099,7 @@ pmap_kremove(vm_offset_t va)
 
 void
 pmap_kremove_quick(vm_offset_t va)
+READY1
 {
        pt_entry_t *pte;
        pte = vtopte(va);
@@ -855,6 +1112,7 @@ pmap_kremove_quick(vm_offset_t va)
  */
 void
 pmap_kmodify_rw(vm_offset_t va)
+READY1
 {
        *vtopte(va) |= PG_RW;
        cpu_invlpg((void *)va);
@@ -862,6 +1120,7 @@ pmap_kmodify_rw(vm_offset_t va)
 
 void
 pmap_kmodify_nc(vm_offset_t va)
+READY1
 {
        *vtopte(va) |= PG_N;
        cpu_invlpg((void *)va);
@@ -876,7 +1135,12 @@ pmap_kmodify_nc(vm_offset_t va)
  */
 vm_offset_t
 pmap_map(vm_offset_t virt, vm_paddr_t start, vm_paddr_t end, int prot)
+READY1
 {
+       /*
+        * JG Are callers prepared to get an address in the DMAP,
+        * instead of the passed-in virt?
+        */
        while (start < end) {
                pmap_kenter(virt, start);
                virt += PAGE_SIZE;
@@ -896,6 +1160,7 @@ pmap_map(vm_offset_t virt, vm_paddr_t start, vm_paddr_t end, int prot)
  */
 void
 pmap_qenter(vm_offset_t va, vm_page_t *m, int count)
+READY1
 {
        vm_offset_t end_va;
 
@@ -917,6 +1182,7 @@ pmap_qenter(vm_offset_t va, vm_page_t *m, int count)
 
 void
 pmap_qenter2(vm_offset_t va, vm_page_t *m, int count, cpumask_t *mask)
+READY1
 {
        vm_offset_t end_va;
        cpumask_t cmask = mycpu->gd_cpumask;
@@ -955,10 +1221,11 @@ pmap_qenter2(vm_offset_t va, vm_page_t *m, int count, cpumask_t *mask)
  */
 void
 pmap_qremove(vm_offset_t va, int count)
+READY1
 {
        vm_offset_t end_va;
 
-       end_va = va + count*PAGE_SIZE;
+       end_va = va + count * PAGE_SIZE;
 
        while (va < end_va) {
                pt_entry_t *pte;
@@ -983,6 +1250,7 @@ pmap_qremove(vm_offset_t va, int count)
  */
 static vm_page_t
 pmap_page_lookup(vm_object_t object, vm_pindex_t pindex)
+READY1
 {
        vm_page_t m;
 
@@ -999,11 +1267,12 @@ pmap_page_lookup(vm_object_t object, vm_pindex_t pindex)
  */
 void
 pmap_init_thread(thread_t td)
+READY1
 {
        /* enforce pcb placement */
        td->td_pcb = (struct pcb *)(td->td_kstack + td->td_kstack_size) - 1;
        td->td_savefpu = &td->td_pcb->pcb_save;
-       td->td_sp = (char *)td->td_pcb - 16;
+       td->td_sp = (char *)td->td_pcb - 16; /* JG is -16 needed on amd64? */
 }
 
 /*
@@ -1011,6 +1280,7 @@ pmap_init_thread(thread_t td)
  */
 void
 pmap_init_proc(struct proc *p)
+READY1
 {
 }
 
@@ -1020,6 +1290,7 @@ pmap_init_proc(struct proc *p)
  */
 void
 pmap_dispose_proc(struct proc *p)
+READY1
 {
        KASSERT(p->p_lock == 0, ("attempt to dispose referenced proc! %p", p));
 }
@@ -1033,7 +1304,8 @@ pmap_dispose_proc(struct proc *p)
  * drops to zero, then it decrements the wire count.
  */
 static int 
-_pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m, pmap_inval_info_t info) 
+_pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m, pmap_inval_info_t info) 
+READY1
 {
        /* 
         * Wait until we can busy the page ourselves.  We cannot have
@@ -1053,7 +1325,23 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m, pmap_inval_info_t info)
                 */
                vm_page_busy(m);
                pmap_inval_add(info, pmap, -1);
-               pmap->pm_pdir[m->pindex] = 0;
+
+               if (m->pindex >= (NUPDE + NUPDPE)) {
+                       /* PDP page */
+                       pml4_entry_t *pml4;
+                       pml4 = pmap_pml4e(pmap, va);
+                       *pml4 = 0;
+               } else if (m->pindex >= NUPDE) {
+                       /* PD page */
+                       pdp_entry_t *pdp;
+                       pdp = pmap_pdpe(pmap, va);
+                       *pdp = 0;
+               } else {
+                       /* PTE page */
+                       pd_entry_t *pd;
+                       pd = pmap_pde(pmap, va);
+                       *pd = 0;
+               }
 
                KKASSERT(pmap->pm_stats.resident_count > 0);
                --pmap->pm_stats.resident_count;
@@ -1061,6 +1349,23 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m, pmap_inval_info_t info)
                if (pmap->pm_ptphint == m)
                        pmap->pm_ptphint = NULL;
 
+#if JG
+               if (m->pindex < NUPDE) {
+                       /* We just released a PT, unhold the matching PD */
+                       vm_page_t pdpg;
+       
+                       pdpg = PHYS_TO_VM_PAGE(*pmap_pdpe(pmap, va) & PG_FRAME);
+                       pmap_unwire_pte_hold(pmap, va, pdpg, info);
+               }
+               if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) {
+                       /* We just released a PD, unhold the matching PDP */
+                       vm_page_t pdppg;
+       
+                       pdppg = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, va) & PG_FRAME);
+                       pmap_unwire_pte_hold(pmap, va, pdppg, info);
+               }
+#endif
+
                /*
                 * This was our last hold, the page had better be unwired
                 * after we decrement wire_count.
@@ -1084,14 +1389,15 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m, pmap_inval_info_t info)
 }
 
 static PMAP_INLINE int
-pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m, pmap_inval_info_t info)
+pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m, pmap_inval_info_t info)
+READY1
 {
        KKASSERT(m->hold_count > 0);
        if (m->hold_count > 1) {
                vm_page_unhold(m);
                return 0;
        } else {
-               return _pmap_unwire_pte_hold(pmap, m, info);
+               return _pmap_unwire_pte_hold(pmap, va, m, info);
        }
 }
 
@@ -1102,24 +1408,30 @@ pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m, pmap_inval_info_t info)
 static int
 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, vm_page_t mpte,
                pmap_inval_info_t info)
+READY1
 {
+       /* JG Use FreeBSD/amd64 or FreeBSD/i386 ptepde approaches? */
        vm_pindex_t ptepindex;
-       if (va >= UPT_MIN_ADDRESS)
+       if (va >= VM_MAX_USER_ADDRESS)
                return 0;
 
        if (mpte == NULL) {
-               ptepindex = (va >> PDRSHIFT);
+               ptepindex = pmap_pde_pindex(va);
+#if JGHINT
                if (pmap->pm_ptphint &&
                        (pmap->pm_ptphint->pindex == ptepindex)) {
                        mpte = pmap->pm_ptphint;
                } else {
+#endif
                        pmap_inval_flush(info);
-                       mpte = pmap_page_lookup( pmap->pm_pteobj, ptepindex);
+                       mpte = pmap_page_lookup(pmap->pm_pteobj, ptepindex);
                        pmap->pm_ptphint = mpte;
+#if JGHINT
                }
+#endif
        }
 
-       return pmap_unwire_pte_hold(pmap, mpte, info);
+       return pmap_unwire_pte_hold(pmap, va, mpte, info);
 }
 
 /*
@@ -1132,10 +1444,14 @@ pmap_unuse_pt(pmap_t pmap, vm_offset_t va, vm_page_t mpte,
  */
 void
 pmap_pinit0(struct pmap *pmap)
+READY1
 {
+#if JGPMAP32
        pmap->pm_pdir =
                (pd_entry_t *)kmem_alloc_pageable(&kernel_map, PAGE_SIZE);
        pmap_kenter((vm_offset_t)pmap->pm_pdir, (vm_offset_t) IdlePTD);
+#endif
+       pmap->pm_pml4 = (pml4_entry_t *)(PTOV_OFFSET + KPML4phys);
        pmap->pm_count = 1;
        pmap->pm_active = 0;
        pmap->pm_ptphint = NULL;
@@ -1149,6 +1465,7 @@ pmap_pinit0(struct pmap *pmap)
  */
 void
 pmap_pinit(struct pmap *pmap)
+READY1
 {
        vm_page_t ptdpg;
 
@@ -1156,16 +1473,16 @@ pmap_pinit(struct pmap *pmap)
         * No need to allocate page table space yet but we do need a valid
         * page directory table.
         */
-       if (pmap->pm_pdir == NULL) {
-               pmap->pm_pdir =
-                   (pd_entry_t *)kmem_alloc_pageable(&kernel_map, PAGE_SIZE);
+       if (pmap->pm_pml4 == NULL) {
+               pmap->pm_pml4 =
+                   (pml4_entry_t *)kmem_alloc_pageable(&kernel_map, PAGE_SIZE);
        }
 
        /*
         * Allocate an object for the ptes
         */
        if (pmap->pm_pteobj == NULL)
-               pmap->pm_pteobj = vm_object_allocate(OBJT_DEFAULT, PTDPTDI + 1);
+               pmap->pm_pteobj = vm_object_allocate(OBJT_DEFAULT, PML4PML4I + 1);
 
        /*
         * Allocate the page directory page, unless we already have
@@ -1173,23 +1490,23 @@ pmap_pinit(struct pmap *pmap)
         * already be set appropriately.
         */
        if ((ptdpg = pmap->pm_pdirm) == NULL) {
-               ptdpg = vm_page_grab(pmap->pm_pteobj, PTDPTDI,
+               ptdpg = vm_page_grab(pmap->pm_pteobj, PML4PML4I,
                                     VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
                pmap->pm_pdirm = ptdpg;
                vm_page_flag_clear(ptdpg, PG_MAPPED | PG_BUSY);
                ptdpg->valid = VM_PAGE_BITS_ALL;
                ptdpg->wire_count = 1;
                ++vmstats.v_wire_count;
-               pmap_kenter((vm_offset_t)pmap->pm_pdir, VM_PAGE_TO_PHYS(ptdpg));
+               pmap_kenter((vm_offset_t)pmap->pm_pml4, VM_PAGE_TO_PHYS(ptdpg));
        }
        if ((ptdpg->flags & PG_ZERO) == 0)
-               bzero(pmap->pm_pdir, PAGE_SIZE);
+               bzero(pmap->pm_pml4, PAGE_SIZE);
 
-       pmap->pm_pdir[MPPTDI] = PTD[MPPTDI];
+       pmap->pm_pml4[KPML4I] = KPDPphys | PG_RW | PG_V | PG_U;
+       pmap->pm_pml4[DMPML4I] = DMPDPphys | PG_RW | PG_V | PG_U;
 
        /* install self-referential address mapping entry */
-       *(pd_entry_t *) (pmap->pm_pdir + PTDPTDI) =
-               VM_PAGE_TO_PHYS(ptdpg) | PG_V | PG_RW | PG_A | PG_M;
+       pmap->pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(ptdpg) | PG_V | PG_RW | PG_A | PG_M;
 
        pmap->pm_count = 1;
        pmap->pm_active = 0;
@@ -1207,13 +1524,15 @@ pmap_pinit(struct pmap *pmap)
  */
 void
 pmap_puninit(pmap_t pmap)
+READY1
 {
        vm_page_t p;
 
        KKASSERT(pmap->pm_active == 0);
        if ((p = pmap->pm_pdirm) != NULL) {
-               KKASSERT(pmap->pm_pdir != NULL);
-               pmap_kremove((vm_offset_t)pmap->pm_pdir);
+               KKASSERT(pmap->pm_pml4 != NULL);
+               KKASSERT(pmap->pm_pml4 != (PTOV_OFFSET + KPML4phys));
+               pmap_kremove((vm_offset_t)pmap->pm_pml4);
                p->wire_count--;
                vmstats.v_wire_count--;
                KKASSERT((p->flags & PG_BUSY) == 0);
@@ -1221,9 +1540,10 @@ pmap_puninit(pmap_t pmap)
                vm_page_free_zero(p);
                pmap->pm_pdirm = NULL;
        }
-       if (pmap->pm_pdir) {
-               kmem_free(&kernel_map, (vm_offset_t)pmap->pm_pdir, PAGE_SIZE);
-               pmap->pm_pdir = NULL;
+       if (pmap->pm_pml4) {
+               KKASSERT(pmap->pm_pml4 != (PTOV_OFFSET + KPML4phys));
+               kmem_free(&kernel_map, (vm_offset_t)pmap->pm_pml4, PAGE_SIZE);
+               pmap->pm_pml4 = NULL;
        }
        if (pmap->pm_pteobj) {
                vm_object_deallocate(pmap->pm_pteobj);
@@ -1239,11 +1559,14 @@ pmap_puninit(pmap_t pmap)
  */
 void
 pmap_pinit2(struct pmap *pmap)
+READY0
 {
        crit_enter();
        TAILQ_INSERT_TAIL(&pmap_list, pmap, pm_pmnode);
        /* XXX copies current process, does not fill in MPPTDI */
+#if JGPMAP32
        bcopy(PTD + KPTDI, pmap->pm_pdir + KPTDI, nkpt * PTESIZE);
+#endif
        crit_exit();
 }
 
@@ -1257,8 +1580,9 @@ pmap_pinit2(struct pmap *pmap)
  */
 static int
 pmap_release_free_page(struct pmap *pmap, vm_page_t p)
+READY1
 {
-       pd_entry_t *pde = (pd_entry_t *) pmap->pm_pdir;
+       pml4_entry_t *pml4 = pmap->pm_pml4;
        /*
         * This code optimizes the case of freeing non-busy
         * page-table pages.  Those pages are zero now, and
@@ -1272,7 +1596,12 @@ pmap_release_free_page(struct pmap *pmap, vm_page_t p)
        /*
         * Remove the page table page from the processes address space.
         */
+       /* JG XXX we need to turn 'pindex' into a page table level
+        * (PML4, PDP, PD, PT) and index within the page table page
+        */
+#if JGPMAP32
        pde[p->pindex] = 0;
+#endif
        KKASSERT(pmap->pm_stats.resident_count > 0);
        --pmap->pm_stats.resident_count;
 
@@ -1282,22 +1611,9 @@ pmap_release_free_page(struct pmap *pmap, vm_page_t p)
        if (pmap->pm_ptphint && (pmap->pm_ptphint->pindex == p->pindex))
                pmap->pm_ptphint = NULL;
 
-       /*
-        * We leave the page directory page cached, wired, and mapped in
-        * the pmap until the dtor function (pmap_puninit()) gets called.
-        * However, still clean it up so we can set PG_ZERO.
-        */
-       if (p->pindex == PTDPTDI) {
-               bzero(pde + KPTDI, nkpt * PTESIZE);
-               pde[MPPTDI] = 0;
-               pde[APTDPTDI] = 0;
-               vm_page_flag_set(p, PG_ZERO);
-               vm_page_wakeup(p);
-       } else {
-               p->wire_count--;
-               vmstats.v_wire_count--;
-               vm_page_free_zero(p);
-       }
+       p->wire_count--;
+       vmstats.v_wire_count--;
+       vm_page_free_zero(p);
        return 1;
 }
 
@@ -1307,9 +1623,9 @@ pmap_release_free_page(struct pmap *pmap, vm_page_t p)
  */
 static vm_page_t
 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex)
+READY1
 {
-       vm_offset_t pteva, ptepa;
-       vm_page_t m;
+       vm_page_t m, pdppg, pdpg;
 
        /*
         * Find or fabricate a new pagetable page
@@ -1317,6 +1633,11 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex)
        m = vm_page_grab(pmap->pm_pteobj, ptepindex,
                        VM_ALLOC_NORMAL | VM_ALLOC_ZERO | VM_ALLOC_RETRY);
 
+
+       if ((m->flags & PG_ZERO) == 0) {
+               pmap_zero_page(VM_PAGE_TO_PHYS(m));
+       }
+
        KASSERT(m->queue == PQ_NONE,
                ("_pmap_allocpte: %p->queue != PQ_NONE", m));
 
@@ -1331,11 +1652,13 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex)
         * directory page while we were blocked, if so just unbusy and
         * return the held page.
         */
+#if JGPMAP32
        if ((ptepa = pmap->pm_pdir[ptepindex]) != 0) {
                KKASSERT((ptepa & PG_FRAME) == VM_PAGE_TO_PHYS(m));
                vm_page_wakeup(m);
                return(m);
        }
+#endif
 
        if (m->wire_count == 0)
                vmstats.v_wire_count++;
@@ -1349,29 +1672,103 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex)
 
        ++pmap->pm_stats.resident_count;
 
+#if JGPMAP32
        ptepa = VM_PAGE_TO_PHYS(m);
        pmap->pm_pdir[ptepindex] =
                (pd_entry_t) (ptepa | PG_U | PG_RW | PG_V | PG_A | PG_M);
+#endif
+       if (ptepindex >= (NUPDE + NUPDPE)) {
+               pml4_entry_t *pml4;
+               vm_pindex_t pml4index;
+
+               /* Wire up a new PDPE page */
+               pml4index = ptepindex - (NUPDE + NUPDPE);
+               pml4 = &pmap->pm_pml4[pml4index];
+               *pml4 = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
+
+       } else if (ptepindex >= NUPDE) {
+               vm_pindex_t pml4index;
+               vm_pindex_t pdpindex;
+               pml4_entry_t *pml4;
+               pdp_entry_t *pdp;
+
+               /* Wire up a new PDE page */
+               pdpindex = ptepindex - NUPDE;
+               pml4index = pdpindex >> NPML4EPGSHIFT;
+
+               pml4 = &pmap->pm_pml4[pml4index];
+               if ((*pml4 & PG_V) == 0) {
+                       /* Have to allocate a new pdp, recurse */
+                       if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml4index)
+                            == NULL) {
+                               --m->wire_count;
+                               vm_page_free(m);
+                               return (NULL);
+                       }
+               } else {
+                       /* Add reference to pdp page */
+                       pdppg = PHYS_TO_VM_PAGE(*pml4 & PG_FRAME);
+                       pdppg->wire_count++;
+               }
+               pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
 
-       /*
-        * Set the page table hint
-        */
-       pmap->pm_ptphint = m;
+               /* Now find the pdp page */
+               pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
+               *pdp = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 
-       /*
-        * Try to use the new mapping, but if we cannot, then
-        * do it with the routine that maps the page explicitly.
-        */
-       if ((m->flags & PG_ZERO) == 0) {
-               if ((pmap->pm_pdir[PTDPTDI] & PG_FRAME) ==
-                       (((pd_entry_t) PTDpde) & PG_FRAME)) {
-                       pteva = UPT_MIN_ADDRESS + amd64_ptob(ptepindex);
-                       bzero((caddr_t) pteva, PAGE_SIZE);
+       } else {
+               vm_pindex_t pml4index;
+               vm_pindex_t pdpindex;
+               pml4_entry_t *pml4;
+               pdp_entry_t *pdp;
+               pd_entry_t *pd;
+
+               /* Wire up a new PTE page */
+               pdpindex = ptepindex >> NPDPEPGSHIFT;
+               pml4index = pdpindex >> NPML4EPGSHIFT;
+
+               /* First, find the pdp and check that its valid. */
+               pml4 = &pmap->pm_pml4[pml4index];
+               if ((*pml4 & PG_V) == 0) {
+                       /* Have to allocate a new pd, recurse */
+                       if (_pmap_allocpte(pmap, NUPDE + pdpindex)
+                            == NULL) {
+                               --m->wire_count;
+                               vm_page_free(m);
+                               return (NULL);
+                       }
+                       pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
+                       pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
                } else {
-                       pmap_zero_page(ptepa);
+                       pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
+                       pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
+                       if ((*pdp & PG_V) == 0) {
+                               /* Have to allocate a new pd, recurse */
+                               if (_pmap_allocpte(pmap, NUPDE + pdpindex)
+                                    == NULL) {
+                                       --m->wire_count;
+                                       vm_page_free(m);
+                                       return (NULL);
+                               }
+                       } else {
+                               /* Add reference to the pd page */
+                               pdpg = PHYS_TO_VM_PAGE(*pdp & PG_FRAME);
+                               pdpg->wire_count++;
+                       }
                }
+               pd = (pd_entry_t *)PHYS_TO_DMAP(*pdp & PG_FRAME);
+
+               /* Now we know where the page directory page is */
+               pd = &pd[ptepindex & ((1ul << NPDEPGSHIFT) - 1)];
+               *pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
        }
 
+
+       /*
+        * Set the page table hint
+        */
+       pmap->pm_ptphint = m;
+
        m->valid = VM_PAGE_BITS_ALL;
        vm_page_flag_clear(m, PG_ZERO);
        vm_page_flag_set(m, PG_MAPPED);
@@ -1382,28 +1779,29 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex)
 
 static vm_page_t
 pmap_allocpte(pmap_t pmap, vm_offset_t va)
+READY1
 {
        vm_pindex_t ptepindex;
-       vm_offset_t ptepa;
+       pd_entry_t *pd;
        vm_page_t m;
 
        /*
         * Calculate pagetable page index
         */
-       ptepindex = va >> PDRSHIFT;
+       ptepindex = pmap_pde_pindex(va);
 
        /*
         * Get the page directory entry
         */
-       ptepa = (vm_offset_t) pmap->pm_pdir[ptepindex];
+       pd = pmap_pde(pmap, va);
 
        /*
-        * This supports switching from a 4MB page to a
+        * This supports switching from a 2MB page to a
         * normal 4K page.
         */
-       if (ptepa & PG_PS) {
-               pmap->pm_pdir[ptepindex] = 0;
-               ptepa = 0;
+       if (pd != NULL && (*pd & (PG_PS | PG_V)) == (PG_PS | PG_V)) {
+               *pd = 0;
+               pd = NULL;
                cpu_invltlb();
                smp_invltlb();
        }
@@ -1412,18 +1810,10 @@ pmap_allocpte(pmap_t pmap, vm_offset_t va)
         * If the page table page is mapped, we just increment the
         * hold count, and activate it.
         */
-       if (ptepa) {
-               /*
-                * In order to get the page table page, try the
-                * hint first.
-                */
-               if (pmap->pm_ptphint &&
-                       (pmap->pm_ptphint->pindex == ptepindex)) {
-                       m = pmap->pm_ptphint;
-               } else {
-                       m = pmap_page_lookup( pmap->pm_pteobj, ptepindex);
-                       pmap->pm_ptphint = m;
-               }
+       if (pd != NULL && (*pd & PG_V) != 0) {
+               /* YYY hint is used here on i386 */
+               m = pmap_page_lookup( pmap->pm_pteobj, ptepindex);
+               pmap->pm_ptphint = m;
                m->hold_count++;
                return m;
        }
@@ -1447,6 +1837,7 @@ static int pmap_release_callback(struct vm_page *p, void *data);
 
 void
 pmap_release(struct pmap *pmap)
+READY1
 {
        vm_object_t object = pmap->pm_pteobj;
        struct rb_vm_page_scan_info info;
@@ -1481,10 +1872,11 @@ pmap_release(struct pmap *pmap)
 
 static int
 pmap_release_callback(struct vm_page *p, void *data)
+READY1
 {
        struct rb_vm_page_scan_info *info = data;
 
-       if (p->pindex == PTDPTDI) {
+       if (p->pindex == PML4PML4I) {
                info->mpte = p;
                return(0);
        }
@@ -1505,32 +1897,61 @@ pmap_release_callback(struct vm_page *p, void *data)
 
 void
 pmap_growkernel(vm_offset_t addr)
+READY1
 {
+       vm_paddr_t paddr;
        struct pmap *pmap;
        vm_offset_t ptppaddr;
        vm_page_t nkpg;
-       pd_entry_t newpdir;
+       pd_entry_t *pde, newpdir;
+       pdp_entry_t newpdp;
 
        crit_enter();
        if (kernel_vm_end == 0) {
                kernel_vm_end = KERNBASE;
                nkpt = 0;
-               while (pdir_pde(PTD, kernel_vm_end)) {
+               while ((*pmap_pde(&kernel_pmap, kernel_vm_end) & PG_V) != 0) {
                        kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
                        nkpt++;
+                       if (kernel_vm_end - 1 >= kernel_map.max_offset) {
+                               kernel_vm_end = kernel_map.max_offset;
+                               break;                       
+                       }
                }
        }
-       addr = (addr + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
+       addr = roundup2(addr, PAGE_SIZE * NPTEPG);
+       if (addr - 1 >= kernel_map.max_offset)
+               addr = kernel_map.max_offset;
        while (kernel_vm_end < addr) {
-               if (pdir_pde(PTD, kernel_vm_end)) {
+               pde = pmap_pde(&kernel_pmap, kernel_vm_end);
+               if (pde == NULL) {
+                       /* We need a new PDP entry */
+                       nkpg = vm_page_alloc(kptobj, nkpt,
+                                            VM_ALLOC_NORMAL | VM_ALLOC_SYSTEM
+                                            | VM_ALLOC_INTERRUPT);
+                       if (nkpg == NULL)
+                               panic("pmap_growkernel: no memory to grow kernel");
+                       if ((nkpg->flags & PG_ZERO) == 0)
+                               pmap_zero_page(nkpg);
+                       paddr = VM_PAGE_TO_PHYS(nkpg);
+                       newpdp = (pdp_entry_t)
+                               (paddr | PG_V | PG_RW | PG_A | PG_M);
+                       *pmap_pdpe(&kernel_pmap, kernel_vm_end) = newpdp;
+                       continue; /* try again */
+               }
+               if ((*pde & PG_V) != 0) {
                        kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
+                       if (kernel_vm_end - 1 >= kernel_map.max_offset) {
+                               kernel_vm_end = kernel_map.max_offset;
+                               break;                       
+                       }
                        continue;
                }
 
                /*
                 * This index is bogus, but out of the way
                 */
-               nkpg = vm_page_alloc(kptobj, nkpt, 
+               nkpg = vm_page_alloc(kptobj, nkpt,
                        VM_ALLOC_NORMAL | VM_ALLOC_SYSTEM | VM_ALLOC_INTERRUPT);
                if (nkpg == NULL)
                        panic("pmap_growkernel: no memory to grow kernel");
@@ -1539,18 +1960,14 @@ pmap_growkernel(vm_offset_t addr)
                ptppaddr = VM_PAGE_TO_PHYS(nkpg);
                pmap_zero_page(ptppaddr);
                newpdir = (pd_entry_t) (ptppaddr | PG_V | PG_RW | PG_A | PG_M);
-               pdir_pde(PTD, kernel_vm_end) = newpdir;
                *pmap_pde(&kernel_pmap, kernel_vm_end) = newpdir;
                nkpt++;
 
-               /*
-                * This update must be interlocked with pmap_pinit2.
-                */
-               TAILQ_FOREACH(pmap, &pmap_list, pm_pmnode) {
-                       *pmap_pde(pmap, kernel_vm_end) = newpdir;
+               kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
+               if (kernel_vm_end - 1 >= kernel_map.max_offset) {
+                       kernel_vm_end = kernel_map.max_offset;
+                       break;                       
                }
-               kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) &
-                               ~(PAGE_SIZE * NPTEPG - 1);
        }
        crit_exit();
 }
@@ -1562,6 +1979,7 @@ pmap_growkernel(vm_offset_t addr)
  */
 void
 pmap_destroy(pmap_t pmap)
+READY0
 {
        int count;
 
@@ -1580,6 +1998,7 @@ pmap_destroy(pmap_t pmap)
  */
 void
 pmap_reference(pmap_t pmap)
+READY2
 {
        if (pmap != NULL) {
                pmap->pm_count++;
@@ -1596,8 +2015,10 @@ pmap_reference(pmap_t pmap)
  */
 static PMAP_INLINE void
 free_pv_entry(pv_entry_t pv)
+READY2
 {
        pv_entry_count--;
+       KKASSERT(pv_entry_count >= 0);
        zfree(pvzone, pv);
 }
 
@@ -1607,13 +2028,14 @@ free_pv_entry(pv_entry_t pv)
  */
 static pv_entry_t
 get_pv_entry(void)
+READY2
 {
        pv_entry_count++;
        if (pv_entry_high_water &&
-           (pv_entry_count > pv_entry_high_water) &&
-           (pmap_pagedaemon_waken == 0)) {
+               (pv_entry_count > pv_entry_high_water) &&
+               (pmap_pagedaemon_waken == 0)) {
                pmap_pagedaemon_waken = 1;
-               wakeup (&vm_pages_needed);
+               wakeup(&vm_pages_needed);
        }
        return zalloc(pvzone);
 }
@@ -1624,6 +2046,7 @@ get_pv_entry(void)
  */
 void
 pmap_collect(void)
+READY0
 {
        int i;
        vm_page_t m;
@@ -1631,7 +2054,6 @@ pmap_collect(void)
 
        if (pmap_pagedaemon_waken == 0)
                return;
-       pmap_pagedaemon_waken = 0;
 
        if (warningdone < 5) {
                kprintf("pmap_collect: collecting pv entries -- suggest increasing PMAP_SHPGPERPROC\n");
@@ -1645,6 +2067,7 @@ pmap_collect(void)
                        continue;
                pmap_remove_all(m);
        }
+       pmap_pagedaemon_waken = 0;
 }
        
 
@@ -1657,6 +2080,7 @@ pmap_collect(void)
 static int
 pmap_remove_entry(struct pmap *pmap, vm_page_t m, 
                        vm_offset_t va, pmap_inval_info_t info)
+READY1
 {
        pv_entry_t pv;
        int rtval;
@@ -1675,9 +2099,11 @@ pmap_remove_entry(struct pmap *pmap, vm_page_t m,
        }
 
        rtval = 0;
+       /* JGXXX When can 'pv' be NULL? */
        if (pv) {
                TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
                m->md.pv_list_count--;
+               KKASSERT(m->md.pv_list_count >= 0);
                if (TAILQ_EMPTY(&m->md.pv_list))
                        vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE);
                TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
@@ -1695,6 +2121,7 @@ pmap_remove_entry(struct pmap *pmap, vm_page_t m,
  */
 static void
 pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t mpte, vm_page_t m)
+READY1
 {
        pv_entry_t pv;
 
@@ -1717,6 +2144,7 @@ pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t mpte, vm_page_t m)
 static int
 pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va,
        pmap_inval_info_t info)
+READY1
 {
        pt_entry_t oldpte;
        vm_page_t m;
@@ -1740,7 +2168,7 @@ pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va,
 #if defined(PMAP_DIAGNOSTIC)
                        if (pmap_nw_modified((pt_entry_t) oldpte)) {
                                kprintf(
-       "pmap_remove: modified page not writable: va: 0x%x, pte: 0x%x\n",
+       "pmap_remove: modified page not writable: va: 0x%lx, pte: 0x%lx\n",
                                    va, oldpte);
                        }
 #endif
@@ -1767,19 +2195,16 @@ pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va,
  */
 static void
 pmap_remove_page(struct pmap *pmap, vm_offset_t va, pmap_inval_info_t info)
+READY1
 {
-       pt_entry_t *ptq;
+       pt_entry_t *pte;
 
-       /*
-        * if there is no pte for this address, just skip it!!!  Otherwise
-        * get a local va for mappings for this pmap and remove the entry.
-        */
-       if (*pmap_pde(pmap, va) != 0) {
-               ptq = get_ptbase(pmap) + amd64_btop(va);
-               if (*ptq) {
-                       pmap_remove_pte(pmap, ptq, va, info);
-               }
-       }
+       pte = pmap_pte(pmap, va);
+       if (pte == NULL)
+               return;
+       if ((*pte & PG_V) == 0)
+               return;
+       pmap_remove_pte(pmap, pte, va, info);
 }
 
 /*
@@ -1795,11 +2220,13 @@ pmap_remove_page(struct pmap *pmap, vm_offset_t va, pmap_inval_info_t info)
  */
 void
 pmap_remove(struct pmap *pmap, vm_offset_t sva, vm_offset_t eva)
+READY1
 {
-       pt_entry_t *ptbase;
-       vm_offset_t pdnxt;
-       vm_offset_t ptpaddr;
-       vm_offset_t sindex, eindex;
+       vm_offset_t va_next;
+       pml4_entry_t *pml4e;
+       pdp_entry_t *pdpe;
+       pd_entry_t ptpaddr, *pde;
+       pt_entry_t *pte;
        struct pmap_inval_info info;
 
        if (pmap == NULL)
@@ -1815,65 +2242,75 @@ pmap_remove(struct pmap *pmap, vm_offset_t sva, vm_offset_t eva)
         * common operation and easy to short circuit some
         * code.
         */
-       if (((sva + PAGE_SIZE) == eva) && 
-               ((pmap->pm_pdir[(sva >> PDRSHIFT)] & PG_PS) == 0)) {
-               pmap_remove_page(pmap, sva, &info);
-               pmap_inval_flush(&info);
-               return;
+       if (sva + PAGE_SIZE == eva) {
+               pde = pmap_pde(pmap, sva);
+               if (pde && (*pde & PG_PS) == 0) {
+                       pmap_remove_page(pmap, sva, &info);
+                       pmap_inval_flush(&info);
+                       return;
+               }
        }
 
-       /*
-        * Get a local virtual address for the mappings that are being
-        * worked with.
-        */
-       sindex = amd64_btop(sva);
-       eindex = amd64_btop(eva);
+       for (; sva < eva; sva = va_next) {
+               pml4e = pmap_pml4e(pmap, sva);
+               if ((*pml4e & PG_V) == 0) {
+                       va_next = (sva + NBPML4) & ~PML4MASK;
+                       if (va_next < sva)
+                               va_next = eva;
+                       continue;
+               }
 
-       for (; sindex < eindex; sindex = pdnxt) {
-               vm_pindex_t pdirindex;
+               pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
+               if ((*pdpe & PG_V) == 0) {
+                       va_next = (sva + NBPDP) & ~PDPMASK;
+                       if (va_next < sva)
+                               va_next = eva;
+                       continue;
+               }
 
                /*
                 * Calculate index for next page table.
                 */
-               pdnxt = ((sindex + NPTEPG) & ~(NPTEPG - 1));
-               if (pmap->pm_stats.resident_count == 0)
-                       break;
+               va_next = (sva + NBPDR) & ~PDRMASK;
+               if (va_next < sva)
+                       va_next = eva;
 
-               pdirindex = sindex / NPDEPG;
-               if (((ptpaddr = pmap->pm_pdir[pdirindex]) & PG_PS) != 0) {
-                       pmap_inval_add(&info, pmap, -1);
-                       pmap->pm_pdir[pdirindex] = 0;
-                       pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
-                       continue;
-               }
+               pde = pmap_pdpe_to_pde(pdpe, sva);
+               ptpaddr = *pde;
 
                /*
-                * Weed out invalid mappings. Note: we assume that the page
-                * directory table is always allocated, and in kernel virtual.
+                * Weed out invalid mappings.
                 */
                if (ptpaddr == 0)
                        continue;
 
                /*
+                * Check for large page.
+                */
+               if ((ptpaddr & PG_PS) != 0) {
+                       /* JG FreeBSD has more complex treatment here */
+                       pmap_inval_add(&info, pmap, -1);
+                       *pde = 0;
+                       pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
+                       continue;
+               }
+
+               /*
                 * Limit our scan to either the end of the va represented
                 * by the current page table page, or to the end of the
                 * range being removed.
                 */
-               if (pdnxt > eindex) {
-                       pdnxt = eindex;
-               }
+               if (va_next > eva)
+                       va_next = eva;
 
                /*
                 * NOTE: pmap_remove_pte() can block.
                 */
-               for (; sindex != pdnxt; sindex++) {
-                       vm_offset_t va;
-
-                       ptbase = get_ptbase(pmap);
-                       if (ptbase[sindex] == 0)
+               for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
+                   sva += PAGE_SIZE) {
+                       if (*pte == 0)
                                continue;
-                       va = amd64_ptob(sindex);
-                       if (pmap_remove_pte(pmap, ptbase + sindex, va, &info))
+                       if (pmap_remove_pte(pmap, pte, sva, &info))
                                break;
                }
        }
@@ -1891,6 +2328,7 @@ pmap_remove(struct pmap *pmap, vm_offset_t sva, vm_offset_t eva)
 
 static void
 pmap_remove_all(vm_page_t m)
+READY1
 {
        struct pmap_inval_info info;
        pt_entry_t *pte, tpte;
@@ -1920,9 +2358,9 @@ pmap_remove_all(vm_page_t m)
                 */
                if (tpte & PG_M) {
 #if defined(PMAP_DIAGNOSTIC)
-                       if (pmap_nw_modified((pt_entry_t) tpte)) {
+                       if (pmap_nw_modified(tpte)) {
                                kprintf(
-       "pmap_remove_all: modified page not writable: va: 0x%x, pte: 0x%x\n",
+       "pmap_remove_all: modified page not writable: va: 0x%lx, pte: 0x%lx\n",
                                    pv->pv_va, tpte);
                        }
 #endif
@@ -1933,6 +2371,7 @@ pmap_remove_all(vm_page_t m)
                TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist);
                ++pv->pv_pmap->pm_generation;
                m->md.pv_list_count--;
+               KKASSERT(m->md.pv_list_count >= 0);
                if (TAILQ_EMPTY(&m->md.pv_list))
                        vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE);
                pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem, &info);
@@ -1954,12 +2393,17 @@ pmap_remove_all(vm_page_t m)
  */
 void
 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
+READY1
 {
-       pt_entry_t *ptbase;
-       vm_offset_t pdnxt, ptpaddr;
-       vm_pindex_t sindex, eindex;
+       vm_offset_t va_next;
+       pml4_entry_t *pml4e;
+       pdp_entry_t *pdpe;
+       pd_entry_t ptpaddr, *pde;
+       pt_entry_t *pte;
        pmap_inval_info info;
 
+       /* JG review for NX */
+
        if (pmap == NULL)
                return;
 
@@ -1973,21 +2417,37 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 
        pmap_inval_init(&info);
 
-       ptbase = get_ptbase(pmap);
+       for (; sva < eva; sva = va_next) {
 
-       sindex = amd64_btop(sva);
-       eindex = amd64_btop(eva);
+               pml4e = pmap_pml4e(pmap, sva);
+               if ((*pml4e & PG_V) == 0) {
+                       va_next = (sva + NBPML4) & ~PML4MASK;
+                       if (va_next < sva)
+                               va_next = eva;
+                       continue;
+               }
 
-       for (; sindex < eindex; sindex = pdnxt) {
+               pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
+               if ((*pdpe & PG_V) == 0) {
+                       va_next = (sva + NBPDP) & ~PDPMASK;
+                       if (va_next < sva)
+                               va_next = eva;
+                       continue;
+               }
 
-               vm_pindex_t pdirindex;
+               va_next = (sva + NBPDR) & ~PDRMASK;
+               if (va_next < sva)
+                       va_next = eva;
 
-               pdnxt = ((sindex + NPTEPG) & ~(NPTEPG - 1));
+               pde = pmap_pdpe_to_pde(pdpe, sva);
+               ptpaddr = *pde;
 
-               pdirindex = sindex / NPDEPG;
-               if (((ptpaddr = pmap->pm_pdir[pdirindex]) & PG_PS) != 0) {
+               /*
+                * Check for large page.
+                */
+               if ((ptpaddr & PG_PS) != 0) {
                        pmap_inval_add(&info, pmap, -1);
-                       pmap->pm_pdir[pdirindex] &= ~(PG_M|PG_RW);
+                       *pde &= ~(PG_M|PG_RW);
                        pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
                        continue;
                }
@@ -1999,13 +2459,12 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
                if (ptpaddr == 0)
                        continue;
 
-               if (pdnxt > eindex) {
-                       pdnxt = eindex;
-               }
+               if (va_next > eva)
+                       va_next = eva;
 
-               for (; sindex != pdnxt; sindex++) {
-
-                       pt_entry_t pbits;
+               for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
+                   sva += PAGE_SIZE) {
+                       pt_entry_t obits, pbits;
                        vm_page_t m;
 
                        /*
@@ -2014,19 +2473,21 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
                         * ptbase[sindex] (or otherwise we have to do another
                         * pmap_inval_add() call).
                         */
-                       pmap_inval_add(&info, pmap, amd64_ptob(sindex));
-                       pbits = ptbase[sindex];
-
+                       pmap_inval_add(&info, pmap, sva);
+                       obits = pbits = *pte;
+                       if ((pbits & PG_V) == 0)
+                               continue;
                        if (pbits & PG_MANAGED) {
                                m = NULL;
                                if (pbits & PG_A) {
-                                       m = PHYS_TO_VM_PAGE(pbits);
+                                       m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
                                        vm_page_flag_set(m, PG_REFERENCED);
                                        pbits &= ~PG_A;
                                }
                                if (pbits & PG_M) {
-                                       if (pmap_track_modified(amd64_ptob(sindex))) {
+                                       if (pmap_track_modified(sva)) {
                                                if (m == NULL)
+                                                       KKASSERT(pbits == (pbits & PG_FRAME));
                                                        m = PHYS_TO_VM_PAGE(pbits);
                                                vm_page_dirty(m);
                                                pbits &= ~PG_M;
@@ -2036,8 +2497,8 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 
                        pbits &= ~PG_RW;
 
-                       if (pbits != ptbase[sindex]) {
-                               ptbase[sindex] = pbits;
+                       if (pbits != obits) {
+                               *pte = pbits;
                        }
                }
        }
@@ -2059,60 +2520,63 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 void
 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
           boolean_t wired)
+READY1
 {
        vm_paddr_t pa;
+       pd_entry_t *pde;
        pt_entry_t *pte;
        vm_paddr_t opa;
-       vm_offset_t origpte, newpte;
+       pt_entry_t origpte, newpte;
        vm_page_t mpte;
        pmap_inval_info info;
 
        if (pmap == NULL)
                return;
 
-       va &= PG_FRAME;
+       va = trunc_page(va);
 #ifdef PMAP_DIAGNOSTIC
        if (va >= KvaEnd)
                panic("pmap_enter: toobig");
        if ((va >= UPT_MIN_ADDRESS) && (va < UPT_MAX_ADDRESS))
-               panic("pmap_enter: invalid to pmap_enter page table pages (va: 0x%x)", va);
+               panic("pmap_enter: invalid to pmap_enter page table pages (va: 0x%lx)", va);
 #endif
        if (va < UPT_MAX_ADDRESS && pmap == &kernel_pmap) {
                kprintf("Warning: pmap_enter called on UVA with kernel_pmap\n");
-               print_backtrace();
+#ifdef DDB
+               db_print_backtrace();
+#endif
        }
        if (va >= UPT_MAX_ADDRESS && pmap != &kernel_pmap) {
                kprintf("Warning: pmap_enter called on KVA without kernel_pmap\n");
-               print_backtrace();
+#ifdef DDB
+               db_print_backtrace();
+#endif
        }
 
        /*
         * In the case that a page table page is not
         * resident, we are creating it here.
         */
-       if (va < UPT_MIN_ADDRESS)
+       if (va < VM_MAX_USER_ADDRESS)
                mpte = pmap_allocpte(pmap, va);
        else
                mpte = NULL;
 
        pmap_inval_init(&info);
-       pte = pmap_pte(pmap, va);
-
-       /*
-        * Page Directory table entry not valid, we need a new PT page
-        */
-       if (pte == NULL) {
-               panic("pmap_enter: invalid page directory pdir=%x, va=0x%x\n",
-                    pmap->pm_pdir[PTDPTDI], va);
-       }
-
-       pa = VM_PAGE_TO_PHYS(m) & PG_FRAME;
-       origpte = *(vm_offset_t *)pte;
+       pde = pmap_pde(pmap, va);
+       if (pde != NULL && (*pde & PG_V) != 0) {
+               if ((*pde & PG_PS) != 0)
+                       panic("pmap_enter: attempted pmap_enter on 2MB page");
+               pte = pmap_pde_to_pte(pde, va);
+       } else
+               panic("pmap_enter: invalid page directory va=%#lx", va);
+
+       KKASSERT(pte != NULL);
+       pa = VM_PAGE_TO_PHYS(m);
+       KKASSERT(pa == (pa & PG_FRAME));
+       origpte = *pte;
        opa = origpte & PG_FRAME;
 
-       if (origpte & PG_PS)
-               panic("pmap_enter: attempted pmap_enter on 4MB page");
-
        /*
         * Mapping has not changed, must be protection or wiring change.
         */
@@ -2129,9 +2593,9 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
                        pmap->pm_stats.wired_count--;
 
 #if defined(PMAP_DIAGNOSTIC)
-               if (pmap_nw_modified((pt_entry_t) origpte)) {
+               if (pmap_nw_modified(origpte)) {
                        kprintf(
-       "pmap_enter: modified page not writable: va: 0x%x, pte: 0x%x\n",
+       "pmap_enter: modified page not writable: va: 0x%lx, pte: 0x%lx\n",
                            va, origpte);
                }
 #endif
@@ -2168,7 +2632,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
                int err;
                err = pmap_remove_pte(pmap, pte, va, &info);
                if (err)
-                       panic("pmap_enter: pte vanished, va: 0x%x", va);
+                       panic("pmap_enter: pte vanished, va: 0x%lx", va);
        }
 
        /*
@@ -2194,11 +2658,11 @@ validate:
        /*
         * Now validate mapping with desired protection/wiring.
         */
-       newpte = (vm_offset_t) (pa | pte_prot(pmap, prot) | PG_V);
+       newpte = (pt_entry_t) (pa | pte_prot(pmap, prot) | PG_V);
 
        if (wired)
                newpte |= PG_W;
-       if (va < UPT_MIN_ADDRESS)
+       if (va < VM_MAX_USER_ADDRESS)
                newpte |= PG_U;
        if (pmap == &kernel_pmap)
                newpte |= pgeflag;
@@ -2226,23 +2690,28 @@ validate:
  */
 static void
 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m)
+READY1
 {
        pt_entry_t *pte;
        vm_paddr_t pa;
        vm_page_t mpte;
        vm_pindex_t ptepindex;
-       vm_offset_t ptepa;
+       pd_entry_t *ptepa;
        pmap_inval_info info;
 
        pmap_inval_init(&info);
 
        if (va < UPT_MAX_ADDRESS && pmap == &kernel_pmap) {
                kprintf("Warning: pmap_enter_quick called on UVA with kernel_pmap\n");
-               print_backtrace();
+#ifdef DDB
+               db_print_backtrace();
+#endif
        }
        if (va >= UPT_MAX_ADDRESS && pmap != &kernel_pmap) {
                kprintf("Warning: pmap_enter_quick called on KVA without kernel_pmap\n");
-               print_backtrace();
+#ifdef DDB
+               db_print_backtrace();
+#endif
        }
 
        KKASSERT(va < UPT_MIN_ADDRESS); /* assert used on user pmaps only */
@@ -2253,32 +2722,32 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m)
         * A held page table page (mpte), or NULL, is passed onto the
         * section following.
         */
-       if (va < UPT_MIN_ADDRESS) {
+       if (va < VM_MAX_USER_ADDRESS) {
                /*
                 * Calculate pagetable page index
                 */
-               ptepindex = va >> PDRSHIFT;
+               ptepindex = pmap_pde_pindex(va);
 
                do {
                        /*
                         * Get the page directory entry
                         */
-                       ptepa = (vm_offset_t) pmap->pm_pdir[ptepindex];
+                       ptepa = pmap_pde(pmap, va);
 
                        /*
                         * If the page table page is mapped, we just increment
                         * the hold count, and activate it.
                         */
-                       if (ptepa) {
-                               if (ptepa & PG_PS)
-                                       panic("pmap_enter_quick: unexpected mapping into 4MB page");
-                               if (pmap->pm_ptphint &&
-                                   (pmap->pm_ptphint->pindex == ptepindex)) {
-                                       mpte = pmap->pm_ptphint;
-                               } else {
+                       if (ptepa && (*ptepa & PG_V) != 0) {
+                               if (*ptepa & PG_PS)
+                                       panic("pmap_enter_quick: unexpected mapping into 2MB page");
+//                             if (pmap->pm_ptphint &&
+//                                 (pmap->pm_ptphint->pindex == ptepindex)) {
+//                                     mpte = pmap->pm_ptphint;
+//                             } else {
                                        mpte = pmap_page_lookup( pmap->pm_pteobj, ptepindex);
                                        pmap->pm_ptphint = mpte;
-                               }
+//                             }
                                if (mpte)
                                        mpte->hold_count++;
                        } else {
@@ -2298,7 +2767,7 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m)
        pte = vtopte(va);
        if (*pte & PG_V) {
                if (mpte)
-                       pmap_unwire_pte_hold(pmap, mpte, &info);
+                       pmap_unwire_pte_hold(pmap, va, mpte, &info);
                pa = VM_PAGE_TO_PHYS(m);
                KKASSERT(((*pte ^ pa) & PG_FRAME) == 0);
                return;
@@ -2334,8 +2803,10 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m)
  * Make a temporary mapping for a physical address.  This is only intended
  * to be used for panic dumps.
  */
+/* JG Needed on amd64? */
 void *
 pmap_kenter_temporary(vm_paddr_t pa, int i)
+READY2
 {
        pmap_kenter((vm_offset_t)crashdumpmap + (i * PAGE_SIZE), pa);
        return ((void *)crashdumpmap);
@@ -2354,10 +2825,11 @@ void
 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_prot_t prot,
                    vm_object_t object, vm_pindex_t pindex, 
                    vm_size_t size, int limit)
+READY1
 {
        struct rb_vm_page_scan_info info;
        struct lwp *lp;
-       int psize;
+       vm_size_t psize;
 
        /*
         * We can't preinit if read access isn't set or there is no pmap
@@ -2413,6 +2885,7 @@ pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_prot_t prot,
 static
 int
 pmap_object_init_pt_callback(vm_page_t p, void *data)
+READY1
 {
        struct rb_vm_page_scan_info *info = data;
        vm_pindex_t rel_index;
@@ -2455,6 +2928,7 @@ static int pmap_prefault_pageorder[] = {
 
 void
 pmap_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry)
+READY0
 {
        int i;
        vm_offset_t starta;
@@ -2550,6 +3024,7 @@ pmap_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry)
  */
 void
 pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired)
+READY0
 {
        pt_entry_t *pte;
 
@@ -2595,6 +3070,7 @@ pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired)
 void
 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 
        vm_size_t len, vm_offset_t src_addr)
+READY0
 {
        pmap_inval_info info;
        vm_offset_t addr;
@@ -2614,6 +3090,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
         */
        return;
 
+#if JGPMAP32
        src_frame = src_pmap->pm_pdir[PTDPTDI] & PG_FRAME;
        if (src_frame != (PTDpde & PG_FRAME)) {
                return;
@@ -2625,6 +3102,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
                /* The page directory is not shared between CPUs */
                cpu_invltlb();
        }
+#endif
        pmap_inval_init(&info);
        pmap_inval_add(&info, dst_pmap, -1);
        pmap_inval_add(&info, src_pmap, -1);
@@ -2656,15 +3134,19 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
                pdnxt = ((addr + PAGE_SIZE*NPTEPG) & ~(PAGE_SIZE*NPTEPG - 1));
                ptepindex = addr >> PDRSHIFT;
 
+#if JGPMAP32
                srcptepaddr = (vm_offset_t) src_pmap->pm_pdir[ptepindex];
+#endif
                if (srcptepaddr == 0)
                        continue;
                        
                if (srcptepaddr & PG_PS) {
+#if JGPMAP32
                        if (dst_pmap->pm_pdir[ptepindex] == 0) {
                                dst_pmap->pm_pdir[ptepindex] = (pd_entry_t) srcptepaddr;
                                dst_pmap->pm_stats.resident_count += NBPDR / PAGE_SIZE;
                        }
+#endif
                        continue;
                }
 
@@ -2678,7 +3160,9 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
                        pdnxt = end_addr;
 
                src_pte = vtopte(addr);
+#if JGPMAP32
                dst_pte = avtopte(addr);
+#endif
                while (addr < pdnxt) {
                        pt_entry_t ptetemp;
 
@@ -2697,6 +3181,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
                                 */
                                dstmpte = pmap_allocpte(dst_pmap, addr);
 
+#if JGPMAP32
                                if (src_frame != (PTDpde & PG_FRAME) ||
                                    dst_frame != (APTDpde & PG_FRAME)
                                ) {
@@ -2722,6 +3207,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
                                        pmap_unwire_pte_hold(dst_pmap, dstmpte, &info);
                                        goto failed;
                                }
+#endif
                                if (dstmpte->hold_count >= srcmpte->hold_count)
                                        break;
                        }
@@ -2738,32 +3224,18 @@ failed:
 /*
  * pmap_zero_page:
  *
- *     Zero the specified PA by mapping the page into KVM and clearing its
- *     contents.
+ *     Zero the specified physical page.
  *
  *     This function may be called from an interrupt and no locking is
  *     required.
  */
 void
 pmap_zero_page(vm_paddr_t phys)
+READY1
 {
-       struct mdglobaldata *gd = mdcpu;
+       vm_offset_t va = PHYS_TO_DMAP(phys);
 
-       crit_enter();
-       if (*gd->gd_CMAP3)
-               panic("pmap_zero_page: CMAP3 busy");
-       *gd->gd_CMAP3 =
-                   PG_V | PG_RW | (phys & PG_FRAME) | PG_A | PG_M;
-       cpu_invlpg(gd->gd_CADDR3);
-
-#if defined(I686_CPU)
-       if (cpu_class == CPUCLASS_686)
-               i686_pagezero(gd->gd_CADDR3);
-       else
-#endif
-               bzero(gd->gd_CADDR3, PAGE_SIZE);
-       *gd->gd_CMAP3 = 0;
-       crit_exit();
+       pagezero((void *)va);
 }
 
 /*
@@ -2773,23 +3245,20 @@ pmap_zero_page(vm_paddr_t phys)
  */
 void
 pmap_page_assertzero(vm_paddr_t phys)
+READY1
 {
        struct mdglobaldata *gd = mdcpu;
        int i;
 
        crit_enter();
-       if (*gd->gd_CMAP3)
-               panic("pmap_zero_page: CMAP3 busy");
-       *gd->gd_CMAP3 =
-                   PG_V | PG_RW | (phys & PG_FRAME) | PG_A | PG_M;
-       cpu_invlpg(gd->gd_CADDR3);
+       vm_offset_t virt = PHYS_TO_DMAP(phys);
+
        for (i = 0; i < PAGE_SIZE; i += sizeof(int)) {
-           if (*(int *)((char *)gd->gd_CADDR3 + i) != 0) {
+           if (*(int *)((char *)virt + i) != 0) {
                panic("pmap_page_assertzero() @ %p not zero!\n",
-                   (void *)gd->gd_CADDR3);
+                   (void *)virt);
            }
        }
-       *gd->gd_CMAP3 = 0;
        crit_exit();
 }
 
@@ -2803,22 +3272,13 @@ pmap_page_assertzero(vm_paddr_t phys)
  */
 void
 pmap_zero_page_area(vm_paddr_t phys, int off, int size)
+READY1
 {
        struct mdglobaldata *gd = mdcpu;
 
        crit_enter();
-       if (*gd->gd_CMAP3)
-               panic("pmap_zero_page: CMAP3 busy");
-       *gd->gd_CMAP3 = PG_V | PG_RW | (phys & PG_FRAME) | PG_A | PG_M;
-       cpu_invlpg(gd->gd_CADDR3);
-
-#if defined(I686_CPU)
-       if (cpu_class == CPUCLASS_686 && off == 0 && size == PAGE_SIZE)
-               i686_pagezero(gd->gd_CADDR3);
-       else
-#endif
-               bzero((char *)gd->gd_CADDR3 + off, size);
-       *gd->gd_CMAP3 = 0;
+       vm_offset_t virt = PHYS_TO_DMAP(phys);
+       bzero((char *)virt + off, size);
        crit_exit();
 }
 
@@ -2831,25 +3291,14 @@ pmap_zero_page_area(vm_paddr_t phys, int off, int size)
  */
 void
 pmap_copy_page(vm_paddr_t src, vm_paddr_t dst)
+READY1
 {
-       struct mdglobaldata *gd = mdcpu;
+       vm_offset_t src_virt, dst_virt;
 
        crit_enter();
-       if (*gd->gd_CMAP1)
-               panic("pmap_copy_page: CMAP1 busy");
-       if (*gd->gd_CMAP2)
-               panic("pmap_copy_page: CMAP2 busy");
-
-       *gd->gd_CMAP1 = PG_V | (src & PG_FRAME) | PG_A;
-       *gd->gd_CMAP2 = PG_V | PG_RW | (dst & PG_FRAME) | PG_A | PG_M;
-
-       cpu_invlpg(gd->gd_CADDR1);
-       cpu_invlpg(gd->gd_CADDR2);
-
-       bcopy(gd->gd_CADDR1, gd->gd_CADDR2, PAGE_SIZE);
-
-       *gd->gd_CMAP1 = 0;
-       *gd->gd_CMAP2 = 0;
+       src_virt = PHYS_TO_DMAP(src);
+       dst_virt = PHYS_TO_DMAP(dst);
+       bcopy(src_virt, dst_virt, PAGE_SIZE);
        crit_exit();
 }
 
@@ -2862,27 +3311,16 @@ pmap_copy_page(vm_paddr_t src, vm_paddr_t dst)
  */
 void
 pmap_copy_page_frag(vm_paddr_t src, vm_paddr_t dst, size_t bytes)
+READY1
 {
-       struct mdglobaldata *gd = mdcpu;
+       vm_offset_t src_virt, dst_virt;
 
        crit_enter();
-       if (*gd->gd_CMAP1)
-               panic("pmap_copy_page: CMAP1 busy");
-       if (*gd->gd_CMAP2)
-               panic("pmap_copy_page: CMAP2 busy");
-
-       *gd->gd_CMAP1 = PG_V | (src & PG_FRAME) | PG_A;
-       *gd->gd_CMAP2 = PG_V | PG_RW | (dst & PG_FRAME) | PG_A | PG_M;
-
-       cpu_invlpg(gd->gd_CADDR1);
-       cpu_invlpg(gd->gd_CADDR2);
-
-       bcopy((char *)gd->gd_CADDR1 + (src & PAGE_MASK),
-             (char *)gd->gd_CADDR2 + (dst & PAGE_MASK),
+       src_virt = PHYS_TO_DMAP(src);
+       dst_virt = PHYS_TO_DMAP(dst);
+       bcopy((char *)src_virt + (src & PAGE_MASK),
+             (char *)dst_virt + (dst & PAGE_MASK),
              bytes);
-
-       *gd->gd_CMAP1 = 0;
-       *gd->gd_CMAP2 = 0;
        crit_exit();
 }
 
@@ -2895,6 +3333,7 @@ pmap_copy_page_frag(vm_paddr_t src, vm_paddr_t dst, size_t bytes)
  */
 boolean_t
 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
+READY2
 {
        pv_entry_t pv;
        int loops = 0;
@@ -2927,6 +3366,7 @@ pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
  */
 void
 pmap_remove_pages(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
+READY1
 {
        struct lwp *lp;
        pt_entry_t *pte, tpte;
@@ -2934,7 +3374,7 @@ pmap_remove_pages(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
        vm_page_t m;
        pmap_inval_info info;
        int iscurrentpmap;
-       int32_t save_generation;
+       int save_generation;
 
        lp = curthread->td_lwp;
        if (lp && pmap == vmspace_pmap(lp->lwp_vmspace))
@@ -2969,10 +3409,10 @@ pmap_remove_pages(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
                }
                tpte = pte_load_clear(pte);
 
-               m = PHYS_TO_VM_PAGE(tpte);
+               m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
 
                KASSERT(m < &vm_page_array[vm_page_array_size],
-                       ("pmap_remove_pages: bad tpte %x", tpte));
+                       ("pmap_remove_pages: bad tpte %lx", tpte));
 
                KKASSERT(pmap->pm_stats.resident_count > 0);
                --pmap->pm_stats.resident_count;
@@ -3016,6 +3456,7 @@ pmap_remove_pages(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
  */
 static boolean_t
 pmap_testbit(vm_page_t m, int bit)
+READY1
 {
        pv_entry_t pv;
        pt_entry_t *pte;
@@ -3040,8 +3481,8 @@ pmap_testbit(vm_page_t m, int bit)
                }
 
 #if defined(PMAP_DIAGNOSTIC)
-               if (!pv->pv_pmap) {
-                       kprintf("Null pmap (tb) at va: 0x%x\n", pv->pv_va);
+               if (pv->pv_pmap == NULL) {
+                       kprintf("Null pmap (tb) at va: 0x%lx\n", pv->pv_va);
                        continue;
                }
 #endif
@@ -3060,6 +3501,7 @@ pmap_testbit(vm_page_t m, int bit)
  */
 static __inline void
 pmap_clearbit(vm_page_t m, int bit)
+READY1
 {
        struct pmap_inval_info info;
        pv_entry_t pv;
@@ -3086,8 +3528,8 @@ pmap_clearbit(vm_page_t m, int bit)
                }
 
 #if defined(PMAP_DIAGNOSTIC)
-               if (!pv->pv_pmap) {
-                       kprintf("Null pmap (cb) at va: 0x%x\n", pv->pv_va);
+               if (pv->pv_pmap == NULL) {
+                       kprintf("Null pmap (cb) at va: 0x%lx\n", pv->pv_va);
                        continue;
                }
 #endif
@@ -3111,14 +3553,14 @@ again:
                        if (bit == PG_RW) {
                                if (pbits & PG_M) {
                                        vm_page_dirty(m);
-                                       atomic_clear_int(pte, PG_M|PG_RW);
+                                       atomic_clear_long(pte, PG_M|PG_RW);
                                } else {
                                        /*
                                         * The cpu may be trying to set PG_M
                                         * simultaniously with our clearing
                                         * of PG_RW.
                                         */
-                                       if (!atomic_cmpset_int(pte, pbits,
+                                       if (!atomic_cmpset_long(pte, pbits,
                                                               pbits & ~PG_RW))
                                                goto again;
                                }
@@ -3131,9 +3573,9 @@ again:
                                 * when they clear the VPTE_M bit in their
                                 * virtual page tables.
                                 */
-                               atomic_clear_int(pte, PG_M);
+                               atomic_clear_long(pte, PG_M);
                        } else {
-                               atomic_clear_int(pte, bit);
+                               atomic_clear_long(pte, bit);
                        }
                }
        }
@@ -3148,7 +3590,9 @@ again:
  */
 void
 pmap_page_protect(vm_page_t m, vm_prot_t prot)
+READY1
 {
+       /* JG NX support? */
        if ((prot & VM_PROT_WRITE) == 0) {
                if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) {
                        pmap_clearbit(m, PG_RW);
@@ -3161,6 +3605,7 @@ pmap_page_protect(vm_page_t m, vm_prot_t prot)
 
 vm_paddr_t
 pmap_phys_address(vm_pindex_t ppn)
+READY2
 {
        return (amd64_ptob(ppn));
 }
@@ -3179,6 +3624,7 @@ pmap_phys_address(vm_pindex_t ppn)
  */
 int
 pmap_ts_referenced(vm_page_t m)
+READY1
 {
        pv_entry_t pv, pvf, pvn;
        pt_entry_t *pte;
@@ -3207,9 +3653,9 @@ pmap_ts_referenced(vm_page_t m)
 
                        if (pte && (*pte & PG_A)) {
 #ifdef SMP
-                               atomic_clear_int(pte, PG_A);
+                               atomic_clear_long(pte, PG_A);
 #else
-                               atomic_clear_int_nonlocked(pte, PG_A);
+                               atomic_clear_long_nonlocked(pte, PG_A);
 #endif
                                rtval++;
                                if (rtval > 4) {
@@ -3231,6 +3677,7 @@ pmap_ts_referenced(vm_page_t m)
  */
 boolean_t
 pmap_is_modified(vm_page_t m)
+READY2
 {
        return pmap_testbit(m, PG_M);
 }
@@ -3240,6 +3687,7 @@ pmap_is_modified(vm_page_t m)
  */
 void
 pmap_clear_modify(vm_page_t m)
+READY2
 {
        pmap_clearbit(m, PG_M);
 }
@@ -3251,6 +3699,7 @@ pmap_clear_modify(vm_page_t m)
  */
 void
 pmap_clear_reference(vm_page_t m)
+READY2
 {
        pmap_clearbit(m, PG_A);
 }
@@ -3261,9 +3710,11 @@ pmap_clear_reference(vm_page_t m)
 
 static void
 i386_protection_init(void)
+READY0
 {
        int *kp, prot;
 
+       /* JG NX support may go here; No VM_PROT_EXECUTE ==> set NX bit  */
        kp = protection_codes;
        for (prot = 0; prot < 8; prot++) {
                switch (prot) {
@@ -3298,6 +3749,7 @@ i386_protection_init(void)
  */
 void *
 pmap_mapdev(vm_paddr_t pa, vm_size_t size)
+READY1
 {
        vm_offset_t va, tmpva, offset;
        pt_entry_t *pte;
@@ -3306,10 +3758,10 @@ pmap_mapdev(vm_paddr_t pa, vm_size_t size)
        size = roundup(offset + size, PAGE_SIZE);
 
        va = kmem_alloc_nofault(&kernel_map, size);
-       if (!va)
+       if (va == 0)
                panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
 
-       pa = pa & PG_FRAME;
+       pa = pa & ~PAGE_MASK;
        for (tmpva = va; size > 0;) {
                pte = vtopte(tmpva);
                *pte = pa | PG_RW | PG_V; /* | pgeflag; */
@@ -3325,10 +3777,11 @@ pmap_mapdev(vm_paddr_t pa, vm_size_t size)
 
 void
 pmap_unmapdev(vm_offset_t va, vm_size_t size)
+READY1
 {
        vm_offset_t base, offset;
 
-       base = va & PG_FRAME;
+       base = va & ~PAGE_MASK;
        offset = va & PAGE_MASK;
        size = roundup(offset + size, PAGE_SIZE);
        pmap_qremove(va, size >> PAGE_SHIFT);
@@ -3340,6 +3793,7 @@ pmap_unmapdev(vm_offset_t va, vm_size_t size)
  */
 int
 pmap_mincore(pmap_t pmap, vm_offset_t addr)
+READY0
 {
        pt_entry_t *ptep, pte;
        vm_page_t m;
@@ -3397,6 +3851,7 @@ pmap_mincore(pmap_t pmap, vm_offset_t addr)
  */
 void
 pmap_replacevm(struct proc *p, struct vmspace *newvm, int adjrefs)
+READY2
 {
        struct vmspace *oldvm;
        struct lwp *lp;
@@ -3423,6 +3878,7 @@ pmap_replacevm(struct proc *p, struct vmspace *newvm, int adjrefs)
  */
 void
 pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm)
+READY1
 {
        struct vmspace *oldvm;
        struct pmap *pmap;
@@ -3442,10 +3898,8 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm)
 #if defined(SWTCH_OPTIM_STATS)
                        tlb_flush_count++;
 #endif
-                       curthread->td_pcb->pcb_cr3 = vtophys(pmap->pm_pdir);
-                       curthread->td_pcb->pcb_cr3 |= PG_RW | PG_U | PG_V;
-                       *link_pdpe = curthread->td_pcb->pcb_cr3 | PG_RW | PG_U | PG_V;
-                       load_cr3(common_lvl4_phys);
+                       curthread->td_pcb->pcb_cr3 = vtophys(pmap->pm_pml4);
+                       load_cr3(curthread->td_pcb->pcb_cr3);
                        pmap = vmspace_pmap(oldvm);
 #if defined(SMP)
                        atomic_clear_int(&pmap->pm_active,
@@ -3460,6 +3914,7 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm)
 
 vm_offset_t
 pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size)
+READY0
 {
 
        if ((obj == NULL) || (size < NBPDR) || (obj->type != OBJT_DEVICE)) {
@@ -3479,6 +3934,7 @@ void              pmap_pvdump (vm_paddr_t pa);
 /* print address space of pmap*/
 static void
 pads(pmap_t pm)
+READY0
 {
        vm_offset_t va;
        unsigned i, j;
@@ -3488,6 +3944,7 @@ pads(pmap_t pm)
                return;
        crit_enter();
        for (i = 0; i < NPDEPG; i++) {
+#if JGPMAP32
                if (pm->pm_pdir[i]) {
                        for (j = 0; j < NPTEPG; j++) {
                                va = (i << PDRSHIFT) + (j << PAGE_SHIFT);
@@ -3500,6 +3957,7 @@ pads(pmap_t pm)
                                        kprintf("%lx:%lx ", va, *ptep);
                        };
                }
+#endif
        }
        crit_exit();
 
@@ -3507,6 +3965,7 @@ pads(pmap_t pm)
 
 void
 pmap_pvdump(vm_paddr_t pa)
+READY0
 {
        pv_entry_t pv;
        vm_page_t m;
index 6995903..8c61fb0 100644 (file)
@@ -248,12 +248,12 @@ ENTRY(copyout)
        addq    %rdx,%rax
        jc      copyout_fault
 /*
- * XXX STOP USING VM_MAXUSER_ADDRESS.
+ * XXX STOP USING VM_MAX_USER_ADDRESS.
  * It is an end address, not a max, so every time it is used correctly it
  * looks like there is an off by one error, and of course it caused an off
  * by one error in several places.
  */
-       movq    $VM_MAXUSER_ADDRESS,%rcx
+       movq    $VM_MAX_USER_ADDRESS,%rcx
        cmpq    %rcx,%rax
        ja      copyout_fault
 
@@ -302,7 +302,7 @@ ENTRY(copyin)
        movq    %rdi,%rax
        addq    %rdx,%rax
        jc      copyin_fault
-       movq    $VM_MAXUSER_ADDRESS,%rcx
+       movq    $VM_MAX_USER_ADDRESS,%rcx
        cmpq    %rcx,%rax
        ja      copyin_fault
 
@@ -342,7 +342,7 @@ ENTRY(casuword32)
        movq    TD_PCB(%rcx), %rcx
        movq    $fusufault,PCB_ONFAULT(%rcx)
 
-       movq    $VM_MAXUSER_ADDRESS-4,%rax
+       movq    $VM_MAX_USER_ADDRESS-4,%rax
        cmpq    %rax,%rdi                       /* verify address is valid */
        ja      fusufault
 
@@ -372,7 +372,7 @@ ENTRY(casuword)
        movq    TD_PCB(%rcx), %rcx
        movq    $fusufault,PCB_ONFAULT(%rcx)
 
-       movq    $VM_MAXUSER_ADDRESS-4,%rax
+       movq    $VM_MAX_USER_ADDRESS-4,%rax
        cmpq    %rax,%rdi                       /* verify address is valid */
        ja      fusufault
 
@@ -406,7 +406,7 @@ ENTRY(fuword)
        movq    TD_PCB(%rcx), %rcx
        movq    $fusufault,PCB_ONFAULT(%rcx)
 
-       movq    $VM_MAXUSER_ADDRESS-8,%rax
+       movq    $VM_MAX_USER_ADDRESS-8,%rax
        cmpq    %rax,%rdi                       /* verify address is valid */
        ja      fusufault
 
@@ -419,7 +419,7 @@ ENTRY(fuword32)
        movq    TD_PCB(%rcx), %rcx
        movq    $fusufault,PCB_ONFAULT(%rcx)
 
-       movq    $VM_MAXUSER_ADDRESS-4,%rax
+       movq    $VM_MAX_USER_ADDRESS-4,%rax
        cmpq    %rax,%rdi                       /* verify address is valid */
        ja      fusufault
 
@@ -444,7 +444,7 @@ ENTRY(fuword16)
        movq    TD_PCB(%rcx), %rcx
        movq    $fusufault,PCB_ONFAULT(%rcx)
 
-       movq    $VM_MAXUSER_ADDRESS-2,%rax
+       movq    $VM_MAX_USER_ADDRESS-2,%rax
        cmpq    %rax,%rdi
        ja      fusufault
 
@@ -457,7 +457,7 @@ ENTRY(fubyte)
        movq    TD_PCB(%rcx), %rcx
        movq    $fusufault,PCB_ONFAULT(%rcx)
 
-       movq    $VM_MAXUSER_ADDRESS-1,%rax
+       movq    $VM_MAX_USER_ADDRESS-1,%rax
        cmpq    %rax,%rdi
        ja      fusufault
 
@@ -485,7 +485,7 @@ ENTRY(suword)
        movq    TD_PCB(%rcx), %rcx
        movq    $fusufault,PCB_ONFAULT(%rcx)
 
-       movq    $VM_MAXUSER_ADDRESS-8,%rax
+       movq    $VM_MAX_USER_ADDRESS-8,%rax
        cmpq    %rax,%rdi                       /* verify address validity */
        ja      fusufault
 
@@ -501,7 +501,7 @@ ENTRY(suword32)
        movq    TD_PCB(%rcx), %rcx
        movq    $fusufault,PCB_ONFAULT(%rcx)
 
-       movq    $VM_MAXUSER_ADDRESS-4,%rax
+       movq    $VM_MAX_USER_ADDRESS-4,%rax
        cmpq    %rax,%rdi                       /* verify address validity */
        ja      fusufault
 
@@ -517,7 +517,7 @@ ENTRY(suword16)
        movq    TD_PCB(%rcx), %rcx
        movq    $fusufault,PCB_ONFAULT(%rcx)
 
-       movq    $VM_MAXUSER_ADDRESS-2,%rax
+       movq    $VM_MAX_USER_ADDRESS-2,%rax
        cmpq    %rax,%rdi                       /* verify address validity */
        ja      fusufault
 
@@ -533,7 +533,7 @@ ENTRY(subyte)
        movq    TD_PCB(%rcx), %rcx
        movq    $fusufault,PCB_ONFAULT(%rcx)
 
-       movq    $VM_MAXUSER_ADDRESS-1,%rax
+       movq    $VM_MAX_USER_ADDRESS-1,%rax
        cmpq    %rax,%rdi                       /* verify address validity */
        ja      fusufault
 
@@ -562,13 +562,13 @@ ENTRY(copyinstr)
        movq    TD_PCB(%rcx), %rcx
        movq    $cpystrflt,PCB_ONFAULT(%rcx)
 
-       movq    $VM_MAXUSER_ADDRESS,%rax
+       movq    $VM_MAX_USER_ADDRESS,%rax
 
        /* make sure 'from' is within bounds */
        subq    %rsi,%rax
        jbe     cpystrflt
 
-       /* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */
+       /* restrict maxlen to <= VM_MAX_USER_ADDRESS-from */
        cmpq    %rdx,%rax
        jae     1f
        movq    %rax,%rdx
@@ -592,7 +592,7 @@ ENTRY(copyinstr)
        jmp     cpystrflt_x
 3:
        /* rdx is zero - return ENAMETOOLONG or EFAULT */
-       movq    $VM_MAXUSER_ADDRESS,%rax
+       movq    $VM_MAX_USER_ADDRESS,%rax
        cmpq    %rax,%rsi
        jae     cpystrflt
 4:
index 2b10dd6..f25bdd9 100644 (file)
@@ -225,21 +225,13 @@ ENTRY(cpu_exit_switch)
        /*
         * Get us out of the vmspace
         */
-#if JG
+       movq    KPML4phys,%rcx
        movq    %cr3,%rax
        cmpq    %rcx,%rax
        je      1f
        /* JG no increment of statistics counters? see cpu_heavy_restore */
        movq    %rcx,%cr3
 1:
-#else
-       movq    IdlePTD, %rcx
-       orq     $(PG_RW|PG_V), %rcx
-       movq    link_pdpe,%r12
-       movq    %rcx, (%r12)
-       movq    %cr3, %rcx
-       movq    %rcx, %cr3
-#endif
        movq    PCPU(curthread),%rbx
 
        /*
@@ -317,8 +309,6 @@ ENTRY(cpu_heavy_restore)
         * YYY which naturally also means that the PM_ACTIVE bit had better
         * already have been set before we set it above, check? YYY
         */
-#if JG
-#error x
        movq    %cr3,%rsi
        movq    PCB_CR3(%rdx),%rcx
        cmpq    %rsi,%rcx
@@ -329,15 +319,6 @@ ENTRY(cpu_heavy_restore)
 #endif
        movq    %rcx,%cr3
 4:
-#else
-       movq    PCB_CR3(%rdx),%rcx
-       orq     $(PG_RW|PG_U|PG_V), %rcx
-       /*XXX*/
-       movq    link_pdpe,%r12
-       movq    %rcx, (%r12)
-       movq    %cr3, %rcx
-       movq    %rcx, %cr3
-#endif
        /*
         * Clear TDF_RUNNING flag in old thread only after cleaning up
         * %cr3.  The target thread is already protected by being TDF_RUNQ
@@ -513,13 +494,6 @@ ENTRY(savectx)
        movq    %rax,PCB_RIP(%rcx)
 
        movq    %cr3,%rax
-#ifndef JG
-       movq    (%rax), %rax
-       movq    $0x000ffffffffff000, %rcx
-       andq    %rcx, %rax
-       movq    (%rax), %rax
-       andq    %rcx, %rax
-#endif
        movq    %rax,PCB_CR3(%rcx)
 
        movq    %rbx,PCB_RBX(%rcx)
@@ -590,15 +564,11 @@ ENTRY(savectx)
  */
 ENTRY(cpu_idle_restore)
        /* cli */
-       movq    IdlePTD,%rcx
+       movq    KPML4phys,%rcx
        /* JG xor? */
        movl    $0,%ebp
        /* JG push RBP? */
        pushq   $0
-       orq     $(PG_RW|PG_V), %rcx
-       movq    link_pdpe,%r12
-       movq    %rcx, (%r12)
-       movq    %cr3, %rcx
        movq    %rcx,%cr3
        andl    $~TDF_RUNNING,TD_FLAGS(%rbx)
        orl     $TDF_RUNNING,TD_FLAGS(%rax)
@@ -628,14 +598,10 @@ ENTRY(cpu_idle_restore)
  */
 ENTRY(cpu_kthread_restore)
        sti
-       movq    IdlePTD,%rcx
+       movq    KPML4phys,%rcx
        movq    TD_PCB(%rax),%rdx
        /* JG "movq $0, %rbp"? "xorq %rbp, %rbp"? */
        movl    $0,%ebp
-       orq     $(PG_RW|PG_V), %rcx
-       movq    link_pdpe,%r12
-       movq    %rcx, (%r12)
-       movq    %cr3, %rcx
        movq    %rcx,%cr3
        /* rax and rbx come from the switchout code */
        andl    $~TDF_RUNNING,TD_FLAGS(%rbx)
@@ -718,22 +684,12 @@ ENTRY(cpu_lwkt_switch)
  *     deleted due to a process exiting.
  */
 ENTRY(cpu_lwkt_restore)
-#if JG
-       movq    common_lvl4_phys,%rcx   /* YYY borrow but beware desched/cpuchg/exit */
-#endif
-       movq    IdlePTD, %rcx
-       orq     $(PG_RW|PG_V), %rcx
-       movq    link_pdpe,%r12
-       movq    %rcx, (%r12)
-       movq    %cr3, %rcx
-       movq    %rcx, %cr3
-#if JG
+       movq    KPML4phys,%rcx  /* YYY borrow but beware desched/cpuchg/exit */
        movq    %cr3,%rdx
        cmpq    %rcx,%rdx
        je      1f
        movq    %rcx,%cr3
 1:
-#endif
        andl    $~TDF_RUNNING,TD_FLAGS(%rbx)
        orl     $TDF_RUNNING,TD_FLAGS(%rax)
        popfq
index a1a907d..c9b4d16 100644 (file)
@@ -141,7 +141,7 @@ cpu_fork(struct lwp *lp1, struct lwp *lp2, int flags)
         * Set registers for trampoline to user mode.  Leave space for the
         * return address on stack.  These are the kernel mode register values.
         */
-       pcb2->pcb_cr3 = vtophys(vmspace_pmap(lp2->lwp_proc->p_vmspace)->pm_pdir);
+       pcb2->pcb_cr3 = vtophys(vmspace_pmap(lp2->lwp_proc->p_vmspace)->pm_pml4);
        pcb2->pcb_cr3 |= PG_RW | PG_U | PG_V;
        pcb2->pcb_rbx = (unsigned long)fork_return;     /* fork_trampoline argument */
        pcb2->pcb_rbp = 0;
@@ -234,10 +234,11 @@ void
 cpu_lwp_exit(void)
 {
        struct thread *td = curthread;
-       struct pcb *pcb = td->td_pcb;
+       struct pcb *pcb;
 #if NNPX > 0
        npxexit();
 #endif /* NNPX */
+       pcb = td->td_pcb;
        KKASSERT(pcb->pcb_ext == NULL); /* Some i386 functionality was dropped */
         if (pcb->pcb_flags & PCB_DBREGS) {
                 /*
index 6cae749..175a7f9 100644 (file)
@@ -9,7 +9,7 @@
 # reserved for user applications.
 #
 CFLAGS+=       -mpreferred-stack-boundary=4
-CFLAGS+=       -mcmodel=small -mno-red-zone \
+CFLAGS+=       -mcmodel=kernel -mno-red-zone \
                -mfpmath=387 -mno-sse -mno-sse2 -mno-sse3 -mno-mmx -mno-3dnow \
                -msoft-float -fno-asynchronous-unwind-tables \
                -fno-omit-frame-pointer
index 36e8971..6a6d8ad 100644 (file)
@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 1991 Regents of the University of California.
+ * Copyright (c) 2003 Peter Wemm.
  * Copyright (c) 2008 The DragonFly Project.
  * All rights reserved.
  *
 #endif
 
 /*
- * Pte related macros
+ * Pte related macros.  This is complicated by having to deal with
+ * the sign extension of the 48th bit.
  */
-#define VADDR(pdi, pti) ((vm_offset_t)(((pdi)<<PDRSHIFT)|((pti)<<PAGE_SHIFT)))
-
+#define KVADDR(l4, l3, l2, l1) ( \
+       ((unsigned long)-1 << 47) | \
+       ((unsigned long)(l4) << PML4SHIFT) | \
+       ((unsigned long)(l3) << PDPSHIFT) | \
+       ((unsigned long)(l2) << PDRSHIFT) | \
+       ((unsigned long)(l1) << PAGE_SHIFT))
+
+#define UVADDR(l4, l3, l2, l1) ( \
+       ((unsigned long)(l4) << PML4SHIFT) | \
+       ((unsigned long)(l3) << PDPSHIFT) | \
+       ((unsigned long)(l2) << PDRSHIFT) | \
+       ((unsigned long)(l1) << PAGE_SHIFT))
+
+/* Initial number of kernel page tables. */
 #ifndef NKPT
-#define        NKPT            30      /* actual number of kernel page tables */
-#endif
-#ifndef NKPDE
-#define NKPDE  (KVA_PAGES - 2) /* addressable number of page tables/pde's */
-#endif
-#if NKPDE > KVA_PAGES - 2
-#error "Maximum NKPDE is KVA_PAGES - 2"
+#define        NKPT            32
 #endif
 
+#define NKPML4E                1               /* number of kernel PML4 slots */
+#define NKPDPE         howmany(NKPT, NPDEPG)/* number of kernel PDP slots */
+
+#define        NUPML4E         (NPML4EPG/2)    /* number of userland PML4 pages */
+#define        NUPDPE          (NUPML4E*NPDPEPG)/* number of userland PDP pages */
+#define        NUPDE           (NUPDPE*NPDEPG) /* number of userland PD entries */
+
+#define        NDMPML4E        1               /* number of dmap PML4 slots */
+
 /*
- * The *PTDI values control the layout of virtual memory
- *
- * XXX This works for now, but I am not real happy with it, I'll fix it
- * right after I fix locore.s and the magic 28K hole
- *
- * SMP_PRIVPAGES: The per-cpu address space is 0xff80000 -> 0xffbfffff
+ * The *PML4I values control the layout of virtual memory
  */
-#define        APTDPTDI        (NPDEPG-1)      /* alt ptd entry that points to APTD */
-#define MPPTDI         (APTDPTDI-1)    /* per cpu ptd entry */
-#define        KPTDI           (MPPTDI-NKPDE)  /* start of kernel virtual pde's */
-#define        PTDPTDI         (KPTDI-1)       /* ptd entry that points to ptd! */
-#define        UMAXPTDI        (PTDPTDI-1)     /* ptd entry for user space end */
-#define        UMAXPTEOFF      (NPTEPG)        /* pte entry for user space end */
+#define        PML4PML4I       (NPML4EPG/2)    /* Index of recursive pml4 mapping */
+
+#define        KPML4I          (NPML4EPG-1)    /* Top 512GB for KVM */
+#define        DMPML4I         (KPML4I-1)      /* Next 512GB down for direct map */
 
-#define LINKPML4I      0
-#define LINKPDPI       0
+#define        KPDPI           (NPDPEPG-2)     /* kernbase at -2GB */
+
+/* per-CPU data is at -2MB */
+/* XXX can the kernel decide to use this memory for something else? */
+#define        MPPML4I         KPML4I
+#define        MPPDPI          KPDPI
+#define        MPPTDI          (NPDEPG-1)
 
 /*
  * XXX doesn't really belong here I guess...
  * and directories.
  */
 #ifdef _KERNEL
-extern pt_entry_t PTmap[], APTmap[], Upte;
-extern pd_entry_t PTD[], APTD[], PTDpde, APTDpde, Upde;
-
-extern uint64_t IdlePTD;       /* physical address of "Idle" state directory */
-
-extern uint64_t common_lvl4_phys; 
-extern uint64_t common_lvl3_phys; 
-extern pdp_entry_t *link_pdpe;
+#define        addr_PTmap      (KVADDR(PML4PML4I, 0, 0, 0))
+#define        addr_PDmap      (KVADDR(PML4PML4I, PML4PML4I, 0, 0))
+#define        addr_PDPmap     (KVADDR(PML4PML4I, PML4PML4I, PML4PML4I, 0))
+#define        addr_PML4map    (KVADDR(PML4PML4I, PML4PML4I, PML4PML4I, PML4PML4I))
+#define        addr_PML4pml4e  (addr_PML4map + (PML4PML4I * sizeof(pml4_entry_t)))
+#define        PTmap           ((pt_entry_t *)(addr_PTmap))
+#define        PDmap           ((pd_entry_t *)(addr_PDmap))
+#define        PDPmap          ((pd_entry_t *)(addr_PDPmap))
+#define        PML4map         ((pd_entry_t *)(addr_PML4map))
+#define        PML4pml4e       ((pd_entry_t *)(addr_PML4pml4e))
+
+extern u_int64_t KPML4phys;    /* physical address of kernel level 4 */
 #endif
 
 #ifdef _KERNEL
@@ -137,29 +156,10 @@ extern pdp_entry_t *link_pdpe;
  * Note: these work recursively, thus vtopte of a pte will give
  * the corresponding pde that in turn maps it.
  */
-#define        vtopte(va)      (PTmap + amd64_btop(va))
+pt_entry_t *vtopte(vm_offset_t);
 
 #define        avtopte(va)     (APTmap + amd64_btop(va))
 
-/*
- *     Routine:        pmap_kextract
- *     Function:
- *             Extract the physical page address associated
- *             kernel virtual address.
- */
-static __inline vm_paddr_t
-pmap_kextract(vm_offset_t va)
-{
-       vm_paddr_t pa;
-
-       if ((pa = (vm_offset_t) PTD[va >> PDRSHIFT]) & PG_PS) {
-               pa = (pa & ~(NBPDR - 1)) | (va & (NBPDR - 1));
-       } else {
-               pa = *(vm_offset_t *)vtopte(va);
-               pa = (pa & PG_FRAME) | (va & PAGE_MASK);
-       }
-       return pa;
-}
 
 /*
  * XXX
@@ -171,6 +171,15 @@ pmap_kextract(vm_offset_t va)
 
 #define        pte_load_clear(pte)     atomic_readandclear_long(pte)
 
+static __inline void
+pte_store(pt_entry_t *ptep, pt_entry_t pte)
+{
+
+       *ptep = pte;
+}
+
+#define        pde_store(pdep, pde)    pte_store((pdep), (pde))
+
 /*
  * Pmap stuff
  */
@@ -199,7 +208,7 @@ struct pmap_statistics {
 typedef struct pmap_statistics *pmap_statistics_t;
 
 struct pmap {
-       pd_entry_t              *pm_pdir;       /* KVA of page directory */
+       pml4_entry_t            *pm_pml4;       /* KVA of level 4 page table */
        struct vm_page          *pm_pdirm;      /* VM page for pg directory */
        struct vm_object        *pm_pteobj;     /* Container for pte's */
        TAILQ_ENTRY(pmap)       pm_pmnode;      /* list of pmaps */
@@ -250,14 +259,17 @@ extern vm_offset_t clean_eva;
 extern vm_offset_t clean_sva;
 extern char *ptvmmap;          /* poor name! */
 
-void   pmap_bootstrap ( vm_paddr_t *, vm_paddr_t);
+void   pmap_bootstrap ( vm_paddr_t *);
 void   *pmap_mapdev (vm_paddr_t, vm_size_t);
 void   pmap_unmapdev (vm_offset_t, vm_size_t);
+#if JG
 pt_entry_t *pmap_pte (pmap_t, vm_offset_t) __pure2;
+#endif
 struct vm_page *pmap_use_pt (pmap_t, vm_offset_t);
 #ifdef SMP
 void   pmap_set_opt (void);
 #endif
+vm_paddr_t pmap_kextract(vm_offset_t);
 
 #endif /* _KERNEL */
 
index 98c1174..b660e21 100644 (file)
  * messy at times, but hey, we'll do anything to save a page :-)
  */
 
-#define        VM_MAX_KERNEL_ADDRESS   (1024UL * 1024 * 1024)
-#define        VM_MIN_KERNEL_ADDRESS   (512UL * 1024 * 1024)
+#define        VM_MAX_KERNEL_ADDRESS   KVADDR(KPML4I, NPDPEPG-1, NPDEPG-1, NPTEPG-1)
+#define        VM_MIN_KERNEL_ADDRESS   KVADDR(KPML4I, NPDPEPG-7, 0, 0)
 
 #define        DMAP_MIN_ADDRESS        KVADDR(DMPML4I, 0, 0, 0)
 #define        DMAP_MAX_ADDRESS        KVADDR(DMPML4I+1, 0, 0, 0)
 
-#define        KERNBASE                (512 * 1024 * 1024)
-#define PTOV_OFFSET            KERNBASE
+#define        KERNBASE                KVADDR(KPML4I, KPDPI, 0, 0)
+#define        PTOV_OFFSET             KERNBASE
 
+#if JG
 #define KPT_MAX_ADDRESS                VADDR(PTDPTDI, KPTDI+NKPT)
 #define KPT_MIN_ADDRESS                VADDR(PTDPTDI, KPTDI)
+#endif
 
-#define UPT_MAX_ADDRESS                VADDR(PTDPTDI, PTDPTDI)
-#define UPT_MIN_ADDRESS                VADDR(PTDPTDI, 0)
-
-#define        VM_MAXUSER_ADDRESS      UPT_MIN_ADDRESS
-
-#define        USRSTACK                VM_MAXUSER_ADDRESS
-
-#define        VM_MAX_ADDRESS          UPT_MAX_ADDRESS
-#define        VM_MIN_ADDRESS          (0)
+#define UPT_MAX_ADDRESS                KVADDR(PML4PML4I, PML4PML4I, PML4PML4I, PML4PML4I)
+#define UPT_MIN_ADDRESS                KVADDR(PML4PML4I, 0, 0, 0)
 
 #define VM_MIN_USER_ADDRESS    ((vm_offset_t)0)
-#define VM_MAX_USER_ADDRESS    VM_MAXUSER_ADDRESS
+#define VM_MAX_USER_ADDRESS    UVADDR(NUPML4E, 0, 0, 0)
+
+#define USRSTACK               VM_MAX_USER_ADDRESS
 
 #define        PHYS_TO_DMAP(x)         ((x) | DMAP_MIN_ADDRESS)
 #define        DMAP_TO_PHYS(x)         ((x) & ~DMAP_MIN_ADDRESS)
 
-
 /* initial pagein size of beginning of executable file */
 #ifndef VM_INITIAL_PAGEIN
 #define        VM_INITIAL_PAGEIN       16