static boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */
static int pgeflag; /* PG_G or-in */
static int pseflag; /* PG_PS or-in */
-static cpumask_t APTmask;
static vm_object_t kptobj;
/*
* pmap_pte_quick:
*
- * Super fast pmap_pte routine best used when scanning the pv lists.
- * This eliminates many course-grained invltlb calls. Note that many of
- * the pv list scans are across different pmaps and it is very wasteful
- * to do an entire invltlb when checking a single mapping.
+ * Super fast pmap_pte routine best used when scanning the pv lists.
+ * This eliminates many course-grained invltlb calls. Note that many of
+ * the pv list scans are across different pmaps and it is very wasteful
+ * to do an entire invltlb when checking a single mapping.
*
- * Should only be called while in a critical section.
+ * Should only be called while in a critical section.
+ *
+ * Unlike get_ptbase(), this function MAY be called from an interrupt or
+ * interrupt thread.
*/
static unsigned *
pmap_pte_quick(pmap_t pmap, vm_offset_t va)
gd->gd_CMAP2 = &SMPpt[pg + 1];
gd->gd_CMAP3 = &SMPpt[pg + 2];
gd->gd_PMAP1 = &SMPpt[pg + 3];
+ gd->gd_GDMAP1 = &PTD[KGDTDI];
gd->gd_CADDR1 = CPU_prvspace[0].CPAGE1;
gd->gd_CADDR2 = CPU_prvspace[0].CPAGE2;
gd->gd_CADDR3 = CPU_prvspace[0].CPAGE3;
gd->gd_PADDR1 = (unsigned *)CPU_prvspace[0].PPAGE1;
+ gd->gd_GDADDR1= (unsigned *)VADDR(KGDTDI, 0);
cpu_invltlb();
}
if (pmap == &kernel_pmap || frame == (((unsigned)PTDpde) & PG_FRAME)) {
return;
}
- KKASSERT(frame == (((unsigned)APTDpde) & PG_FRAME));
+ KKASSERT(frame == (*mycpu->gd_GDMAP1 & PG_FRAME));
}
#else
return 0;
}
+/*
+ * Retrieve the mapped page table base for a particular pmap. Use our self
+ * mapping for the kernel_pmap or our current pmap.
+ *
+ * For foreign pmaps we use the per-cpu page table map. Since this involves
+ * installing a ptd it's actually (per-process x per-cpu). However, we
+ * still cannot depend on our mapping to survive thread switches because
+ * the process might be threaded and switching to another thread for the
+ * same process on the same cpu will allow that other thread to make its
+ * own mapping.
+ *
+ * This could be a bit confusing but the jist is for something like the
+ * vkernel which uses foreign pmaps all the time this represents a pretty
+ * good cache that avoids unnecessary invltlb()s.
+ */
static unsigned *
get_ptbase(pmap_t pmap)
{
unsigned frame = (unsigned) pmap->pm_pdir[PTDPTDI] & PG_FRAME;
- struct globaldata *gd __debugvar = mycpu;
+ struct mdglobaldata *gd = mdcpu;
/*
* We can use PTmap if the pmap is our current address space or
}
/*
- * Otherwise we use the alternative address space, APTmap. This
- * map is stored in the user portion of the current pmap. However,
- * the pmap may still be shared across cpus. Since we are only
- * doing a local invltlb we have to keep track of which cpus have
- * synced up.
+ * Otherwise we use the per-cpu alternative page table map. Each
+ * cpu gets its own map. Because of this we cannot use this map
+ * from interrupts or threads which can preempt.
*/
- KKASSERT(gd->gd_intr_nesting_level == 0 &&
- (gd->gd_curthread->td_flags & TDF_INTTHREAD) == 0);
+ KKASSERT(gd->mi.gd_intr_nesting_level == 0 &&
+ (gd->mi.gd_curthread->td_flags & TDF_INTTHREAD) == 0);
- if (frame != (((unsigned) APTDpde) & PG_FRAME)) {
- APTDpde = (pd_entry_t)(frame | PG_RW | PG_V);
- APTmask = gd->gd_cpumask;
- cpu_invltlb();
- } else if ((APTmask & gd->gd_cpumask) == 0) {
- APTmask |= gd->gd_cpumask;
+ if ((*gd->gd_GDMAP1 & PG_FRAME) != frame) {
+ *gd->gd_GDMAP1 = frame | PG_RW | PG_V;
cpu_invltlb();
}
- return (unsigned *) APTmap;
+ return ((unsigned *)gd->gd_GDADDR1);
}
/*
* We leave the page directory page cached, wired, and mapped in
* the pmap until the dtor function (pmap_puninit()) gets called.
* However, still clean it up so we can set PG_ZERO.
+ *
+ * The pmap has already been removed from the pmap_list in the
+ * PTDPTDI case.
*/
if (p->pindex == PTDPTDI) {
bzero(pde + KPTDI, nkpt * PTESIZE);
- pde[MPPTDI] = 0;
- pde[APTDPTDI] = 0;
+ bzero(pde + KGDTDI, (NPDEPG - KGDTDI) * PTESIZE);
vm_page_flag_set(p, PG_ZERO);
vm_page_wakeup(p);
} else {
}
dst_frame = ((unsigned) dst_pmap->pm_pdir[PTDPTDI]) & PG_FRAME;
- if (dst_frame != (((unsigned) APTDpde) & PG_FRAME)) {
- APTDpde = (pd_entry_t) (dst_frame | PG_RW | PG_V);
- APTmask = gd->gd_cpumask;
- cpu_invltlb();
- } else if ((APTmask & gd->gd_cpumask) == 0) {
- APTmask |= gd->gd_cpumask;
+ if ((*gd->gd_GDMAP1 & PG_FRAME) != dst_frame) {
+ *gd->gd_GDMAP1 = dst_frame | PG_RW | PG_V;
cpu_invltlb();
}
pmap_inval_init(&info);
dstmpte = pmap_allocpte(dst_pmap, addr);
if (src_frame != (((unsigned) PTDpde) & PG_FRAME) ||
- dst_frame != (((unsigned) APTDpde) & PG_FRAME)
+ XXX dst_frame != (((unsigned) xxx) & PG_FRAME)
) {
kprintf("WARNING: pmap_copy: detected and corrected race\n");
pmap_unwire_pte_hold(dst_pmap, dstmpte, &info);
#endif
/*
- * Pte related macros
+ * PTE related macros
*/
#define VADDR(pdi, pti) ((vm_offset_t)(((pdi)<<PDRSHIFT)|((pti)<<PAGE_SHIFT)))
#ifndef NKPT
-#define NKPT 30 /* actual number of kernel page tables */
+#define NKPT 30 /* starting general kptds */
#endif
+
+#define NKGDPDE SMP_MAXCPU /* 16 typical */
+
#ifndef NKPDE
-#define NKPDE (KVA_PAGES - 2) /* addressable number of page tables/pde's */
+#define NKPDE (KVA_PAGES - NKGDPDE - 2) /* max general kptds */
#endif
-#if NKPDE > KVA_PAGES - 2
-#error "Maximum NKPDE is KVA_PAGES - 2"
+#if NKPDE > KVA_PAGES - NKGDPDE - 2
+#error "Maximum NKPDE is KVA_PAGES - NKGDPDE - 2"
#endif
/*
* The *PTDI values control the layout of virtual memory
*
- * XXX This works for now, but I am not real happy with it, I'll fix it
- * right after I fix locore.s and the magic 28K hole
- *
- * SMP_PRIVPAGES: The per-cpu address space is 0xff80000 -> 0xffbfffff
- *
* NPEDEPG - number of pde's in the page directory (1024)
- * NKPDE - typically (KVA_PAGES - 2) where KVA_PAGES is typically 256
+ * NKPDE - max general kernel page table pages not including
+ * special PTDs. Typically KVA_PAGES minus the number
+ * of special PTDs.
+ *
+ * +---------------+ End of kernel memory
+ * | APTDPTDI | currently unused alt page table map
+ * +---------------+
+ * | MPPTDI | globaldata array
+ * +---------------+
+ * | |
+ * | | per-cpu page table self-maps
+ * |KGDTDI[NKGDPDE]|
+ * +---------------+
+ * | |
+ * | |
+ * | |
+ * | | general kernel page table pages
+ * | |
+ * | KPTDI[NKPDE] |
+ * +---------------+ Start of kernel memory
+ * | PTDPTDI | self-mapping of current pmap
+ * +---------------+
*
* This typically places PTDPTDI at the index corresponding to VM address
- * (0xc0000000 - 4M) = bfc00000, and that is where PTmap[] is based.
+ * (0xc0000000 - 4M) = bfc00000, and that is where PTmap[] is based for
+ * the self-mapped page table. PTD points to the self-mapped page
+ * directory itself and any indexes >= KPTDI will correspond to the
+ * common kernel page directory pages since all pmaps map the same ones.
+ *
+ * We no longer use APTmap or APTDpde (corresponding to APTDPTDI). This
+ * was a global page table map for accessing pmaps other then the current
+ * pmap. Instead we now implement an alternative pmap for EACH cpu
+ * use the ptds at KGDTDI.
*
- * APTmap[] is typically based at 0xffc00000 corresponding to the page
- * directory index of APTDPTDI. APTDpde is self-mapped at 0xbfeffffc,
- * the last pde entry in the pmap's page directory just before the
- * kernel pde's start.
+ * Even though the maps are per-cpu the PTD entries are stored in the
+ * individual pmaps and obviously not replicated so each process pmap
+ * essentially gets its own per-cpu cache (PxN) making for fairly efficient
+ * access.
*
* UMAXPTDI - highest inclusive ptd index for user space
*/
#define APTDPTDI (NPDEPG-1) /* alt ptd entry that points to APTD */
-#define MPPTDI (APTDPTDI-1) /* per cpu ptd entry */
-#define KPTDI (MPPTDI-NKPDE) /* start of kernel virtual pde's */
+#define MPPTDI (APTDPTDI-1) /* globaldata array ptd entry */
+#define KGDTDI (MPPTDI-NKGDPDE) /* per-cpu page table mappings */
+#define KPTDI (KGDTDI-NKPDE) /* start of kernel virtual pde's */
#define PTDPTDI (KPTDI-1) /* ptd entry that points to ptd! */
#define UMAXPTDI (PTDPTDI-1) /* ptd entry for user space end */
projects / dragonfly.git / commitdiff