kernel - Fix i386 wire_count panics

* Tracked down to a situation where a pmap structure is being dtor'd by
  the objcache simultaniously with a vm_page_protect() operation on
  a page table page's vm_page_t.

  (1) vm_page_protect() begins running, finds page table page to remove,
      removes the related pv_entry, but then gets stuck waiting for the
      pmap->pm_pteobj (vm_object token).

  (2) Exit on another thread simultaniously removes all remaining VM
      pages from the pmap.  However, due to #(1), there is still an
      active page table page in pmap->pm_pteobj that the exit code has
      no visibility to.

  (3) The related pmap is then dtor'd due to heavy fork/exec/exit load
      on the system.  The VM page is still present, vm_page_protect()
      is still stuck on the token (or hasn't gotten cpu back).

  (4) Nominal vm_object_terminate() destroys the page table page.

  (5) vm_page_protect() unblocks and tries to destroy the page.

  (6) BOOM.

* This fix places a barrier between the normal process exit code and the
  dtor which will block while a vm_page_protect() is active on the pmap.

* This time for sure, but if not we still know that the problem is related
  to this exit race.

diff --git a/sys/platform/pc32/i386/pmap.c b/sys/platform/pc32/i386/pmap.c
index b93e6f9..d7a6fe6 100644 (file)
--- a/sys/platform/pc32/i386/pmap.c
+++ b/sys/platform/pc32/i386/pmap.c
@@ -215,6 +215,10 @@ static vm_page_t pmap_page_lookup (vm_object_t object, vm_pindex_t pindex);
 static void pmap_unuse_pt (pmap_t, vm_offset_t, vm_page_t, pmap_inval_info_t);
 static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
 
+static void pmap_hold(pmap_t pmap);
+static void pmap_drop(pmap_t pmap);
+static void pmap_wait(pmap_t pmap, int count);
+
 static unsigned pdir4mb;
 
 /*
@@ -1231,6 +1235,7 @@ pmap_puninit(pmap_t pmap)
 {
        vm_page_t p;
 
+       pmap_wait(pmap, -1);
        KKASSERT(pmap->pm_active == 0);
        if ((p = pmap->pm_pdirm) != NULL) {
                KKASSERT(pmap->pm_pdir != NULL);
@@ -1626,6 +1631,45 @@ pmap_reference(pmap_t pmap)
        }
 }
 
+/*
+ * vm_token must be held
+ */
+static
+void
+pmap_hold(pmap_t pmap)
+{
+       ++pmap->pm_count;
+}
+
+/*
+ * vm_token must be held
+ */
+static
+void
+pmap_drop(pmap_t pmap)
+{
+       --pmap->pm_count;
+       if ((pmap->pm_count & 0x7FFFFFFF) == 0)
+               wakeup(pmap);
+}
+
+static
+void
+pmap_wait(pmap_t pmap, int count)
+{
+       lwkt_gettoken(&vm_token);
+       pmap->pm_count += count;
+       if (pmap->pm_count & 0x7FFFFFFF) {
+               while (pmap->pm_count & 0x7FFFFFFF) {
+                       pmap->pm_count |= 0x80000000;
+                       tsleep(pmap, 0, "pmapd", 0);
+                       pmap->pm_count &= ~0x80000000;
+                       kprintf("pmap_wait: race averted\n");
+               }
+       }
+       lwkt_reltoken(&vm_token);
+}
+
 /***************************************************
  * page management routines.
  ***************************************************/
@@ -1997,7 +2041,7 @@ pmap_remove(struct pmap *pmap, vm_offset_t sva, vm_offset_t eva)
  * Removes this physical page from all physical maps in which it resides.
  * Reflects back modify bits to the pager.
  *
- * No requirements.
+ * vm_token must be held by caller.
  */
 static void
 pmap_remove_all(vm_page_t m)
@@ -2005,21 +2049,24 @@ pmap_remove_all(vm_page_t m)
        struct pmap_inval_info info;
        unsigned *pte, tpte;
        pv_entry_t pv;
+       pmap_t pmap;
 
        if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
                return;
 
        pmap_inval_init(&info);
        while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
-               KKASSERT(pv->pv_pmap->pm_stats.resident_count > 0);
-               --pv->pv_pmap->pm_stats.resident_count;
+               pmap = pv->pv_pmap;
+               KKASSERT(pmap->pm_stats.resident_count > 0);
+               --pmap->pm_stats.resident_count;
+               pmap_hold(pmap);
 
-               pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
-               pmap_inval_interlock(&info, pv->pv_pmap, pv->pv_va);
+               pte = pmap_pte_quick(pmap, pv->pv_va);
+               pmap_inval_interlock(&info, pmap, pv->pv_va);
                tpte = loadandclear(pte);
                if (tpte & PG_W)
-                       pv->pv_pmap->pm_stats.wired_count--;
-               pmap_inval_deinterlock(&info, pv->pv_pmap);
+                       pmap->pm_stats.wired_count--;
+               pmap_inval_deinterlock(&info, pmap);
                if (tpte & PG_A)
                        vm_page_flag_set(m, PG_REFERENCED);
                KKASSERT(PHYS_TO_VM_PAGE(tpte) == m);
@@ -2043,17 +2090,18 @@ pmap_remove_all(vm_page_t m)
 #endif
                KKASSERT(pv == TAILQ_FIRST(&m->md.pv_list));
                TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
-               TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist);
-               ++pv->pv_pmap->pm_generation;
+               TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
+               ++pmap->pm_generation;
                m->md.pv_list_count--;
                if (m->object)
                        atomic_add_int(&m->object->agg_pv_list_count, -1);
                if (TAILQ_EMPTY(&m->md.pv_list))
                        vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE);
-               vm_object_hold(pv->pv_pmap->pm_pteobj);
-               pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem, &info);
-               vm_object_drop(pv->pv_pmap->pm_pteobj);
+               vm_object_hold(pmap->pm_pteobj);
+               pmap_unuse_pt(pmap, pv->pv_va, pv->pv_ptem, &info);
+               vm_object_drop(pmap->pm_pteobj);
                free_pv_entry(pv);
+               pmap_drop(pmap);
        }
        KKASSERT((m->flags & (PG_MAPPED|PG_WRITEABLE)) == 0);
        pmap_inval_done(&info);

diff --git a/sys/platform/pc64/x86_64/pmap.c b/sys/platform/pc64/x86_64/pmap.c
index 50a6990..551816f 100644 (file)
--- a/sys/platform/pc64/x86_64/pmap.c
+++ b/sys/platform/pc64/x86_64/pmap.c
@@ -2360,16 +2360,6 @@ pmap_reference(pmap_t pmap)
        }
 }
 
-void
-pmap_drop(pmap_t pmap)
-{
-       if (pmap != NULL) {
-               lwkt_gettoken(&pmap->pm_token);
-               --pmap->pm_count;
-               lwkt_reltoken(&pmap->pm_token);
-       }
-}
-
 /***************************************************
  * page management routines.
  ***************************************************/

diff --git a/sys/vm/pmap.h b/sys/vm/pmap.h
index 315f807..9231184 100644 (file)
--- a/sys/vm/pmap.h
+++ b/sys/vm/pmap.h
@@ -184,7 +184,6 @@ void                 pmap_kmodify_nc(vm_offset_t va);
 void            pmap_kremove (vm_offset_t);
 void            pmap_kremove_quick (vm_offset_t);
 void            pmap_reference (pmap_t);
-void            pmap_drop (pmap_t);
 void            pmap_remove (pmap_t, vm_offset_t, vm_offset_t);
 void            pmap_remove_pages (pmap_t, vm_offset_t, vm_offset_t);
 void            pmap_zero_page (vm_paddr_t);