From: Matthew Dillon <dillon@apollo.backplane.com>
Date: Wed, 31 Mar 2010 16:14:28 +0000 (-0700)
Subject: kernel - more lwbuf followup work
X-Git-Tag: v2.7.1~37^2~19
X-Git-Url: https://gitweb.dragonflybsd.org/dragonfly.git/commitdiff_plain/1f3936d6eb39f987d7f105fd29c6c81bcd2335d7

kernel - more lwbuf followup work

* Make lwbuf objcache only, removing all the manual per-cpu allocation
  tracking.  Keep the cpumask stuff.  We will deal with the KVM
  fragmentation issue inside objcache later on.

* This basically takes us back to Sam's original objcache implementation.

* Remove unnecessary assembly symbols (assembly didn't use those globaldata
  fields).  Remove related globaldata fields now that we are back to the
  objcache-only implementation.
---

diff --git a/sys/cpu/i386/include/lwbuf.h b/sys/cpu/i386/include/lwbuf.h
index 9933a6a36d..e253ba6270 100644
--- a/sys/cpu/i386/include/lwbuf.h
+++ b/sys/cpu/i386/include/lwbuf.h
@@ -62,16 +62,11 @@
 struct mdglobaldata;
 
 struct lwbuf {
-    SLIST_ENTRY(lwbuf)	next;		/* when on per-cpu free list */
-    struct mdglobaldata *gd;		/* originating cpu */
     vm_page_t		m;		/* currently mapped page */
     vm_offset_t		kva;		/* va of mapping */
     cpumask_t		cpumask;	/* cpu mapping synchronization */
-    boolean_t		ephemeral;	/* lives out of objcache if true */
 };
 
-SLIST_HEAD(lwbuf_list, lwbuf);
-
 static __inline vm_page_t
 lwbuf_page(struct lwbuf *lwb)
 {
diff --git a/sys/cpu/i386/misc/lwbuf.c b/sys/cpu/i386/misc/lwbuf.c
index a51c9535a5..3819cae380 100644
--- a/sys/cpu/i386/misc/lwbuf.c
+++ b/sys/cpu/i386/misc/lwbuf.c
@@ -60,45 +60,37 @@ SYSINIT(sock_lwb, SI_BOOT2_MACHDEP, SI_ORDER_ANY, lwbuf_init, NULL);
 static struct objcache *lwbuf_cache;
 
 MALLOC_DEFINE(M_LWBUF, "lwbuf", "Lightweight buffers");
-struct objcache_malloc_args lwbuf_malloc_args = { sizeof(struct lwbuf), M_LWBUF };
+struct objcache_malloc_args lwbuf_malloc_args =
+	    { sizeof(struct lwbuf), M_LWBUF };
 
 /* Number of pages of KVA to allocate at boot per cpu (1MB) */
 static int lwbuf_reserve_pages = 256;
+static int lwbuf_count;
+static int lwbuf_kva_bytes;
 
 SYSCTL_INT(_kern_ipc, OID_AUTO, lwbuf_reserve, CTLFLAG_RD,
            &lwbuf_reserve_pages, 0,
-           "Per-CPU count of pre-allocated lightweight buffers");
-
-static int
-do_lwbuf_count(SYSCTL_HANDLER_ARGS)
-{
-    int count[SMP_MAXCPU];
-    struct mdglobaldata *gd;
-    int i;
-
-    for (i = 0; i < ncpus; i++) {
-        gd = &CPU_prvspace[i].mdglobaldata;
-        count[i] = gd->gd_lwbuf_count;
-    }
-
-    return (sysctl_handle_opaque(oidp, count, sizeof(int) * ncpus, req));
-}
-SYSCTL_PROC(_kern_ipc, OID_AUTO, lwbuf_count, CTLTYPE_INT|CTLFLAG_RD,
-            0, 0, do_lwbuf_count, "I",
-            "Per-CPU count of free lightweight buffers");
+           "Number of pre-allocated lightweight buffers");
+SYSCTL_INT(_kern_ipc, OID_AUTO, lwbuf_count, CTLFLAG_RD,
+	   &lwbuf_count, 0,
+	   "Currently allocated lightweight buffers");
+SYSCTL_INT(_kern_ipc, OID_AUTO, lwbuf_kva_bytes, CTLFLAG_RD,
+	   &lwbuf_kva_bytes, 0,
+	   "Currently used KVA for lightweight buffers");
 
 static boolean_t
 lwbuf_cache_ctor(void *obj, void *pdata, int ocflags)
 {
     struct lwbuf *lwb = (struct lwbuf *)obj;
 
+    lwb->m = NULL;
+    lwb->cpumask = 0;
     get_mplock();
     lwb->kva = kmem_alloc_nofault(&kernel_map, PAGE_SIZE, PAGE_SIZE);
     rel_mplock();
     if (lwb->kva == 0)
         return (FALSE);
-    lwb->gd = mdcpu;
-    lwb->ephemeral = TRUE;
+    atomic_add_int(&lwbuf_kva_bytes, PAGE_SIZE);
 
     return (TRUE);
 }
@@ -108,40 +100,17 @@ lwbuf_cache_dtor(void *obj, void *pdata)
 {
     struct lwbuf *lwb = (struct lwbuf *)obj;
 
+    KKASSERT(lwb->kva != 0);
     get_mplock();
     kmem_free(&kernel_map, lwb->kva, PAGE_SIZE);
     rel_mplock();
+    lwb->kva = 0;
+    atomic_add_int(&lwbuf_kva_bytes, -PAGE_SIZE);
 }
 
 static void
 lwbuf_init(void *arg)
 {
-    struct mdglobaldata *gd = mdcpu;
-    struct lwbuf *lwb;
-    vm_offset_t k;
-    int i, j;
-
-    for (i = 0; i < ncpus; ++i) {
-	gd = &CPU_prvspace[i].mdglobaldata;
-        SLIST_INIT(&gd->gd_lwbuf_fpages);
-        get_mplock();
-        k = kmem_alloc_nofault(&kernel_map, PAGE_SIZE * lwbuf_reserve_pages,
-                               PAGE_SIZE);
-        rel_mplock();
-        if (k == 0) {
-            gd->gd_lwbuf_count = 0;
-            continue;
-        }
-        for (j = 0; j < lwbuf_reserve_pages; ++j) {
-            lwb = kmalloc(sizeof(*lwb), M_LWBUF, M_WAITOK | M_ZERO);
-            lwb->kva = k + (j * PAGE_SIZE);
-            lwb->gd = gd;
-            lwb->ephemeral = FALSE;
-            SLIST_INSERT_HEAD(&gd->gd_lwbuf_fpages, lwb, next);
-        }
-        gd->gd_lwbuf_count = lwbuf_reserve_pages;
-    }
-
     lwbuf_cache = objcache_create("lwbuf", 0, 0,
         lwbuf_cache_ctor, lwbuf_cache_dtor, NULL,
         objcache_malloc_alloc, objcache_malloc_free,
@@ -154,62 +123,34 @@ lwbuf_alloc(vm_page_t m)
     struct mdglobaldata *gd = mdcpu;
     struct lwbuf *lwb;
 
-    crit_enter_gd(&gd->mi);
-    while ((lwb = SLIST_FIRST(&gd->gd_lwbuf_fpages)) == NULL) {
-        if ((lwb = objcache_get(lwbuf_cache, M_WAITOK)) == NULL)
-            tsleep(&gd->gd_lwbuf_fpages, 0, "lwbuf", 0);
-    }
-    --gd->gd_lwbuf_count;
-    SLIST_REMOVE_HEAD(&gd->gd_lwbuf_fpages, next);
+    while ((lwb = objcache_get(lwbuf_cache, M_WAITOK)) == NULL)
+	tsleep(lwbuf_cache, 0, "lwbuf", hz);
+    KKASSERT(lwb->m == NULL);
     lwb->m = m;
-    crit_exit_gd(&gd->mi);
-
+    lwb->cpumask = gd->mi.gd_cpumask;
     pmap_kenter_quick(lwb->kva, m->phys_addr);
-    lwb->cpumask |= gd->mi.gd_cpumask;
+    atomic_add_int(&lwbuf_count, 1);
 
     return (lwb);
 }
 
-#ifdef SMP
-static
-void
-lwbuf_free_remote(void *arg)
-{
-    lwbuf_free(arg);
-}
-#endif
-
 void
 lwbuf_free(struct lwbuf *lwb)
 {
-    struct mdglobaldata *gd = mdcpu;
-
+    KKASSERT(lwb->m != NULL);
     lwb->m = NULL;
     lwb->cpumask = 0;
-
-    if (gd == lwb->gd) {
-        if (lwb->ephemeral == FALSE) {
-            crit_enter_gd(&gd->mi);
-            SLIST_INSERT_HEAD(&gd->gd_lwbuf_fpages, lwb, next);
-            if (gd->gd_lwbuf_count++ == 0)
-                wakeup_one(&gd->gd_lwbuf_fpages);
-            crit_exit_gd(&gd->mi);
-        } else {
-            objcache_put(lwbuf_cache, lwb);
-        }
-    }
-#ifdef SMP
-    else {
-	lwkt_send_ipiq_passive(&lwb->gd->mi, lwbuf_free_remote, lwb);
-    }
-#endif
+    objcache_put(lwbuf_cache, lwb);
+    atomic_add_int(&lwbuf_count, -1);
 }
 
 void
 lwbuf_set_global(struct lwbuf *lwb)
 {
-    pmap_kenter_sync(lwb->kva);
-    lwb->cpumask = (cpumask_t)-1;
+    if (lwb->cpumask != (cpumask_t)-1) {
+	pmap_kenter_sync(lwb->kva);
+	lwb->cpumask = (cpumask_t)-1;
+    }
 }
 
 static vm_offset_t
diff --git a/sys/platform/pc32/i386/genassym.c b/sys/platform/pc32/i386/genassym.c
index 5a9b591685..65e5c4daad 100644
--- a/sys/platform/pc32/i386/genassym.c
+++ b/sys/platform/pc32/i386/genassym.c
@@ -210,7 +210,6 @@ ASSYM(GD_SPENDING, offsetof(struct mdglobaldata, gd_spending));
 ASSYM(GD_COMMON_TSS, offsetof(struct mdglobaldata, gd_common_tss));
 ASSYM(GD_COMMON_TSSD, offsetof(struct mdglobaldata, gd_common_tssd));
 ASSYM(GD_TSS_GDT, offsetof(struct mdglobaldata, gd_tss_gdt));
-ASSYM(GD_LWBUF_FPAGES, offsetof(struct mdglobaldata, gd_lwbuf_fpages));
 ASSYM(GD_NPXTHREAD, offsetof(struct mdglobaldata, gd_npxthread));
 ASSYM(GD_FPU_LOCK, offsetof(struct mdglobaldata, gd_fpu_lock));
 ASSYM(GD_SAVEFPU, offsetof(struct mdglobaldata, gd_savefpu));
diff --git a/sys/platform/pc32/i386/globals.s b/sys/platform/pc32/i386/globals.s
index 9ea88d389c..2aa2467c50 100644
--- a/sys/platform/pc32/i386/globals.s
+++ b/sys/platform/pc32/i386/globals.s
@@ -58,9 +58,6 @@
 	.set	gd_reqflags,globaldata + GD_REQFLAGS
 	.set	gd_common_tss,globaldata + GD_COMMON_TSS
 
-	.globl	gd_lwbuf_fpages
-	.set	gd_lwbuf_fpages,globaldata + GD_LWBUF_FPAGES
-
 	.globl	gd_common_tssd, gd_tss_gdt
 	.set	gd_common_tssd,globaldata + GD_COMMON_TSSD
 	.set	gd_tss_gdt,globaldata + GD_TSS_GDT
diff --git a/sys/platform/pc32/include/globaldata.h b/sys/platform/pc32/include/globaldata.h
index 17aa5de3ec..3e3cbeca78 100644
--- a/sys/platform/pc32/include/globaldata.h
+++ b/sys/platform/pc32/include/globaldata.h
@@ -76,8 +76,6 @@ struct mdglobaldata {
 	struct segment_descriptor *gd_tss_gdt;
 	struct thread   *gd_npxthread;
 	struct i386tss  gd_common_tss;
-	struct lwbuf_list gd_lwbuf_fpages; /* lwbuf: free kva */
-	int		gd_lwbuf_count;
 	union savefpu	gd_savefpu;	/* fast bcopy/zero temp fpu save area */
 	int		gd_fpu_lock;	/* fast bcopy/zero cpu lock */
 	int		gd_fpending;	/* fast interrupt pending */
diff --git a/sys/platform/vkernel/i386/genassym.c b/sys/platform/vkernel/i386/genassym.c
index 22c18ee6d5..cbabc033d6 100644
--- a/sys/platform/vkernel/i386/genassym.c
+++ b/sys/platform/vkernel/i386/genassym.c
@@ -200,7 +200,6 @@ ASSYM(GD_SPENDING, offsetof(struct mdglobaldata, gd_spending));
 ASSYM(GD_COMMON_TSS, offsetof(struct mdglobaldata, gd_common_tss));
 ASSYM(GD_COMMON_TSSD, offsetof(struct mdglobaldata, gd_common_tssd));
 ASSYM(GD_TSS_GDT, offsetof(struct mdglobaldata, gd_tss_gdt));
-ASSYM(GD_LWBUF_FPAGES, offsetof(struct mdglobaldata, gd_lwbuf_fpages));
 ASSYM(GD_NPXTHREAD, offsetof(struct mdglobaldata, gd_npxthread));
 ASSYM(GD_FPU_LOCK, offsetof(struct mdglobaldata, gd_fpu_lock));
 ASSYM(GD_SAVEFPU, offsetof(struct mdglobaldata, gd_savefpu));
diff --git a/sys/platform/vkernel/i386/global.s b/sys/platform/vkernel/i386/global.s
index 9772e925f8..c81783fa79 100644
--- a/sys/platform/vkernel/i386/global.s
+++ b/sys/platform/vkernel/i386/global.s
@@ -55,9 +55,6 @@
 	.set	gd_reqflags,globaldata + GD_REQFLAGS
 	.set	gd_common_tss,globaldata + GD_COMMON_TSS
 
-	.globl	gd_lwbuf_fpages
-	.set	gd_lwbuf_fpages,globaldata + GD_LWBUF_FPAGES
-
 	.globl	gd_common_tssd, gd_tss_gdt
 	.set	gd_common_tssd,globaldata + GD_COMMON_TSSD
 	.set	gd_tss_gdt,globaldata + GD_TSS_GDT