From: Nicolas Thery <nth@dragonflybsd.org>
Date: Mon, 26 May 2008 17:11:09 +0000 (+0000)
Subject: Allocate lwkt threads from objcache instead of custom per-cpu cache backed
X-Git-Tag: v2.0.1~544
X-Git-Url: https://gitweb.dragonflybsd.org/~uqs/games.git/commitdiff_plain/40aaf5fcac68b1b01e997489147ad3809009e9c1

Allocate lwkt threads from objcache instead of custom per-cpu cache backed
by zone.

Reviewed-by: dillon@
---

diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index 4beadcb5b3..61076fa977 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -40,7 +40,7 @@
  *
  *	@(#)init_main.c	8.9 (Berkeley) 1/21/94
  * $FreeBSD: src/sys/kern/init_main.c,v 1.134.2.8 2003/06/06 20:21:32 tegge Exp $
- * $DragonFly: src/sys/kern/init_main.c,v 1.85 2008/05/08 01:26:00 dillon Exp $
+ * $DragonFly: src/sys/kern/init_main.c,v 1.86 2008/05/26 17:11:09 nth Exp $
  */
 
 #include "opt_init_path.h"
@@ -662,7 +662,6 @@ SYSINIT(kickinit, SI_SUB_KTHREAD_INIT, SI_ORDER_FIRST, kick_init, NULL)
 void
 mi_gdinit(struct globaldata *gd, int cpuid)
 {
-	TAILQ_INIT(&gd->gd_tdfreeq);    /* for pmap_{new,dispose}_thread() */
 	TAILQ_INIT(&gd->gd_systimerq);
 	gd->gd_sysid_alloc = cpuid;	/* prime low bits for cpu lookup */
 	gd->gd_cpuid = cpuid;
diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c
index f2f63c2c92..62e8745712 100644
--- a/sys/kern/kern_proc.c
+++ b/sys/kern/kern_proc.c
@@ -32,7 +32,7 @@
  *
  *	@(#)kern_proc.c	8.7 (Berkeley) 2/14/95
  * $FreeBSD: src/sys/kern/kern_proc.c,v 1.63.2.9 2003/05/08 07:47:16 kbyanc Exp $
- * $DragonFly: src/sys/kern/kern_proc.c,v 1.43 2008/05/18 20:02:02 nth Exp $
+ * $DragonFly: src/sys/kern/kern_proc.c,v 1.44 2008/05/26 17:11:09 nth Exp $
  */
 
 #include <sys/param.h>
@@ -51,7 +51,6 @@
 #include <vm/pmap.h>
 #include <vm/vm_map.h>
 #include <sys/user.h>
-#include <vm/vm_zone.h>
 #include <machine/smp.h>
 
 #include <sys/spinlock2.h>
@@ -85,7 +84,6 @@ u_long pgrphash;
 struct proclist allproc;
 struct proclist zombproc;
 struct spinlock allproc_spin;
-vm_zone_t thread_zone;
 
 /*
  * Random component to nextpid generation.  We mix in a random factor to make
@@ -128,9 +126,9 @@ procinit(void)
 	LIST_INIT(&allproc);
 	LIST_INIT(&zombproc);
 	spin_init(&allproc_spin);
+	lwkt_init();
 	pidhashtbl = hashinit(maxproc / 4, M_PROC, &pidhash);
 	pgrphashtbl = hashinit(maxproc / 4, M_PROC, &pgrphash);
-	thread_zone = zinit("THREAD", sizeof (struct thread), 0, 0, 5);
 	uihashinit();
 }
 
diff --git a/sys/kern/lwkt_thread.c b/sys/kern/lwkt_thread.c
index dfd33895ee..8673c80331 100644
--- a/sys/kern/lwkt_thread.c
+++ b/sys/kern/lwkt_thread.c
@@ -31,7 +31,7 @@
  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  * 
- * $DragonFly: src/sys/kern/lwkt_thread.c,v 1.113 2008/05/18 20:57:56 nth Exp $
+ * $DragonFly: src/sys/kern/lwkt_thread.c,v 1.114 2008/05/26 17:11:09 nth Exp $
  */
 
 /*
@@ -66,11 +66,12 @@
 #include <vm/vm_map.h>
 #include <vm/vm_pager.h>
 #include <vm/vm_extern.h>
-#include <vm/vm_zone.h>
 
 #include <machine/stdarg.h>
 #include <machine/smp.h>
 
+static MALLOC_DEFINE(M_THREAD, "thread", "lwkt threads");
+
 static int untimely_switch = 0;
 #ifdef	INVARIANTS
 static int panic_on_cscount = 0;
@@ -82,6 +83,7 @@ static __int64_t preempt_weird = 0;
 static __int64_t token_contention_count = 0;
 static __int64_t mplock_contention_count = 0;
 static int lwkt_use_spin_port;
+static struct objcache *thread_cache;
 
 /*
  * We can make all thread ports use the spin backend instead of the thread
@@ -154,6 +156,42 @@ _lwkt_enqueue(thread_t td)
     }
 }
 
+static __boolean_t
+_lwkt_thread_ctor(void *obj, void *privdata, int ocflags)
+{
+	struct thread *td = (struct thread *)obj;
+
+	td->td_kstack = NULL;
+	td->td_kstack_size = 0;
+	td->td_flags = TDF_ALLOCATED_THREAD;
+	return (1);
+}
+
+static void
+_lwkt_thread_dtor(void *obj, void *privdata)
+{
+	struct thread *td = (struct thread *)obj;
+
+	KASSERT(td->td_flags & TDF_ALLOCATED_THREAD,
+	    ("_lwkt_thread_dtor: not allocated from objcache"));
+	KASSERT((td->td_flags & TDF_ALLOCATED_STACK) && td->td_kstack &&
+		td->td_kstack_size > 0,
+	    ("_lwkt_thread_dtor: corrupted stack"));
+	kmem_free(&kernel_map, (vm_offset_t)td->td_kstack, td->td_kstack_size);
+}
+
+/*
+ * Initialize the lwkt s/system.
+ */
+void
+lwkt_init(void)
+{
+    /* An objcache has 2 magazines per CPU so divide cache size by 2. */
+    thread_cache = objcache_create_mbacked(M_THREAD, sizeof(struct thread), 0,
+			CACHE_NTHREADS/2, _lwkt_thread_ctor, _lwkt_thread_dtor,
+			NULL);
+}
+
 /*
  * Schedule a thread to run.  As the current thread we can always safely
  * schedule ourselves, and a shortcut procedure is provided for that
@@ -212,25 +250,13 @@ thread_t
 lwkt_alloc_thread(struct thread *td, int stksize, int cpu, int flags)
 {
     void *stack;
-    globaldata_t gd = mycpu;
 
     if (td == NULL) {
-	crit_enter_gd(gd);
-	if (gd->gd_tdfreecount > 0) {
-	    --gd->gd_tdfreecount;
-	    td = TAILQ_FIRST(&gd->gd_tdfreeq);
-	    KASSERT(td != NULL && (td->td_flags & TDF_RUNNING) == 0,
-		("lwkt_alloc_thread: unexpected NULL or corrupted td"));
-	    TAILQ_REMOVE(&gd->gd_tdfreeq, td, td_threadq);
-	    crit_exit_gd(gd);
-	    flags |= td->td_flags & (TDF_ALLOCATED_STACK|TDF_ALLOCATED_THREAD);
-	} else {
-	    crit_exit_gd(gd);
-	    td = zalloc(thread_zone);
-	    td->td_kstack = NULL;
-	    td->td_kstack_size = 0;
-	    flags |= TDF_ALLOCATED_THREAD;
-	}
+    	td = objcache_get(thread_cache, M_WAITOK);
+    	KASSERT((td->td_flags &
+		 (TDF_ALLOCATED_THREAD|TDF_RUNNING)) == TDF_ALLOCATED_THREAD,
+		("lwkt_alloc_thread: corrupted td flags 0x%X", td->td_flags));
+    	flags |= td->td_flags & (TDF_ALLOCATED_THREAD|TDF_ALLOCATED_STACK);
     }
     if ((stack = td->td_kstack) != NULL && td->td_kstack_size != stksize) {
 	if (flags & TDF_ALLOCATED_STACK) {
@@ -353,29 +379,18 @@ lwkt_wait_free(thread_t td)
 void
 lwkt_free_thread(thread_t td)
 {
-    struct globaldata *gd = mycpu;
-
     KASSERT((td->td_flags & TDF_RUNNING) == 0,
 	("lwkt_free_thread: did not exit! %p", td));
 
-    crit_enter_gd(gd);
-    if (gd->gd_tdfreecount < CACHE_NTHREADS &&
-	(td->td_flags & TDF_ALLOCATED_THREAD)
-    ) {
-	++gd->gd_tdfreecount;
-	TAILQ_INSERT_HEAD(&gd->gd_tdfreeq, td, td_threadq);
-	crit_exit_gd(gd);
-    } else {
-	crit_exit_gd(gd);
-	if (td->td_kstack && (td->td_flags & TDF_ALLOCATED_STACK)) {
-	    kmem_free(&kernel_map, (vm_offset_t)td->td_kstack, td->td_kstack_size);
-	    /* gd invalid */
-	    td->td_kstack = NULL;
-	    td->td_kstack_size = 0;
-	}
-	if (td->td_flags & TDF_ALLOCATED_THREAD) {
-	    zfree(thread_zone, td);
-	}
+    if (td->td_flags & TDF_ALLOCATED_THREAD) {
+    	objcache_put(thread_cache, td);
+    } else if (td->td_flags & TDF_ALLOCATED_STACK) {
+	/* client-allocated struct with internally allocated stack */
+	KASSERT(td->td_kstack && td->td_kstack_size > 0,
+	    ("lwkt_free_thread: corrupted stack"));
+	kmem_free(&kernel_map, (vm_offset_t)td->td_kstack, td->td_kstack_size);
+	td->td_kstack = NULL;
+	td->td_kstack_size = 0;
     }
 }
 
@@ -1271,8 +1286,7 @@ lwkt_exit(void)
     gd = mycpu;
     lwkt_remove_tdallq(td);
     if (td->td_flags & TDF_ALLOCATED_THREAD) {
-	++gd->gd_tdfreecount;
-	TAILQ_INSERT_TAIL(&gd->gd_tdfreeq, td, td_threadq);
+	objcache_put(thread_cache, td);
     }
     cpu_thread_exit();
 }
diff --git a/sys/sys/globaldata.h b/sys/sys/globaldata.h
index 01e66c6d9a..b1fb2299c8 100644
--- a/sys/sys/globaldata.h
+++ b/sys/sys/globaldata.h
@@ -55,7 +55,7 @@
  * SUCH DAMAGE.
  *
  * $FreeBSD: src/sys/i386/include/globaldata.h,v 1.11.2.1 2000/05/16 06:58:10 dillon Exp $
- * $DragonFly: src/sys/sys/globaldata.h,v 1.47 2007/10/16 11:12:59 sephe Exp $
+ * $DragonFly: src/sys/sys/globaldata.h,v 1.48 2008/05/26 17:11:09 nth Exp $
  */
 
 #ifndef _SYS_GLOBALDATA_H_
@@ -125,11 +125,11 @@ struct pipe;
 struct globaldata {
 	struct privatespace *gd_prvspace;	/* self-reference */
 	struct thread	*gd_curthread;
-	int		gd_tdfreecount;		/* new thread cache */
+	int		gd_unused00;
 	__uint32_t	gd_reqflags;		/* (see note above) */
-	void		*gd_unused00;
+	void		*gd_unused00B;
 	lwkt_queue	gd_tdallq;		/* all threads */
-	lwkt_queue	gd_tdfreeq;		/* new thread cache */
+	lwkt_queue	gd_unused00C;
 	lwkt_queue	gd_tdrunq[32];		/* runnable threads */
 	__uint32_t	gd_runqmask;		/* which queues? */
 	__uint32_t	gd_cpuid;
diff --git a/sys/sys/thread.h b/sys/sys/thread.h
index 5267a0bf05..2fbe8fd8c6 100644
--- a/sys/sys/thread.h
+++ b/sys/sys/thread.h
@@ -7,7 +7,7 @@
  * Types which must already be defined when this header is included by
  * userland:	struct md_thread
  * 
- * $DragonFly: src/sys/sys/thread.h,v 1.92 2008/05/09 06:35:10 dillon Exp $
+ * $DragonFly: src/sys/sys/thread.h,v 1.93 2008/05/26 17:11:09 nth Exp $
  */
 
 #ifndef _SYS_THREAD_H_
@@ -267,8 +267,8 @@ struct thread {
 #define TDF_TSLEEPQ		0x0080	/* on a tsleep wait queue */
 
 #define TDF_SYSTHREAD		0x0100	/* system thread */
-#define TDF_ALLOCATED_THREAD	0x0200	/* zalloc allocated thread */
-#define TDF_ALLOCATED_STACK	0x0400	/* zalloc allocated stack */
+#define TDF_ALLOCATED_THREAD	0x0200	/* objcache allocated thread */
+#define TDF_ALLOCATED_STACK	0x0400	/* objcache allocated stack */
 #define TDF_VERBOSE		0x0800	/* verbose on exit */
 #define TDF_DEADLKTREAT		0x1000	/* special lockmgr deadlock treatment */
 #define TDF_STOPREQ		0x2000	/* suspend_kproc */
@@ -321,15 +321,7 @@ struct thread {
 
 #define IN_CRITICAL_SECT(td)	((td)->td_pri >= TDPRI_CRIT)
 
-#ifdef _KERNEL
-
-extern struct vm_zone	*thread_zone;
-
-#endif
-
-/*
- * Applies both to the kernel and to liblwkt.
- */
+extern void lwkt_init(void);
 extern struct thread *lwkt_alloc_thread(struct thread *, int, int, int);
 extern void lwkt_init_thread(struct thread *, void *, int, int,
 			     struct globaldata *);