From c9aa7a821dc01d4a649d667165ccf73c946aace6 Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Tue, 8 Jun 2010 21:37:24 -0700 Subject: [PATCH] kernel - MPSAFE work - Add global tokens * Add global tokens for various subsystems but do not use them (yet). Tie in mpsafe and collision count sysctls. * Add a collision counter to the lwkt_token structure. * Simplify the lwkt_token initializer macros. * Minor namespace cleanup for sys/thread.h, place an #ifdef _KERNEL around the procedure prototypes. --- sys/kern/lwkt_token.c | 68 ++++++++++++++++++- sys/platform/pc32/i386/busdma_machdep.c | 3 +- sys/platform/pc64/x86_64/busdma_machdep.c | 3 +- .../vkernel/platform/busdma_machdep.c | 3 +- .../vkernel64/platform/busdma_machdep.c | 3 +- sys/sys/thread.h | 34 +++++++--- 6 files changed, 97 insertions(+), 17 deletions(-) diff --git a/sys/kern/lwkt_token.c b/sys/kern/lwkt_token.c index 5956c1a3a1..738a95c1f3 100644 --- a/sys/kern/lwkt_token.c +++ b/sys/kern/lwkt_token.c @@ -114,6 +114,68 @@ KTR_INFO(KTR_TOKENS, tokens, contention_stop, 7, UNCONTENDED_STRING, sizeof(void SYSCTL_INT(_lwkt, OID_AUTO, token_debug, CTLFLAG_RW, &token_debug, 0, ""); #endif +/* + * Global tokens. These replace the MP lock for major subsystem locking. + * These tokens are initially used to lockup both global and individual + * operations. + * + * Once individual structures get their own locks these tokens are used + * only to protect global lists & other variables and to interlock + * allocations and teardowns and such. + * + * The UP initializer causes token acquisition to also acquire the MP lock + * for maximum compatibility. The feature may be enabled and disabled at + * any time, the MP state is copied to the tokref when the token is acquired + * and will not race against sysctl changes. + */ +struct lwkt_token pmap_token = LWKT_TOKEN_UP_INITIALIZER; +struct lwkt_token dev_token = LWKT_TOKEN_UP_INITIALIZER; +struct lwkt_token vm_page_token = LWKT_TOKEN_UP_INITIALIZER; +struct lwkt_token vm_object_token = LWKT_TOKEN_UP_INITIALIZER; +struct lwkt_token vm_map_token = LWKT_TOKEN_UP_INITIALIZER; +struct lwkt_token proc_token = LWKT_TOKEN_UP_INITIALIZER; +struct lwkt_token tty_token = LWKT_TOKEN_UP_INITIALIZER; +struct lwkt_token vnode_token = LWKT_TOKEN_UP_INITIALIZER; + +SYSCTL_INT(_lwkt, OID_AUTO, pmap_mpsafe, + CTLFLAG_RW, &pmap_token.t_flags, 0, ""); +SYSCTL_INT(_lwkt, OID_AUTO, dev_mpsafe, + CTLFLAG_RW, &dev_token.t_flags, 0, ""); +SYSCTL_INT(_lwkt, OID_AUTO, vm_page_mpsafe, + CTLFLAG_RW, &vm_page_token.t_flags, 0, ""); +SYSCTL_INT(_lwkt, OID_AUTO, vm_object_mpsafe, + CTLFLAG_RW, &vm_object_token.t_flags, 0, ""); +SYSCTL_INT(_lwkt, OID_AUTO, vm_map_mpsafe, + CTLFLAG_RW, &vm_map_token.t_flags, 0, ""); +SYSCTL_INT(_lwkt, OID_AUTO, proc_mpsafe, + CTLFLAG_RW, &proc_token.t_flags, 0, ""); +SYSCTL_INT(_lwkt, OID_AUTO, tty_mpsafe, + CTLFLAG_RW, &tty_token.t_flags, 0, ""); +SYSCTL_INT(_lwkt, OID_AUTO, vnode_mpsafe, + CTLFLAG_RW, &vnode_token.t_flags, 0, ""); + +/* + * The collision count is bumped every time the LWKT scheduler fails + * to acquire needed tokens in addition to a normal lwkt_gettoken() + * stall. + */ +SYSCTL_LONG(_lwkt, OID_AUTO, pmap_collisions, + CTLFLAG_RW, &pmap_token.t_collisions, 0, ""); +SYSCTL_LONG(_lwkt, OID_AUTO, dev_collisions, + CTLFLAG_RW, &dev_token.t_collisions, 0, ""); +SYSCTL_LONG(_lwkt, OID_AUTO, vm_page_collisions, + CTLFLAG_RW, &vm_page_token.t_collisions, 0, ""); +SYSCTL_LONG(_lwkt, OID_AUTO, vm_object_collisions, + CTLFLAG_RW, &vm_object_token.t_collisions, 0, ""); +SYSCTL_LONG(_lwkt, OID_AUTO, vm_map_collisions, + CTLFLAG_RW, &vm_map_token.t_collisions, 0, ""); +SYSCTL_LONG(_lwkt, OID_AUTO, proc_collisions, + CTLFLAG_RW, &proc_token.t_collisions, 0, ""); +SYSCTL_LONG(_lwkt, OID_AUTO, tty_collisions, + CTLFLAG_RW, &tty_token.t_collisions, 0, ""); +SYSCTL_LONG(_lwkt, OID_AUTO, vnode_collisions, + CTLFLAG_RW, &vnode_token.t_collisions, 0, ""); + /* * Return a pool token given an address */ @@ -149,7 +211,9 @@ _lwkt_tokref_init(lwkt_tokref_t ref, lwkt_token_t tok, thread_t td) * tokens that the thread had acquired prior to going to sleep. * * The scheduler is responsible for maintaining the MP lock count, so - * we don't need to deal with tr_flags here. + * we don't need to deal with tr_flags here. We also do not do any + * logging here. The logging done by lwkt_gettoken() is plenty good + * enough to get a feel for it. * * Called from a critical section. */ @@ -195,6 +259,7 @@ lwkt_getalltokens(thread_t td) * Otherwise we failed to acquire all the tokens. * Undo and return. */ + atomic_add_long(&tok->t_collisions, 1); lwkt_relalltokens(td); return(FALSE); } @@ -333,6 +398,7 @@ _lwkt_gettokref(lwkt_tokref_t ref, thread_t td) * return tr_tok->t_ref should be assigned to this specific * ref. */ + atomic_add_long(&ref->tr_tok->t_collisions, 1); logtoken(fail, ref); lwkt_yield(); logtoken(succ, ref); diff --git a/sys/platform/pc32/i386/busdma_machdep.c b/sys/platform/pc32/i386/busdma_machdep.c index 2721b20b97..6f02811815 100644 --- a/sys/platform/pc32/i386/busdma_machdep.c +++ b/sys/platform/pc32/i386/busdma_machdep.c @@ -138,8 +138,7 @@ struct bounce_zone { #define BZ_UNLOCK(bz) crit_exit() #endif -static struct lwkt_token bounce_zone_tok = - LWKT_TOKEN_MP_INITIALIZER(bounce_zone_tok); +static struct lwkt_token bounce_zone_tok = LWKT_TOKEN_MP_INITIALIZER; static int busdma_zonecount; static STAILQ_HEAD(, bounce_zone) bounce_zone_list = STAILQ_HEAD_INITIALIZER(bounce_zone_list); diff --git a/sys/platform/pc64/x86_64/busdma_machdep.c b/sys/platform/pc64/x86_64/busdma_machdep.c index 2721b20b97..6f02811815 100644 --- a/sys/platform/pc64/x86_64/busdma_machdep.c +++ b/sys/platform/pc64/x86_64/busdma_machdep.c @@ -138,8 +138,7 @@ struct bounce_zone { #define BZ_UNLOCK(bz) crit_exit() #endif -static struct lwkt_token bounce_zone_tok = - LWKT_TOKEN_MP_INITIALIZER(bounce_zone_tok); +static struct lwkt_token bounce_zone_tok = LWKT_TOKEN_MP_INITIALIZER; static int busdma_zonecount; static STAILQ_HEAD(, bounce_zone) bounce_zone_list = STAILQ_HEAD_INITIALIZER(bounce_zone_list); diff --git a/sys/platform/vkernel/platform/busdma_machdep.c b/sys/platform/vkernel/platform/busdma_machdep.c index 24a2bdcbcf..4e1bcc59ee 100644 --- a/sys/platform/vkernel/platform/busdma_machdep.c +++ b/sys/platform/vkernel/platform/busdma_machdep.c @@ -128,8 +128,7 @@ struct bounce_zone { #define BZ_UNLOCK(bz) crit_exit() #endif -static struct lwkt_token bounce_zone_tok = - LWKT_TOKEN_MP_INITIALIZER(bounce_zone_tok); +static struct lwkt_token bounce_zone_tok = LWKT_TOKEN_MP_INITIALIZER; static int busdma_zonecount; static STAILQ_HEAD(, bounce_zone) bounce_zone_list = STAILQ_HEAD_INITIALIZER(bounce_zone_list); diff --git a/sys/platform/vkernel64/platform/busdma_machdep.c b/sys/platform/vkernel64/platform/busdma_machdep.c index b0ff3e8beb..05c391bdda 100644 --- a/sys/platform/vkernel64/platform/busdma_machdep.c +++ b/sys/platform/vkernel64/platform/busdma_machdep.c @@ -126,8 +126,7 @@ struct bounce_zone { #define BZ_UNLOCK(bz) crit_exit() #endif -static struct lwkt_token bounce_zone_tok = - LWKT_TOKEN_MP_INITIALIZER(bounce_zone_tok); +static struct lwkt_token bounce_zone_tok = LWKT_TOKEN_MP_INITIALIZER; static int busdma_zonecount; static STAILQ_HEAD(, bounce_zone) bounce_zone_list = STAILQ_HEAD_INITIALIZER(bounce_zone_list); diff --git a/sys/sys/thread.h b/sys/sys/thread.h index 7525f188f0..5731de09bf 100644 --- a/sys/sys/thread.h +++ b/sys/sys/thread.h @@ -103,6 +103,7 @@ struct intrframe; typedef struct lwkt_token { struct lwkt_tokref *t_ref; /* Owning ref or NULL */ intptr_t t_flags; /* MP lock required */ + long t_collisions; /* Collision counter */ } lwkt_token; #define LWKT_TOKEN_MPSAFE 0x0001 @@ -112,16 +113,18 @@ typedef struct lwkt_token { * UP - Not MPSAFE (full MP lock will also be acquired) * MP - Is MPSAFE (only the token will be acquired) */ -#define LWKT_TOKEN_UP_INITIALIZER(head) \ +#define LWKT_TOKEN_UP_INITIALIZER \ { \ .t_ref = NULL, \ - .t_flags = 0 \ + .t_flags = 0, \ + .t_collisions = 0 \ } -#define LWKT_TOKEN_MP_INITIALIZER(head) \ +#define LWKT_TOKEN_MP_INITIALIZER \ { \ .t_ref = NULL, \ - .t_flags = LWKT_TOKEN_MPSAFE \ + .t_flags = LWKT_TOKEN_MPSAFE, \ + .t_collisions = 0 \ } #define ASSERT_LWKT_TOKEN_HELD(tok) \ @@ -338,14 +341,29 @@ struct thread { #define TDPRI_MASK 31 #define TDPRI_CRIT 32 /* high bits of td_pri used for crit */ -#ifdef _KERNEL #define LWKT_THREAD_STACK (UPAGES * PAGE_SIZE) -#endif #define CACHE_NTHREADS 6 #define IN_CRITICAL_SECT(td) ((td)->td_pri >= TDPRI_CRIT) +#ifdef _KERNEL + +/* + * Global tokens + */ +extern struct lwkt_token pmap_token; +extern struct lwkt_token dev_token; +extern struct lwkt_token vm_page_token; +extern struct lwkt_token vm_object_token; +extern struct lwkt_token vm_map_token; +extern struct lwkt_token proc_token; +extern struct lwkt_token tty_token; +extern struct lwkt_token vnode_token; + +/* + * Procedures + */ extern void lwkt_init(void); extern struct thread *lwkt_alloc_thread(struct thread *, int, int, int); extern void lwkt_init_thread(struct thread *, void *, int, int, @@ -402,9 +420,7 @@ extern int lwkt_send_ipiq3_mask(cpumask_t, ipifunc3_t, void *, int); extern void lwkt_wait_ipiq(struct globaldata *, int); extern int lwkt_seq_ipiq(struct globaldata *); extern void lwkt_process_ipiq(void); -#ifdef _KERNEL extern void lwkt_process_ipiq_frame(struct intrframe *); -#endif extern void lwkt_smp_stopped(void); extern void lwkt_synchronize_ipiqs(const char *); @@ -427,3 +443,5 @@ extern void lwkt_remove_tdallq (struct thread *); #endif +#endif + -- 2.41.0