From b0da0c887b6297fd4334569862260a8c0fee1ec6 Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Fri, 21 Sep 2012 18:34:09 -0700 Subject: [PATCH] kernel - usched_dfly revamp (2), reduce token collisions * Add wakeup_start_delayed() and wakeup_end_delayed(). These functions will attempt to delay any wakeup() calls made between them. Use the functions in the unix domain socket send code. * This removes a lot of volatility from monster's 48:48 pgbench tests by delaying the wakeup()s related to a unix domain socket write until after the pool token has been released. * Adjust usched_dfly parameters. In particular, weight2 can be higher now. --- sys/kern/kern_synch.c | 52 +++++++++++++++++++++++++++++++++++++++++- sys/kern/uipc_usrreq.c | 4 +++- sys/kern/usched_dfly.c | 6 ++--- sys/sys/globaldata.h | 3 ++- sys/sys/systm.h | 2 ++ sys/sys/thread.h | 2 +- 6 files changed, 62 insertions(+), 7 deletions(-) diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c index 8f3d49f06c..36006336b5 100644 --- a/sys/kern/kern_synch.c +++ b/sys/kern/kern_synch.c @@ -462,6 +462,14 @@ tsleep(const volatile void *ident, int flags, const char *wmesg, int timo) int oldpri; struct callout thandle; + /* + * Currently a severe hack. Make sure any delayed wakeups + * are flushed before we sleep or we might deadlock on whatever + * event we are sleeping on. + */ + if (td->td_flags & TDF_DELAYED_WAKEUP) + wakeup_end_delayed(); + /* * NOTE: removed KTRPOINT, it could cause races due to blocking * even in stable. Just scrap it for now. @@ -943,7 +951,17 @@ done: void wakeup(const volatile void *ident) { - _wakeup(__DEALL(ident), PWAKEUP_ENCODE(0, mycpu->gd_cpuid)); + globaldata_t gd = mycpu; + thread_t td = gd->gd_curthread; + + if (td && (td->td_flags & TDF_DELAYED_WAKEUP)) { + if (!atomic_cmpset_ptr(&gd->gd_delayed_wakeup[0], NULL, ident)) { + if (!atomic_cmpset_ptr(&gd->gd_delayed_wakeup[1], NULL, ident)) + _wakeup(__DEALL(ident), PWAKEUP_ENCODE(0, gd->gd_cpuid)); + } + return; + } + _wakeup(__DEALL(ident), PWAKEUP_ENCODE(0, gd->gd_cpuid)); } /* @@ -1046,6 +1064,38 @@ wakeup_domain_one(const volatile void *ident, int domain) PWAKEUP_ENCODE(domain, mycpu->gd_cpuid) | PWAKEUP_ONE); } +void +wakeup_start_delayed(void) +{ + globaldata_t gd = mycpu; + + crit_enter(); + gd->gd_curthread->td_flags |= TDF_DELAYED_WAKEUP; + crit_exit(); +} + +void +wakeup_end_delayed(void) +{ + globaldata_t gd = mycpu; + + if (gd->gd_curthread->td_flags & TDF_DELAYED_WAKEUP) { + crit_enter(); + gd->gd_curthread->td_flags &= ~TDF_DELAYED_WAKEUP; + if (gd->gd_delayed_wakeup[0] || gd->gd_delayed_wakeup[1]) { + if (gd->gd_delayed_wakeup[0]) { + wakeup(gd->gd_delayed_wakeup[0]); + gd->gd_delayed_wakeup[0] = NULL; + } + if (gd->gd_delayed_wakeup[1]) { + wakeup(gd->gd_delayed_wakeup[1]); + gd->gd_delayed_wakeup[1] = NULL; + } + } + crit_exit(); + } +} + /* * setrunnable() * diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c index 8d3f16b5cd..4cc1ec08d6 100644 --- a/sys/kern/uipc_usrreq.c +++ b/sys/kern/uipc_usrreq.c @@ -453,6 +453,8 @@ uipc_send(netmsg_t msg) goto release; } + wakeup_start_delayed(); + if (control && (error = unp_internalize(control, msg->send.nm_td))) goto release; @@ -582,9 +584,9 @@ uipc_send(netmsg_t msg) if (control && error != 0) unp_dispose(control); - release: lwkt_relpooltoken(unp); + wakeup_end_delayed(); done: if (control) diff --git a/sys/kern/usched_dfly.c b/sys/kern/usched_dfly.c index 4d180a0a84..53e52f95e5 100644 --- a/sys/kern/usched_dfly.c +++ b/sys/kern/usched_dfly.c @@ -252,10 +252,10 @@ SYSCTL_INT(_debug, OID_AUTO, dfly_chooser, CTLFLAG_RW, #ifdef SMP static int usched_dfly_smt = 0; static int usched_dfly_cache_coherent = 0; -static int usched_dfly_weight1 = 30; /* keep thread on current cpu */ -static int usched_dfly_weight2 = 15; /* synchronous peer's current cpu */ +static int usched_dfly_weight1 = 50; /* keep thread on current cpu */ +static int usched_dfly_weight2 = 30; /* synchronous peer's current cpu */ static int usched_dfly_weight3 = 10; /* number of threads on queue */ -static int usched_dfly_weight4 = 50; /* availability of idle cores */ +static int usched_dfly_weight4 = 40; /* availability of idle cores */ static int usched_dfly_features = 0x8F; /* allow pulls */ #endif static int usched_dfly_rrinterval = (ESTCPUFREQ + 9) / 10; diff --git a/sys/sys/globaldata.h b/sys/sys/globaldata.h index d1f739312f..5fe3815e1b 100644 --- a/sys/sys/globaldata.h +++ b/sys/sys/globaldata.h @@ -166,7 +166,8 @@ struct globaldata { int gd_ireserved[7]; const char *gd_infomsg; /* debugging */ struct lwkt_tokref gd_handoff; /* hand-off tokref */ - void *gd_preserved[8]; /* future fields */ + void *gd_delayed_wakeup[2]; + void *gd_preserved[6]; /* future fields */ /* extended by */ }; diff --git a/sys/sys/systm.h b/sys/sys/systm.h index b16119ecdf..cb068e28e0 100644 --- a/sys/sys/systm.h +++ b/sys/sys/systm.h @@ -370,6 +370,8 @@ void wakeup_oncpu (struct globaldata *gd, const volatile void *chan); void wakeup_oncpu_one (struct globaldata *gd, const volatile void *chan); void wakeup_domain (const volatile void *chan, int domain); void wakeup_domain_one (const volatile void *chan, int domain); +void wakeup_start_delayed(void); +void wakeup_end_delayed(void); /* * Common `cdev_t' stuff are declared here to avoid #include poisoning diff --git a/sys/sys/thread.h b/sys/sys/thread.h index 8bd5f288d5..cc1f794921 100644 --- a/sys/sys/thread.h +++ b/sys/sys/thread.h @@ -371,7 +371,7 @@ struct thread { #define TDF_EXITING 0x00400000 /* thread exiting */ #define TDF_USINGFP 0x00800000 /* thread using fp coproc */ #define TDF_KERNELFP 0x01000000 /* kernel using fp coproc */ -#define TDF_UNUSED02000000 0x02000000 +#define TDF_DELAYED_WAKEUP 0x02000000 #define TDF_CRYPTO 0x04000000 /* crypto thread */ #define TDF_MP_STOPREQ 0x00000001 /* suspend_kproc */ -- 2.41.0