kernel - usched_dfly revamp (2), reduce token collisions

* Add wakeup_start_delayed() and wakeup_end_delayed().  These functions
  will attempt to delay any wakeup() calls made between them.

  Use the functions in the unix domain socket send code.

* This removes a lot of volatility from monster's 48:48 pgbench tests by
  delaying the wakeup()s related to a unix domain socket write until after
  the pool token has been released.

* Adjust usched_dfly parameters.  In particular, weight2 can be higher now.

diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index 8f3d49f..3600633 100644 (file)
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -463,6 +463,14 @@ tsleep(const volatile void *ident, int flags, const char *wmesg, int timo)
        struct callout thandle;
 
        /*
+        * Currently a severe hack.  Make sure any delayed wakeups
+        * are flushed before we sleep or we might deadlock on whatever
+        * event we are sleeping on.
+        */
+       if (td->td_flags & TDF_DELAYED_WAKEUP)
+               wakeup_end_delayed();
+
+       /*
         * NOTE: removed KTRPOINT, it could cause races due to blocking
         * even in stable.  Just scrap it for now.
         */
@@ -943,7 +951,17 @@ done:
 void
 wakeup(const volatile void *ident)
 {
-    _wakeup(__DEALL(ident), PWAKEUP_ENCODE(0, mycpu->gd_cpuid));
+    globaldata_t gd = mycpu;
+    thread_t td = gd->gd_curthread;
+
+    if (td && (td->td_flags & TDF_DELAYED_WAKEUP)) {
+       if (!atomic_cmpset_ptr(&gd->gd_delayed_wakeup[0], NULL, ident)) {
+           if (!atomic_cmpset_ptr(&gd->gd_delayed_wakeup[1], NULL, ident))
+               _wakeup(__DEALL(ident), PWAKEUP_ENCODE(0, gd->gd_cpuid));
+       }
+       return;
+    }
+    _wakeup(__DEALL(ident), PWAKEUP_ENCODE(0, gd->gd_cpuid));
 }
 
 /*
@@ -1046,6 +1064,38 @@ wakeup_domain_one(const volatile void *ident, int domain)
            PWAKEUP_ENCODE(domain, mycpu->gd_cpuid) | PWAKEUP_ONE);
 }
 
+void
+wakeup_start_delayed(void)
+{
+    globaldata_t gd = mycpu;
+
+    crit_enter();
+    gd->gd_curthread->td_flags |= TDF_DELAYED_WAKEUP;
+    crit_exit();
+}
+
+void
+wakeup_end_delayed(void)
+{
+    globaldata_t gd = mycpu;
+
+    if (gd->gd_curthread->td_flags & TDF_DELAYED_WAKEUP) {
+       crit_enter();
+       gd->gd_curthread->td_flags &= ~TDF_DELAYED_WAKEUP;
+       if (gd->gd_delayed_wakeup[0] || gd->gd_delayed_wakeup[1]) {
+           if (gd->gd_delayed_wakeup[0]) {
+                   wakeup(gd->gd_delayed_wakeup[0]);
+                   gd->gd_delayed_wakeup[0] = NULL;
+           }
+           if (gd->gd_delayed_wakeup[1]) {
+                   wakeup(gd->gd_delayed_wakeup[1]);
+                   gd->gd_delayed_wakeup[1] = NULL;
+           }
+       }
+       crit_exit();
+    }
+}
+
 /*
  * setrunnable()
  *

diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c
index 8d3f16b..4cc1ec0 100644 (file)
--- a/sys/kern/uipc_usrreq.c
+++ b/sys/kern/uipc_usrreq.c
@@ -453,6 +453,8 @@ uipc_send(netmsg_t msg)
                goto release;
        }
 
+       wakeup_start_delayed();
+
        if (control && (error = unp_internalize(control, msg->send.nm_td)))
                goto release;
 
@@ -582,9 +584,9 @@ uipc_send(netmsg_t msg)
 
        if (control && error != 0)
                unp_dispose(control);
-
 release:
        lwkt_relpooltoken(unp);
+       wakeup_end_delayed();
 done:
 
        if (control)

diff --git a/sys/kern/usched_dfly.c b/sys/kern/usched_dfly.c
index 4d180a0..53e52f9 100644 (file)
--- a/sys/kern/usched_dfly.c
+++ b/sys/kern/usched_dfly.c
@@ -252,10 +252,10 @@ SYSCTL_INT(_debug, OID_AUTO, dfly_chooser, CTLFLAG_RW,
 #ifdef SMP
 static int usched_dfly_smt = 0;
 static int usched_dfly_cache_coherent = 0;
-static int usched_dfly_weight1 = 30;   /* keep thread on current cpu */
-static int usched_dfly_weight2 = 15;   /* synchronous peer's current cpu */
+static int usched_dfly_weight1 = 50;   /* keep thread on current cpu */
+static int usched_dfly_weight2 = 30;   /* synchronous peer's current cpu */
 static int usched_dfly_weight3 = 10;   /* number of threads on queue */
-static int usched_dfly_weight4 = 50;   /* availability of idle cores */
+static int usched_dfly_weight4 = 40;   /* availability of idle cores */
 static int usched_dfly_features = 0x8F;        /* allow pulls */
 #endif
 static int usched_dfly_rrinterval = (ESTCPUFREQ + 9) / 10;

diff --git a/sys/sys/globaldata.h b/sys/sys/globaldata.h
index d1f7393..5fe3815 100644 (file)
--- a/sys/sys/globaldata.h
+++ b/sys/sys/globaldata.h
@@ -166,7 +166,8 @@ struct globaldata {
        int             gd_ireserved[7];
        const char      *gd_infomsg;            /* debugging */
        struct lwkt_tokref gd_handoff;          /* hand-off tokref */
-       void            *gd_preserved[8];       /* future fields */
+       void            *gd_delayed_wakeup[2];
+       void            *gd_preserved[6];       /* future fields */
        /* extended by <machine/globaldata.h> */
 };
 

diff --git a/sys/sys/systm.h b/sys/sys/systm.h
index b16119e..cb068e2 100644 (file)
--- a/sys/sys/systm.h
+++ b/sys/sys/systm.h
@@ -370,6 +370,8 @@ void        wakeup_oncpu (struct globaldata *gd, const volatile void *chan);
 void   wakeup_oncpu_one (struct globaldata *gd, const volatile void *chan);
 void   wakeup_domain (const volatile void *chan, int domain);
 void   wakeup_domain_one (const volatile void *chan, int domain);
+void   wakeup_start_delayed(void);
+void   wakeup_end_delayed(void);
 
 /*
  * Common `cdev_t' stuff are declared here to avoid #include poisoning

diff --git a/sys/sys/thread.h b/sys/sys/thread.h
index 8bd5f28..cc1f794 100644 (file)
--- a/sys/sys/thread.h
+++ b/sys/sys/thread.h
@@ -371,7 +371,7 @@ struct thread {
 #define TDF_EXITING            0x00400000      /* thread exiting */
 #define TDF_USINGFP            0x00800000      /* thread using fp coproc */
 #define TDF_KERNELFP           0x01000000      /* kernel using fp coproc */
-#define TDF_UNUSED02000000     0x02000000
+#define TDF_DELAYED_WAKEUP     0x02000000
 #define TDF_CRYPTO             0x04000000      /* crypto thread */
 
 #define TDF_MP_STOPREQ         0x00000001      /* suspend_kproc */