kernel - Improve regressions in usched_dfly (1)

* The new scheduler is MP locked at a very fine-grain.  The old scheduler
  had a global spinlock which effectively serialized competing cores during
  exit/wait sequences.

  With the new scheduler this serialization is gone and resulted in a
  vfork performance regression due to a fallback tsleep loop in the
  reaper.

* This fixes the problem with an explicit signal bit for tsleep/wakeup.
  The sequence is avoided if the reaper determines the thread has
  already completed its exit.

diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c
index 7f98afb..5d99503 100644 (file)
--- a/sys/kern/kern_exit.c
+++ b/sys/kern/kern_exit.c
@@ -711,7 +711,9 @@ lwp_exit(int masterexit)
  * switchout.
  *
  * At the point TDF_EXITING is set a complete exit is accomplished when
- * TDF_RUNNING and TDF_PREEMPT_LOCK are both clear.
+ * TDF_RUNNING and TDF_PREEMPT_LOCK are both clear.  td_mpflags has two
+ * post-switch interlock flags that can be used to wait for the TDF_
+ * flags to clear.
  *
  * Returns non-zero on success, and zero if the caller needs to retry
  * the lwp_wait().
@@ -720,47 +722,59 @@ static int
 lwp_wait(struct lwp *lp)
 {
        struct thread *td = lp->lwp_thread;;
+       u_int mpflags;
 
        KKASSERT(lwkt_preempted_proc() != lp);
 
        /*
-        * Wait until the lp has entered its low level exit and wait
-        * until other cores with refs on the lp (e.g. for ps or signaling)
-        * release them.
+        * This bit of code uses the thread destruction interlock
+        * managed by lwkt_switch_return() to wait for the lwp's
+        * thread to completely disengage.
+        *
+        * It is possible for us to race another cpu core so we
+        * have to do this correctly.
         */
-       if (lp->lwp_lock > 0) {
-               tsleep(lp, 0, "lwpwait1", 1);
-               return(0);
+       for (;;) {
+               mpflags = td->td_mpflags;
+               cpu_ccfence();
+               if (mpflags & TDF_MP_EXITSIG)
+                       break;
+               tsleep_interlock(td, 0);
+               if (atomic_cmpset_int(&td->td_mpflags, mpflags,
+                                     mpflags | TDF_MP_EXITWAIT)) {
+                       tsleep(td, PINTERLOCKED, "lwpxt", 0);
+               }
        }
 
        /*
-        * Wait until the thread is no longer references and no longer
-        * runnable or preempted (i.e. finishes its low level exit).
+        * We've already waited for the core exit but there can still
+        * be other refs from e.g. process scans and such.
         */
+       if (lp->lwp_lock > 0) {
+               tsleep(lp, 0, "lwpwait1", 1);
+               return(0);
+       }
        if (td->td_refs) {
                tsleep(td, 0, "lwpwait2", 1);
                return(0);
        }
 
        /*
-        * The lwp's thread may still be in the middle
-        * of switching away, we can't rip its stack out from
-        * under it until TDF_EXITING is set and both
-        * TDF_RUNNING and TDF_PREEMPT_LOCK are clear.
-        * TDF_PREEMPT_LOCK must be checked because TDF_RUNNING
-        * will be cleared temporarily if a thread gets
-        * preempted.
+        * Now that we have the thread destruction interlock these flags
+        * really should already be cleaned up, keep a check for safety.
         *
-        * YYY no wakeup occurs, so we simply return failure
-        * and let the caller deal with sleeping and calling
-        * us again.
+        * We can't rip its stack out from under it until TDF_EXITING is
+        * set and both TDF_RUNNING and TDF_PREEMPT_LOCK are clear.
+        * TDF_PREEMPT_LOCK must be checked because TDF_RUNNING
+        * will be cleared temporarily if a thread gets preempted.
         */
-       if ((td->td_flags & (TDF_RUNNING |
-                            TDF_PREEMPT_LOCK |
-                            TDF_EXITING)) != TDF_EXITING) {
-               tsleep(lp, 0, "lwpwait2", 1);
+       while ((td->td_flags & (TDF_RUNNING |
+                               TDF_PREEMPT_LOCK |
+                               TDF_EXITING)) != TDF_EXITING) {
+               tsleep(lp, 0, "lwpwait3", 1);
                return (0);
        }
+
        KASSERT((td->td_flags & (TDF_RUNQ|TDF_TSLEEPQ)) == 0,
                ("lwp_wait: td %p (%s) still on run or sleep queue",
                td, td->td_comm));

diff --git a/sys/kern/lwkt_thread.c b/sys/kern/lwkt_thread.c
index 7414c2c..7f3fc32 100644 (file)
--- a/sys/kern/lwkt_thread.c
+++ b/sys/kern/lwkt_thread.c
@@ -977,6 +977,31 @@ lwkt_switch_return(thread_t otd)
 #else
        otd->td_flags &= ~TDF_RUNNING;
 #endif
+
+       /*
+        * Final exit validations (see lwp_wait()).  Note that otd becomes
+        * invalid the *instant* we set TDF_MP_EXITSIG.
+        */
+       while (otd->td_flags & TDF_EXITING) {
+               u_int mpflags;
+
+               mpflags = otd->td_mpflags;
+               cpu_ccfence();
+
+               if (mpflags & TDF_MP_EXITWAIT) {
+                       if (atomic_cmpset_int(&otd->td_mpflags, mpflags,
+                                             mpflags | TDF_MP_EXITSIG)) {
+                               wakeup(otd);
+                               break;
+                       }
+               } else {
+                       if (atomic_cmpset_int(&otd->td_mpflags, mpflags,
+                                             mpflags | TDF_MP_EXITSIG)) {
+                               wakeup(otd);
+                               break;
+                       }
+               }
+       }
 }
 
 /*

diff --git a/sys/sys/thread.h b/sys/sys/thread.h
index 4e244a5..8bd5f28 100644 (file)
--- a/sys/sys/thread.h
+++ b/sys/sys/thread.h
@@ -376,6 +376,8 @@ struct thread {
 
 #define TDF_MP_STOPREQ         0x00000001      /* suspend_kproc */
 #define TDF_MP_WAKEREQ         0x00000002      /* resume_kproc */
+#define TDF_MP_EXITWAIT                0x00000004      /* reaper, see lwp_wait() */
+#define TDF_MP_EXITSIG         0x00000008      /* reaper, see lwp_wait() */
 
 /*
  * Thread priorities.  Typically only one thread from any given