vkernel - Fix corrupt tailq (vkernel64 only)

* Properly remove an exiting thread from any tsleep queue it might be
  enqueued on (due to tsleep_interlock() use cases) prior to exiting.

* Fixes tailq corruption which can occur with threaded programs.

* Fix is only applicable to vkernel64.  All other platforms already
  properly remove the thread.

* Assert that an exiting thread has been removed from any sleep queue
  before freeing it in kern_exit.c to catch any future cases.

diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c
index fd4e72b..57f1a4e 100644 (file)
--- a/sys/kern/kern_exit.c
+++ b/sys/kern/kern_exit.c
@@ -677,6 +677,8 @@ lwp_wait(struct lwp *lp)
                             TDF_EXITING|TDF_RUNQ)) != TDF_EXITING) {
                return (0);
        }
+       KASSERT((td->td_flags & TDF_TSLEEPQ) == 0,
+               ("lwp_wait: td %p (%s) still on sleep queue", td, td->td_comm));
        return (1);
 }
 

diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index 90eb88a..8f812d6 100644 (file)
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -119,7 +119,7 @@ static void endtsleep (void *);
 static void    loadav (void *arg);
 static void    schedcpu (void *arg);
 #ifdef SMP
-static void    tsleep_wakeup(struct thread *td);
+static void    tsleep_wakeup_remote(struct thread *td);
 #endif
 
 /*
@@ -413,6 +413,11 @@ tsleep_remove(thread_t td)
  * This function mus be called while in a critical section but if the
  * target thread is sleeping on a different cpu we cannot safely probe
  * td_flags.
+ *
+ * This function is only called from a different cpu via setrunnable()
+ * when the thread is in a known sleep.  However, multiple wakeups are
+ * possible and we must hold the td to prevent a race against the thread
+ * exiting.
  */
 static __inline
 void
@@ -422,7 +427,8 @@ _tsleep_wakeup(struct thread *td)
        globaldata_t gd = mycpu;
 
        if (td->td_gd != gd) {
-               lwkt_send_ipiq(td->td_gd, (ipifunc1_t)tsleep_wakeup, td);
+               lwkt_hold(td);
+               lwkt_send_ipiq(td->td_gd, (ipifunc1_t)tsleep_wakeup_remote, td);
                return;
        }
 #endif
@@ -436,9 +442,10 @@ _tsleep_wakeup(struct thread *td)
 #ifdef SMP
 static
 void
-tsleep_wakeup(struct thread *td)
+tsleep_wakeup_remote(struct thread *td)
 {
        _tsleep_wakeup(td);
+       lwkt_rele(td);
 }
 #endif
 
@@ -1041,8 +1048,8 @@ wakeup_domain_one(const volatile void *ident, int domain)
  * has an effect if we are in SSLEEP.  We only break out of the
  * tsleep if LWP_BREAKTSLEEP is set, otherwise we just fix-up the state.
  *
- * NOTE: With the MP lock held we can only safely manipulate the process
- * structure.  We cannot safely manipulate the thread structure.
+ * NOTE: With proc_token held we can only safely manipulate the process
+ * structure and the lp's lwp_stat.
  */
 void
 setrunnable(struct lwp *lp)

diff --git a/sys/kern/lwkt_thread.c b/sys/kern/lwkt_thread.c
index 5773195..3939767 100644 (file)
--- a/sys/kern/lwkt_thread.c
+++ b/sys/kern/lwkt_thread.c
@@ -430,14 +430,14 @@ lwkt_set_comm(thread_t td, const char *ctl, ...)
 void
 lwkt_hold(thread_t td)
 {
-    ++td->td_refs;
+    atomic_add_int(&td->td_refs, 1);
 }
 
 void
 lwkt_rele(thread_t td)
 {
     KKASSERT(td->td_refs > 0);
-    --td->td_refs;
+    atomic_add_int(&td->td_refs, -1);
 }
 
 void
@@ -450,6 +450,7 @@ lwkt_wait_free(thread_t td)
 void
 lwkt_free_thread(thread_t td)
 {
+    KKASSERT(td->td_refs == 0);
     KKASSERT((td->td_flags & (TDF_RUNNING|TDF_PREEMPT_LOCK|TDF_RUNQ)) == 0);
     if (td->td_flags & TDF_ALLOCATED_THREAD) {
        objcache_put(thread_cache, td);
@@ -1644,6 +1645,7 @@ lwkt_exit(void)
        tsleep_remove(td);
     lwkt_deschedule_self(td);
     lwkt_remove_tdallq(td);
+    KKASSERT(td->td_refs == 0);
 
     /*
      * Final cleanup

diff --git a/sys/platform/vkernel64/x86_64/vm_machdep.c b/sys/platform/vkernel64/x86_64/vm_machdep.c
index d8c047c..2ff6882 100644 (file)
--- a/sys/platform/vkernel64/x86_64/vm_machdep.c
+++ b/sys/platform/vkernel64/x86_64/vm_machdep.c
@@ -270,6 +270,8 @@ cpu_lwp_exit(void)
        td->td_gd->gd_cnt.v_swtch++;
 
        crit_enter_quick(td);
+       if (td->td_flags & TDF_TSLEEPQ)
+               tsleep_remove(td);
        lwkt_deschedule_self(td);
        lwkt_remove_tdallq(td);
        cpu_thread_exit();