kernel - Add TDF_RUNNING assertions

* Assert that the target lwkt thread being switched to is not flagged as
  running.

* Assert that the originating lwkt thread being switched from is flagged as
  running.

* Fix the running flag initial condition for the idle thread.

diff --git a/sys/kern/lwkt_thread.c b/sys/kern/lwkt_thread.c
index ddc782c..244df96 100644 (file)
--- a/sys/kern/lwkt_thread.c
+++ b/sys/kern/lwkt_thread.c
@@ -594,6 +594,7 @@ lwkt_switch(void)
     int spinning = 0;
 
     KKASSERT(gd->gd_processing_ipiq == 0);
+    KKASSERT(td->td_flags & TDF_RUNNING);
 
     /*
      * Switching from within a 'fast' (non thread switched) interrupt or IPI
@@ -922,6 +923,7 @@ haveidle:
         *
         * We are responsible for marking ntd as TDF_RUNNING.
         */
+       KKASSERT((ntd->td_flags & TDF_RUNNING) == 0);
        ++switch_count;
        KTR_LOG(ctxsw_sw, gd->gd_cpuid, ntd);
        ntd->td_flags |= TDF_RUNNING;
@@ -1032,6 +1034,10 @@ lwkt_preempt(thread_t ntd, int critcount)
        return;
     }
 #ifdef SMP
+    if (td->td_cscount) {
+       ++preempt_miss;
+       return;
+    }
     if (ntd->td_gd != gd) {
        ++preempt_miss;
        return;
@@ -1078,6 +1084,8 @@ lwkt_preempt(thread_t ntd, int critcount)
     KTR_LOG(ctxsw_pre, gd->gd_cpuid, ntd);
     save_gd_intr_nesting_level = gd->gd_intr_nesting_level;
     gd->gd_intr_nesting_level = 0;
+
+    KKASSERT((ntd->td_flags & TDF_RUNNING) == 0);
     ntd->td_flags |= TDF_RUNNING;
     xtd = td->td_switch(ntd);
     KKASSERT(xtd == ntd);

diff --git a/sys/platform/pc32/i386/swtch.s b/sys/platform/pc32/i386/swtch.s
index 29dac4c..0a6d4ad 100644 (file)
--- a/sys/platform/pc32/i386/swtch.s
+++ b/sys/platform/pc32/i386/swtch.s
@@ -558,9 +558,7 @@ ENTRY(cpu_idle_restore)
        pushl   $0
        movl    %ecx,%cr3
        andl    $~TDF_RUNNING,TD_FLAGS(%ebx)
-#if 0
-       orl     $TDF_RUNNING,TD_FLAGS(%eax)
-#endif
+       orl     $TDF_RUNNING,TD_FLAGS(%eax)     /* manual, no switch_return */
 #ifdef SMP
        cmpl    $0,PCPU(cpuid)
        je      1f

diff --git a/sys/platform/pc64/x86_64/support.s b/sys/platform/pc64/x86_64/support.s
index ddd0a38..d814b7a 100644 (file)
--- a/sys/platform/pc64/x86_64/support.s
+++ b/sys/platform/pc64/x86_64/support.s
@@ -44,7 +44,7 @@
 
 /*
  * bcopy family
- * void bzero(void *buf, u_int len)
+ * void bzero(void *buf, size_t len)
  */
 
 /* done */

diff --git a/sys/platform/pc64/x86_64/swtch.s b/sys/platform/pc64/x86_64/swtch.s
index 48a0eae..14b543c 100644 (file)
--- a/sys/platform/pc64/x86_64/swtch.s
+++ b/sys/platform/pc64/x86_64/swtch.s
@@ -298,6 +298,8 @@ ENTRY(cpu_exit_switch)
  */
 
 ENTRY(cpu_heavy_restore)
+       movq    TD_PCB(%rax),%rdx               /* RDX = PCB */
+       movq    %rdx, PCPU(common_tss) + TSS_RSP0
        popfq
 
 #if defined(SWTCH_OPTIM_STATS)
@@ -337,6 +339,13 @@ ENTRY(cpu_heavy_restore)
         */
        btq     $CPUMASK_BIT,%rax       /* test interlock */
        jnc     1f
+
+#if 0
+       movq    TD_PCB(%r12),%rdx       /* XXX debugging unconditional */
+       movq    PCB_CR3(%rdx),%rdx      /*     reloading of %cr3 */
+       movq    %rdx,%cr3
+#endif
+
        movq    %rcx,%rdi               /* (found to be set) */
        call    pmap_interlock_wait     /* pmap_interlock_wait(%rdi:vm) */
 
@@ -621,9 +630,7 @@ ENTRY(cpu_idle_restore)
        pushq   $0
        movq    %rcx,%cr3
        andl    $~TDF_RUNNING,TD_FLAGS(%rbx)
-#if 0
-       orl     $TDF_RUNNING,TD_FLAGS(%rax)
-#endif
+       orl     $TDF_RUNNING,TD_FLAGS(%rax)     /* manual, no switch_return */
 #ifdef SMP
        cmpl    $0,PCPU(cpuid)
        je      1f
@@ -750,6 +757,15 @@ ENTRY(cpu_lwkt_restore)
        movq    %rcx,%cr3
 1:
        /*
+        * Safety, clear RSP0 in the tss so it isn't pointing at the
+        * previous thread's kstack (if a heavy weight user thread).
+        * RSP0 should only be used in ring 3 transitions and kernel
+        * threads run in ring 0 so there should be none.
+        */
+       xorq    %rdx,%rdx
+       movq    %rdx, PCPU(common_tss) + TSS_RSP0
+
+       /*
         * NOTE: %rbx is the previous thread and %rax is the new thread.
         *       %rbx is retained throughout so we can return it.
         *

diff --git a/sys/platform/vkernel/i386/swtch.s b/sys/platform/vkernel/i386/swtch.s
index 174b830..edb9b39 100644 (file)
--- a/sys/platform/vkernel/i386/swtch.s
+++ b/sys/platform/vkernel/i386/swtch.s
@@ -502,9 +502,7 @@ ENTRY(cpu_idle_restore)
        movl    $0,%ebp
        pushl   $0
        andl    $~TDF_RUNNING,TD_FLAGS(%ebx)
-#if 0
-       orl     $TDF_RUNNING,TD_FLAGS(%eax)
-#endif
+       orl     $TDF_RUNNING,TD_FLAGS(%eax)     /* manual, no switch_return */
 #ifdef SMP
        cmpl    $0,PCPU(cpuid)
        je      1f

diff --git a/sys/platform/vkernel64/x86_64/swtch.s b/sys/platform/vkernel64/x86_64/swtch.s
index 88e57ec..2f32e21 100644 (file)
--- a/sys/platform/vkernel64/x86_64/swtch.s
+++ b/sys/platform/vkernel64/x86_64/swtch.s
@@ -561,9 +561,7 @@ ENTRY(cpu_idle_restore)
        /* JG push RBP? */
        pushq   $0
        andl    $~TDF_RUNNING,TD_FLAGS(%rbx)
-#if 0
-       orl     $TDF_RUNNING,TD_FLAGS(%rax)
-#endif
+       orl     $TDF_RUNNING,TD_FLAGS(%rax)     /* manual, no switch_return */
 #ifdef SMP
        cmpl    $0,PCPU(cpuid)
        je      1f