Fix a number of SMP issues.
authorMatthew Dillon <dillon@dragonflybsd.org>
Sat, 20 Nov 2004 20:50:36 +0000 (20:50 +0000)
committerMatthew Dillon <dillon@dragonflybsd.org>
Sat, 20 Nov 2004 20:50:36 +0000 (20:50 +0000)
* Add required pause instructions in code paths that skip running "HLT".  This
  occurs when e.g. cpu #1 is running code with the BGL while cpu #2's only
  runnable thread requires the BGL.  cpu #2's LWKT scheduler spins in that
  case.  Similar situations can occur when the only runnable threads on a
  cpu are waiting for a token.

* Add required pause instructions to spin loops.  DragonFly has very few spin
  locks (e.g. things like com_lock()) but its a good idea anyway to avoid
  known livelock issues on Intel cpus.

* Fix a pending interrupt / HLT race.  We were not atomically retiring
  pending interrupts prior to potentially HLTing the cpu.  This could
  result in an SMP machine's network locking up until a key is hit on
  the console, then magically resuming.

Lockups-reported-by: Peter Avalos <pavalos@theshell.com>
sys/dev/acpica5/acpi_cpu.c
sys/i386/i386/machdep.c
sys/i386/i386/mplock.s
sys/i386/include/lock.h
sys/platform/pc32/i386/machdep.c
sys/platform/pc32/i386/mplock.s
sys/platform/pc32/include/lock.h

index 7c487b1..1b95355 100644 (file)
@@ -25,7 +25,7 @@
  * SUCH DAMAGE.
  *
  * $FreeBSD: src/sys/dev/acpica/acpi_cpu.c,v 1.41 2004/06/24 00:38:51 njl Exp $
- * $DragonFly: src/sys/dev/acpica5/acpi_cpu.c,v 1.6 2004/08/07 00:59:53 dillon Exp $
+ * $DragonFly: src/sys/dev/acpica5/acpi_cpu.c,v 1.7 2004/11/20 20:50:30 dillon Exp $
  */
 
 #include "opt_acpi.h"
@@ -989,8 +989,19 @@ acpi_cpu_c1()
 #ifdef __ia64__
     ia64_call_pal_static(PAL_HALT_LIGHT, 0, 0, 0);
 #else
-    __asm __volatile("sti; hlt");
+    splz();
+#ifdef SMP
+    if (!lwkt_runnable())
+       __asm __volatile("sti; hlt");
+    else
+       __asm __volatile("sti; pause");
+#else
+    if (!lwkt_runnable())
+       __asm __volatile("sti; hlt");
+    else
+       __asm __volatile("sti");
 #endif
+#endif /* !__ia64__ */
 }
 
 /*
index 5aa62f6..36bad64 100644 (file)
@@ -36,7 +36,7 @@
  *
  *     from: @(#)machdep.c     7.4 (Berkeley) 6/3/91
  * $FreeBSD: src/sys/i386/i386/machdep.c,v 1.385.2.30 2003/05/31 08:48:05 alc Exp $
- * $DragonFly: src/sys/i386/i386/Attic/machdep.c,v 1.67 2004/10/25 13:48:42 simokawa Exp $
+ * $DragonFly: src/sys/i386/i386/Attic/machdep.c,v 1.68 2004/11/20 20:50:33 dillon Exp $
  */
 
 #include "use_apm.h"
@@ -862,12 +862,21 @@ cpu_idle(void)
                    (td->td_flags & TDF_IDLE_NOHLT) == 0) {
                        __asm __volatile("cli");
                        splz();
-                       cpu_idle_hook();
+                       if (!lwkt_runnable())
+                           cpu_idle_hook();
+#ifdef SMP
+                       else
+                           __asm __volatile("pause");
+#endif
                        ++cpu_idle_hltcnt;
                } else {
                        td->td_flags &= ~TDF_IDLE_NOHLT;
                        splz();
+#ifdef SMP
+                       __asm __volatile("sti; pause");
+#else
                        __asm __volatile("sti");
+#endif
                        ++cpu_idle_spincnt;
                }
        }
index 5fe0d09..def4db0 100644 (file)
@@ -1,6 +1,6 @@
 /*
  * $FreeBSD: src/sys/i386/i386/mplock.s,v 1.29.2.2 2000/05/16 06:58:06 dillon Exp $
- * $DragonFly: src/sys/i386/i386/Attic/mplock.s,v 1.14 2004/07/16 05:48:29 dillon Exp $
+ * $DragonFly: src/sys/i386/i386/Attic/mplock.s,v 1.15 2004/11/20 20:50:33 dillon Exp $
  *
  * Copyright (c) 2003,2004 The DragonFly Project.  All rights reserved.
  * 
@@ -161,6 +161,7 @@ NON_GPROF_ENTRY(get_mplock)
         * and 'block' until the mp_lock is obtained.
         */
 2:
+       pause
        call    lwkt_switch
 #ifdef INVARIANTS
        movl    PCPU(cpuid),%eax        /* failure */
index c1c8644..c98ab02 100644 (file)
@@ -32,7 +32,7 @@
  * SUCH DAMAGE.
  * 
  * $FreeBSD: src/sys/i386/include/lock.h,v 1.11.2.2 2000/09/30 02:49:34 ps Exp $
- * $DragonFly: src/sys/i386/include/Attic/lock.h,v 1.9 2004/07/16 05:48:46 dillon Exp $
+ * $DragonFly: src/sys/i386/include/Attic/lock.h,v 1.10 2004/11/20 20:50:36 dillon Exp $
  */
 
 #ifndef _MACHINE_LOCK_H_
@@ -77,6 +77,7 @@
 7: ;                                                           \
        movl    $0,%eax ;       /* expected contents of lock */ \
        lock cmpxchgl %ecx,mem ; /* Z=1 (jz) on success */      \
+       pause ;                                                 \
        jnz     7b ;                                            \
 
 #define SPIN_LOCK_PUSH_REGS                                    \
index 5a76623..cd0ba78 100644 (file)
@@ -36,7 +36,7 @@
  *
  *     from: @(#)machdep.c     7.4 (Berkeley) 6/3/91
  * $FreeBSD: src/sys/i386/i386/machdep.c,v 1.385.2.30 2003/05/31 08:48:05 alc Exp $
- * $DragonFly: src/sys/platform/pc32/i386/machdep.c,v 1.67 2004/10/25 13:48:42 simokawa Exp $
+ * $DragonFly: src/sys/platform/pc32/i386/machdep.c,v 1.68 2004/11/20 20:50:33 dillon Exp $
  */
 
 #include "use_apm.h"
@@ -862,12 +862,21 @@ cpu_idle(void)
                    (td->td_flags & TDF_IDLE_NOHLT) == 0) {
                        __asm __volatile("cli");
                        splz();
-                       cpu_idle_hook();
+                       if (!lwkt_runnable())
+                           cpu_idle_hook();
+#ifdef SMP
+                       else
+                           __asm __volatile("pause");
+#endif
                        ++cpu_idle_hltcnt;
                } else {
                        td->td_flags &= ~TDF_IDLE_NOHLT;
                        splz();
+#ifdef SMP
+                       __asm __volatile("sti; pause");
+#else
                        __asm __volatile("sti");
+#endif
                        ++cpu_idle_spincnt;
                }
        }
index b5fd1ae..09ea0c2 100644 (file)
@@ -1,6 +1,6 @@
 /*
  * $FreeBSD: src/sys/i386/i386/mplock.s,v 1.29.2.2 2000/05/16 06:58:06 dillon Exp $
- * $DragonFly: src/sys/platform/pc32/i386/mplock.s,v 1.14 2004/07/16 05:48:29 dillon Exp $
+ * $DragonFly: src/sys/platform/pc32/i386/mplock.s,v 1.15 2004/11/20 20:50:33 dillon Exp $
  *
  * Copyright (c) 2003,2004 The DragonFly Project.  All rights reserved.
  * 
@@ -161,6 +161,7 @@ NON_GPROF_ENTRY(get_mplock)
         * and 'block' until the mp_lock is obtained.
         */
 2:
+       pause
        call    lwkt_switch
 #ifdef INVARIANTS
        movl    PCPU(cpuid),%eax        /* failure */
index 9f5b942..e364be3 100644 (file)
@@ -32,7 +32,7 @@
  * SUCH DAMAGE.
  * 
  * $FreeBSD: src/sys/i386/include/lock.h,v 1.11.2.2 2000/09/30 02:49:34 ps Exp $
- * $DragonFly: src/sys/platform/pc32/include/lock.h,v 1.9 2004/07/16 05:48:46 dillon Exp $
+ * $DragonFly: src/sys/platform/pc32/include/lock.h,v 1.10 2004/11/20 20:50:36 dillon Exp $
  */
 
 #ifndef _MACHINE_LOCK_H_
@@ -77,6 +77,7 @@
 7: ;                                                           \
        movl    $0,%eax ;       /* expected contents of lock */ \
        lock cmpxchgl %ecx,mem ; /* Z=1 (jz) on success */      \
+       pause ;                                                 \
        jnz     7b ;                                            \
 
 #define SPIN_LOCK_PUSH_REGS                                    \