serialize: Rework adaptive enter
authorSepherosa Ziehau <sephe@dragonflybsd.org>
Fri, 3 Jan 2014 13:22:12 +0000 (21:22 +0800)
committerSepherosa Ziehau <sephe@dragonflybsd.org>
Thu, 9 Jan 2014 13:15:53 +0000 (21:15 +0800)
- Don't increase wait counter before the spinning phrase; this avoids
  unnecessary wakeup() calls on other CPU
- Rework the spinning phrase according to our spinlock contest processing

Tested on 8 HT (i7-3770) box, using kq_accept_server/kq_connect_client:
- 4/4 TX/RX rings device (BCM5719, using MSI-X), slight improvement.
- 8/8 TX/RX rings device (Intel 82580, using MSI-X), slight improvement.
- 1/2 TX/RX rings device (Intel 82599, using MSI), slight improvement.

sys/cpu/i386/include/atomic.h
sys/cpu/x86_64/include/atomic.h
sys/kern/lwkt_serialize.c

index b444d04..0c649fd 100644 (file)
@@ -274,6 +274,8 @@ int atomic_intr_cond_test(__atomic_intr_t *p);
 int atomic_intr_cond_try(__atomic_intr_t *p);
 void atomic_intr_cond_enter(__atomic_intr_t *p, void (*func)(void *), void *arg);
 void atomic_intr_cond_exit(__atomic_intr_t *p, void (*func)(void *), void *arg);
+void atomic_intr_cond_inc(__atomic_intr_t *p);
+void atomic_intr_cond_dec(__atomic_intr_t *p);
 uint64_t atomic_load_acq_64_i586(volatile uint64_t *p);
 
 #else
@@ -315,6 +317,18 @@ atomic_intr_handler_is_enabled(__atomic_intr_t *p)
        return(data);
 }
 
+static __inline void
+atomic_intr_cond_inc(__atomic_intr_t *p)
+{
+       __asm __volatile(MPLOCKED "incl %0" : "+m" (*p));
+}
+
+static __inline void
+atomic_intr_cond_dec(__atomic_intr_t *p)
+{
+       __asm __volatile(MPLOCKED "decl %0" : "+m" (*p));
+}
+
 static __inline
 void
 atomic_intr_cond_enter(__atomic_intr_t *p, void (*func)(void *), void *arg)
index 8f62d08..3a663f4 100644 (file)
@@ -298,6 +298,8 @@ int atomic_intr_cond_test(__atomic_intr_t *p);
 int atomic_intr_cond_try(__atomic_intr_t *p);
 void atomic_intr_cond_enter(__atomic_intr_t *p, void (*func)(void *), void *arg);
 void atomic_intr_cond_exit(__atomic_intr_t *p, void (*func)(void *), void *arg);
+void atomic_intr_cond_inc(__atomic_intr_t *p);
+void atomic_intr_cond_dec(__atomic_intr_t *p);
 
 #else
 
@@ -338,6 +340,18 @@ atomic_intr_handler_is_enabled(__atomic_intr_t *p)
        return(data);
 }
 
+static __inline void
+atomic_intr_cond_inc(__atomic_intr_t *p)
+{
+       __asm __volatile(MPLOCKED "incl %0" : "+m" (*p));
+}
+
+static __inline void
+atomic_intr_cond_dec(__atomic_intr_t *p)
+{
+       __asm __volatile(MPLOCKED "decl %0" : "+m" (*p));
+}
+
 static __inline
 void
 atomic_intr_cond_enter(__atomic_intr_t *p, void (*func)(void *), void *arg)
index 3ee90df..08c824c 100644 (file)
 #include <machine/cpu.h>
 #include <machine/cpufunc.h>
 #include <machine/specialreg.h>
+#include <machine/clock.h>
 #include <sys/lock.h>
 
-struct exp_backoff {
-       int backoff;
-       int round;
-       lwkt_serialize_t s;
-};
+#ifndef SLZ_ADAPTIVE_SPINMAX
+#define SLZ_ADAPTIVE_SPINMAX   4096
+#endif
 
 #define SLZ_KTR_STRING         "slz=%p"
 #define SLZ_KTR_ARGS           lwkt_serialize_t slz
 
 #ifndef KTR_SERIALIZER
-#define KTR_SERIALIZER KTR_ALL
+#define KTR_SERIALIZER         KTR_ALL
 #endif
 
 KTR_INFO_MASTER(slz);
@@ -81,31 +80,20 @@ KTR_INFO(KTR_SERIALIZER, slz, wakeup_end, 5, SLZ_KTR_STRING, SLZ_KTR_ARGS);
 KTR_INFO(KTR_SERIALIZER, slz, try, 6, SLZ_KTR_STRING, SLZ_KTR_ARGS);
 KTR_INFO(KTR_SERIALIZER, slz, tryfail, 7, SLZ_KTR_STRING, SLZ_KTR_ARGS);
 KTR_INFO(KTR_SERIALIZER, slz, tryok, 8, SLZ_KTR_STRING, SLZ_KTR_ARGS);
-KTR_INFO(KTR_SERIALIZER, slz, spinbo, 9,
-        "slz=%p bo1=%d bo=%d", lwkt_serialize_t slz, int backoff1, int backoff);
-KTR_INFO(KTR_SERIALIZER, slz, enter_end, 10, SLZ_KTR_STRING, SLZ_KTR_ARGS);
-KTR_INFO(KTR_SERIALIZER, slz, exit_beg, 11, SLZ_KTR_STRING, SLZ_KTR_ARGS);
+KTR_INFO(KTR_SERIALIZER, slz, enter_end, 9, SLZ_KTR_STRING, SLZ_KTR_ARGS);
+KTR_INFO(KTR_SERIALIZER, slz, exit_beg, 10, SLZ_KTR_STRING, SLZ_KTR_ARGS);
+KTR_INFO(KTR_SERIALIZER, slz, adapt_beg, 11, SLZ_KTR_STRING, SLZ_KTR_ARGS);
+KTR_INFO(KTR_SERIALIZER, slz, adapt_end, 12, SLZ_KTR_STRING, SLZ_KTR_ARGS);
+KTR_INFO(KTR_SERIALIZER, slz, adapt_spinend, 13, "slz=%p try=%d",
+    lwkt_serialize_t slz, int try);
+KTR_INFO(KTR_SERIALIZER, slz, adapt_sleepb, 14, SLZ_KTR_STRING, SLZ_KTR_ARGS);
+KTR_INFO(KTR_SERIALIZER, slz, adapt_sleepe, 15, SLZ_KTR_STRING, SLZ_KTR_ARGS);
 
 #define logslz(name, slz)              KTR_LOG(slz_ ## name, slz)
-#define logslz_spinbo(slz, bo1, bo)    KTR_LOG(slz_spinbo, slz, bo1, bo)
+#define logslz_spinend(slz, try)       KTR_LOG(slz_adapt_spinend, slz, try)
 
 static void lwkt_serialize_sleep(void *info);
 static void lwkt_serialize_wakeup(void *info);
-static void lwkt_serialize_adaptive_sleep(void *bo);
-
-static int slz_backoff_limit = 128;
-SYSCTL_INT(_debug, OID_AUTO, serialize_bolimit, CTLFLAG_RW,
-    &slz_backoff_limit, 0, "Backoff limit");
-
-static int slz_backoff_shift = 1;
-SYSCTL_INT(_debug, OID_AUTO, serialize_boshift, CTLFLAG_RW,
-    &slz_backoff_shift, 0, "Backoff shift");
-
-static int slz_backoff_round;
-TUNABLE_INT("debug.serialize_boround", &slz_backoff_round);
-SYSCTL_INT(_debug, OID_AUTO, serialize_boround, CTLFLAG_RW,
-    &slz_backoff_round, 0,
-    "Backoff rounding");
 
 void
 lwkt_serialize_init(lwkt_serialize_t s)
@@ -117,25 +105,6 @@ lwkt_serialize_init(lwkt_serialize_t s)
 }
 
 void
-lwkt_serialize_adaptive_enter(lwkt_serialize_t s)
-{
-    struct exp_backoff bo;
-
-    bo.backoff = 1;
-    bo.round = 0;
-    bo.s = s;
-
-    ASSERT_NOT_SERIALIZED(s);
-
-    logslz(enter_beg, s);
-    atomic_intr_cond_enter(&s->interlock, lwkt_serialize_adaptive_sleep, &bo);
-    logslz(enter_end, s);
-#ifdef INVARIANTS
-    s->last_td = curthread;
-#endif
-}
-
-void
 lwkt_serialize_enter(lwkt_serialize_t s)
 {
     ASSERT_NOT_SERIALIZED(s);
@@ -284,49 +253,59 @@ lwkt_serialize_sleep(void *info)
     }
 }
 
-static void
-lwkt_serialize_adaptive_sleep(void *arg)
+void
+lwkt_serialize_adaptive_enter(lwkt_serialize_t s)
 {
-    struct exp_backoff *bo = arg;
-    lwkt_serialize_t s = bo->s;
-    int backoff;
+    int try;
 
-    /*
-     * Randomize backoff value
-     */
-#ifdef _RDTSC_SUPPORTED_
-    if (cpu_feature & CPUID_TSC) {
-       backoff =
-       (((u_long)rdtsc() ^ (((u_long)curthread) >> 5)) &
-        (bo->backoff - 1)) + 1;
-    } else
-#endif
-       backoff = bo->backoff;
+    ASSERT_NOT_SERIALIZED(s);
+    logslz(adapt_beg, s);
 
-    logslz_spinbo(s, bo->backoff, backoff);
+    if (atomic_intr_cond_try(&s->interlock) == 0) {
+#ifdef INVARIANTS
+       s->last_td = curthread;
+#endif
+       logslz(adapt_end, s);
+       return;
+    }
 
+restart:
     /*
-     * Quick backoff
+     * Spinning a little bit, before going to sleep
+     *
+     * See the comment before kern/kern_spinlock.c
+     * _spin_lock_contested() about why atomic_intr_cond_test()
+     * is called first.  atomic_intr_cond_test() contains
+     * _no_ MPLOCKED intruction.
      */
-    for (; backoff; --backoff)
-       cpu_pause();
-    if (bo->backoff < slz_backoff_limit) {
-       bo->backoff <<= slz_backoff_shift;
-       return;
-    } else {
-       bo->backoff = 1;
-       bo->round++;
-       if (bo->round >= slz_backoff_round)
-           bo->round = 0;
-       else
+    for (try = SLZ_ADAPTIVE_SPINMAX; try; --try) {
+       if (atomic_intr_cond_test(&s->interlock) == 0 &&
+           atomic_intr_cond_try(&s->interlock) == 0) {
+#ifdef INVARIANTS
+           s->last_td = curthread;
+#endif
+           logslz_spinend(s, try);
            return;
+       }
     }
 
+    atomic_intr_cond_inc(&s->interlock);
+
     tsleep_interlock(s, 0);
-    if (atomic_intr_cond_test(&s->interlock) != 0) {
-       logslz(sleep_beg, s);
-       tsleep(s, PINTERLOCKED, "slize", 0);
-       logslz(sleep_end, s);
+    if (atomic_intr_cond_try(&s->interlock) == 0) {
+       atomic_intr_cond_dec(&s->interlock);
+#ifdef INVARIANTS
+       s->last_td = curthread;
+#endif
+       logslz_spinend(s, 0);
+       return;
+    } else {
+       logslz(adapt_sleepb, s);
+       tsleep(s, PINTERLOCKED, "aslize", 0);
+       logslz(adapt_sleepe, s);
+
+       atomic_intr_cond_dec(&s->interlock);
+       goto restart;
     }
 }
 
@@ -337,12 +316,3 @@ lwkt_serialize_wakeup(void *info)
     wakeup(info);
     logslz(wakeup_end, info);
 }
-
-static void
-lwkt_serialize_sysinit(void *dummy __unused)
-{
-       if (slz_backoff_round <= 0)
-               slz_backoff_round = ncpus * 2;
-}
-SYSINIT(lwkt_serialize, SI_SUB_PRE_DRIVERS, SI_ORDER_SECOND,
-       lwkt_serialize_sysinit, NULL);