clock/tsc: Run TSC MP synchronization test on APs too.
[dragonfly.git] / sys / platform / pc64 / isa / clock.c
index 7b07069..3413ca8 100644 (file)
@@ -1197,25 +1197,111 @@ hw_i8254_timestamp(SYSCTL_HANDLER_ARGS)
     return(SYSCTL_OUT(req, buf, strlen(buf) + 1));
 }
 
-static uint64_t                tsc_mpsync_target;
+struct tsc_mpsync_arg {
+       volatile uint64_t       tsc_target;
+       volatile int            tsc_mpsync;
+};
+
+struct tsc_mpsync_thr {
+       volatile int            tsc_done_cnt;
+       volatile int            tsc_mpsync_cnt;
+};
 
 static void
-tsc_mpsync_test_remote(void *arg __unused)
+tsc_mpsync_test_remote(void *xarg)
 {
+       struct tsc_mpsync_arg *arg = xarg;
        uint64_t tsc;
 
+       cpu_lfence();
        tsc = rdtsc();
-       if (tsc < tsc_mpsync_target)
-               tsc_mpsync = 0;
+       if (tsc < arg->tsc_target)
+               arg->tsc_mpsync = 0;
 }
 
 static void
-tsc_mpsync_test(void)
+tsc_mpsync_test_loop(struct tsc_mpsync_arg *arg)
 {
        struct globaldata *gd = mycpu;
        uint64_t test_end, test_begin;
        u_int i;
 
+       if (bootverbose) {
+               kprintf("cpu%d: TSC testing MP synchronization ...\n",
+                   gd->gd_cpuid);
+       }
+
+       cpu_lfence();
+       test_begin = rdtsc();
+       /* Run test for 100ms */
+       test_end = test_begin + (tsc_frequency / 10);
+
+       arg->tsc_mpsync = 1;
+       arg->tsc_target = test_begin;
+
+#define TSC_TEST_TRYMAX                1000000 /* Make sure we could stop */
+#define TSC_TEST_TRYMIN                50000
+
+       for (i = 0; i < TSC_TEST_TRYMAX; ++i) {
+               struct lwkt_cpusync cs;
+
+               crit_enter();
+               lwkt_cpusync_init(&cs, gd->gd_other_cpus,
+                   tsc_mpsync_test_remote, arg);
+               lwkt_cpusync_interlock(&cs);
+               cpu_lfence();
+               arg->tsc_target = rdtsc();
+               cpu_mfence();
+               lwkt_cpusync_deinterlock(&cs);
+               crit_exit();
+
+               if (!arg->tsc_mpsync) {
+                       kprintf("cpu%d: TSC is not MP synchronized @%u\n",
+                           gd->gd_cpuid, i);
+                       break;
+               }
+               if (arg->tsc_target > test_end && i >= TSC_TEST_TRYMIN)
+                       break;
+       }
+
+#undef TSC_TEST_TRYMIN
+#undef TSC_TEST_TRYMAX
+
+       if (arg->tsc_target == test_begin) {
+               kprintf("cpu%d: TSC does not tick?!\n", gd->gd_cpuid);
+               /* XXX disable TSC? */
+               tsc_invariant = 0;
+               arg->tsc_mpsync = 0;
+               return;
+       }
+
+       if (arg->tsc_mpsync && bootverbose) {
+               kprintf("cpu%d: TSC is MP synchronized after %u tries\n",
+                   gd->gd_cpuid, i);
+       }
+}
+
+static void
+tsc_mpsync_ap_thread(void *xthr)
+{
+       struct tsc_mpsync_thr *thr = xthr;
+       struct tsc_mpsync_arg arg;
+
+       tsc_mpsync_test_loop(&arg);
+       if (arg.tsc_mpsync) {
+               atomic_add_int(&thr->tsc_mpsync_cnt, 1);
+               cpu_sfence();
+       }
+       atomic_add_int(&thr->tsc_done_cnt, 1);
+
+       lwkt_exit();
+}
+
+static void
+tsc_mpsync_test(void)
+{
+       struct tsc_mpsync_arg arg;
+
        if (!tsc_invariant) {
                /* Not even invariant TSC */
                return;
@@ -1242,50 +1328,39 @@ tsc_mpsync_test(void)
        }
 
        kprintf("TSC testing MP synchronization ...\n");
-       tsc_mpsync = 1;
 
-       /* Run test for 100ms */
-       test_begin = rdtsc();
-       test_end = test_begin + (tsc_frequency / 10);
+       tsc_mpsync_test_loop(&arg);
+       if (arg.tsc_mpsync) {
+               struct tsc_mpsync_thr thr;
+               int cpu;
 
-#define TSC_TEST_TRYMAX                1000000 /* Make sure we could stop */
+               /*
+                * Test TSC MP synchronization on APs.
+                */
 
-       for (i = 0; i < TSC_TEST_TRYMAX; ++i) {
-               struct lwkt_cpusync cs;
+               thr.tsc_done_cnt = 1;
+               thr.tsc_mpsync_cnt = 1;
 
-               crit_enter();
-               lwkt_cpusync_init(&cs, gd->gd_other_cpus,
-                                 tsc_mpsync_test_remote, NULL);
-               lwkt_cpusync_interlock(&cs);
-               tsc_mpsync_target = rdtsc();
-               cpu_mfence();
-               lwkt_cpusync_deinterlock(&cs);
-               crit_exit();
+               for (cpu = 0; cpu < ncpus; ++cpu) {
+                       if (cpu == mycpuid)
+                               continue;
 
-               if (!tsc_mpsync) {
-                       kprintf("TSC is not MP synchronized @%u\n", i);
-                       break;
+                       lwkt_create(tsc_mpsync_ap_thread, &thr, NULL,
+                           NULL, 0, cpu, "tsc mpsync %d", cpu);
                }
-               if (tsc_mpsync_target > test_end)
-                       break;
-       }
 
-#undef TSC_TEST_TRYMAX
-
-       if (tsc_mpsync) {
-               if (tsc_mpsync_target == test_begin) {
-                       kprintf("TSC does not tick?!");
-                       /* XXX disable TSC? */
-                       tsc_invariant = 0;
-                       tsc_mpsync = 0;
-                       return;
+               while (thr.tsc_done_cnt != ncpus) {
+                       cpu_pause();
+                       cpu_lfence();
                }
-
-               kprintf("TSC is MP synchronized");
-               if (bootverbose)
-                       kprintf(", after %u tries", i);
-               kprintf("\n");
+               if (thr.tsc_mpsync_cnt == ncpus)
+                       tsc_mpsync = 1;
        }
+
+       if (tsc_mpsync)
+               kprintf("TSC is MP synchronized\n");
+       else
+               kprintf("TSC is not MP synchronized\n");
 }
 SYSINIT(tsc_mpsync, SI_BOOT2_FINISH_SMP, SI_ORDER_ANY, tsc_mpsync_test, NULL);