From 00ec69c72918b119397bee1830133ef9185120be Mon Sep 17 00:00:00 2001 From: Sepherosa Ziehau Date: Sat, 30 May 2015 16:47:56 +0800 Subject: [PATCH] clock/tsc: Run TSC MP synchronization test on APs too. TSC MP synchronization needs BIOS and hardware coordination. It is known that buggy BIOS could make TSC not MP synchronized if the system has multiple CPU packages. Testing TSC MP synchronization only on BSP is not enough to discover certain types of TSC MP unsynchronization, e.g. APs TSC is ahead of BSP's TSC by great amount. Thus APs also need to run TSC MP synchronization test to make sure that TSC is really MP synchronized. And we are now running at least 50000 TSC MP synchronization tests on each CPU. And use lfence to prevent rdtsc from reordering during TSC MP synchronization test, since we only test TSC MP synchronization on Intel CPUs currently (mfence will be needed for AMD CPUs). --- sys/platform/pc64/isa/clock.c | 155 +++++++++++++++++++++++++--------- 1 file changed, 115 insertions(+), 40 deletions(-) diff --git a/sys/platform/pc64/isa/clock.c b/sys/platform/pc64/isa/clock.c index 7b07069802..3413ca844d 100644 --- a/sys/platform/pc64/isa/clock.c +++ b/sys/platform/pc64/isa/clock.c @@ -1197,25 +1197,111 @@ hw_i8254_timestamp(SYSCTL_HANDLER_ARGS) return(SYSCTL_OUT(req, buf, strlen(buf) + 1)); } -static uint64_t tsc_mpsync_target; +struct tsc_mpsync_arg { + volatile uint64_t tsc_target; + volatile int tsc_mpsync; +}; + +struct tsc_mpsync_thr { + volatile int tsc_done_cnt; + volatile int tsc_mpsync_cnt; +}; static void -tsc_mpsync_test_remote(void *arg __unused) +tsc_mpsync_test_remote(void *xarg) { + struct tsc_mpsync_arg *arg = xarg; uint64_t tsc; + cpu_lfence(); tsc = rdtsc(); - if (tsc < tsc_mpsync_target) - tsc_mpsync = 0; + if (tsc < arg->tsc_target) + arg->tsc_mpsync = 0; } static void -tsc_mpsync_test(void) +tsc_mpsync_test_loop(struct tsc_mpsync_arg *arg) { struct globaldata *gd = mycpu; uint64_t test_end, test_begin; u_int i; + if (bootverbose) { + kprintf("cpu%d: TSC testing MP synchronization ...\n", + gd->gd_cpuid); + } + + cpu_lfence(); + test_begin = rdtsc(); + /* Run test for 100ms */ + test_end = test_begin + (tsc_frequency / 10); + + arg->tsc_mpsync = 1; + arg->tsc_target = test_begin; + +#define TSC_TEST_TRYMAX 1000000 /* Make sure we could stop */ +#define TSC_TEST_TRYMIN 50000 + + for (i = 0; i < TSC_TEST_TRYMAX; ++i) { + struct lwkt_cpusync cs; + + crit_enter(); + lwkt_cpusync_init(&cs, gd->gd_other_cpus, + tsc_mpsync_test_remote, arg); + lwkt_cpusync_interlock(&cs); + cpu_lfence(); + arg->tsc_target = rdtsc(); + cpu_mfence(); + lwkt_cpusync_deinterlock(&cs); + crit_exit(); + + if (!arg->tsc_mpsync) { + kprintf("cpu%d: TSC is not MP synchronized @%u\n", + gd->gd_cpuid, i); + break; + } + if (arg->tsc_target > test_end && i >= TSC_TEST_TRYMIN) + break; + } + +#undef TSC_TEST_TRYMIN +#undef TSC_TEST_TRYMAX + + if (arg->tsc_target == test_begin) { + kprintf("cpu%d: TSC does not tick?!\n", gd->gd_cpuid); + /* XXX disable TSC? */ + tsc_invariant = 0; + arg->tsc_mpsync = 0; + return; + } + + if (arg->tsc_mpsync && bootverbose) { + kprintf("cpu%d: TSC is MP synchronized after %u tries\n", + gd->gd_cpuid, i); + } +} + +static void +tsc_mpsync_ap_thread(void *xthr) +{ + struct tsc_mpsync_thr *thr = xthr; + struct tsc_mpsync_arg arg; + + tsc_mpsync_test_loop(&arg); + if (arg.tsc_mpsync) { + atomic_add_int(&thr->tsc_mpsync_cnt, 1); + cpu_sfence(); + } + atomic_add_int(&thr->tsc_done_cnt, 1); + + lwkt_exit(); +} + +static void +tsc_mpsync_test(void) +{ + struct tsc_mpsync_arg arg; + if (!tsc_invariant) { /* Not even invariant TSC */ return; @@ -1242,50 +1328,39 @@ tsc_mpsync_test(void) } kprintf("TSC testing MP synchronization ...\n"); - tsc_mpsync = 1; - /* Run test for 100ms */ - test_begin = rdtsc(); - test_end = test_begin + (tsc_frequency / 10); + tsc_mpsync_test_loop(&arg); + if (arg.tsc_mpsync) { + struct tsc_mpsync_thr thr; + int cpu; -#define TSC_TEST_TRYMAX 1000000 /* Make sure we could stop */ + /* + * Test TSC MP synchronization on APs. + */ - for (i = 0; i < TSC_TEST_TRYMAX; ++i) { - struct lwkt_cpusync cs; + thr.tsc_done_cnt = 1; + thr.tsc_mpsync_cnt = 1; - crit_enter(); - lwkt_cpusync_init(&cs, gd->gd_other_cpus, - tsc_mpsync_test_remote, NULL); - lwkt_cpusync_interlock(&cs); - tsc_mpsync_target = rdtsc(); - cpu_mfence(); - lwkt_cpusync_deinterlock(&cs); - crit_exit(); + for (cpu = 0; cpu < ncpus; ++cpu) { + if (cpu == mycpuid) + continue; - if (!tsc_mpsync) { - kprintf("TSC is not MP synchronized @%u\n", i); - break; + lwkt_create(tsc_mpsync_ap_thread, &thr, NULL, + NULL, 0, cpu, "tsc mpsync %d", cpu); } - if (tsc_mpsync_target > test_end) - break; - } -#undef TSC_TEST_TRYMAX - - if (tsc_mpsync) { - if (tsc_mpsync_target == test_begin) { - kprintf("TSC does not tick?!"); - /* XXX disable TSC? */ - tsc_invariant = 0; - tsc_mpsync = 0; - return; + while (thr.tsc_done_cnt != ncpus) { + cpu_pause(); + cpu_lfence(); } - - kprintf("TSC is MP synchronized"); - if (bootverbose) - kprintf(", after %u tries", i); - kprintf("\n"); + if (thr.tsc_mpsync_cnt == ncpus) + tsc_mpsync = 1; } + + if (tsc_mpsync) + kprintf("TSC is MP synchronized\n"); + else + kprintf("TSC is not MP synchronized\n"); } SYSINIT(tsc_mpsync, SI_BOOT2_FINISH_SMP, SI_ORDER_ANY, tsc_mpsync_test, NULL); -- 2.41.0