sys/cpu/x86_64/misc/cputimer_tsc.c

   1 /*-
   2  * Copyright (c) 1990 The Regents of the University of California.
   3  * Copyright (c) 2008 The DragonFly Project.
   4  * All rights reserved.
   5  *
   6  * This code is derived from software contributed to Berkeley by
   7  * William Jolitz and Don Ahn.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  * 3. Neither the name of the University nor the names of its contributors
  18  *    may be used to endorse or promote products derived from this software
  19  *    without specific prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  *
  33  *      from: @(#)clock.c       7.2 (Berkeley) 5/12/91
  34  * $FreeBSD: src/sys/i386/isa/clock.c,v 1.149.2.6 2002/11/02 04:41:50 iwasaki Exp $
  35  */
  36
  37 /*
  38  * TSC cputimer.
  39  */
  40
  41 #include <sys/param.h>
  42 #include <sys/systm.h>
  43 #include <sys/kernel.h>
  44 #include <sys/sysctl.h>
  45 #include <sys/systimer.h>
  46 #include <sys/globaldata.h>
  47
  48 #include <machine/clock.h>
  49 #include <machine/cputypes.h>
  50
  51 static sysclock_t tsc_cputimer_count_mfence(void);
  52 static sysclock_t tsc_cputimer_count_lfence(void);
  53 static void tsc_cputimer_construct(struct cputimer *, sysclock_t);
  54
  55 static struct cputimer  tsc_cputimer = {
  56     .next               = SLIST_ENTRY_INITIALIZER,
  57     .name               = "TSC",
  58     .pri                = CPUTIMER_PRI_TSC,
  59     .type               = CPUTIMER_TSC,
  60     .count              = NULL, /* determined later */
  61     .fromhz             = cputimer_default_fromhz,
  62     .fromus             = cputimer_default_fromus,
  63     .construct          = tsc_cputimer_construct,
  64     .destruct           = cputimer_default_destruct,
  65     .freq               = 0     /* determined later */
  66 };
  67
  68 static struct cpucounter tsc_cpucounter = {
  69     .freq               = 0,    /* determined later */
  70     .count              = NULL, /* determined later */
  71     .flags              = 0,    /* adjusted later */
  72     .prio               = CPUCOUNTER_PRIO_TSC,
  73     .type               = CPUCOUNTER_TSC
  74 };
  75
  76 #ifdef _KERNEL_VIRTUAL
  77 extern int allow_tsc_timer;
  78
  79 #define TSC_CPUTIMER_FREQMAX    2000000 /* 2MHz */
  80 #else
  81 #define TSC_CPUTIMER_FREQMAX    128000000       /* 128Mhz */
  82 #endif
  83
  84 static int tsc_cputimer_shift;
  85
  86 static void
  87 tsc_cputimer_construct(struct cputimer *timer, sysclock_t oldclock)
  88 {
  89         timer->base = 0;
  90         timer->base = oldclock - timer->count();
  91 }
  92
  93 static __inline sysclock_t
  94 tsc_cputimer_count(void)
  95 {
  96         uint64_t tsc;
  97
  98         tsc = rdtsc();
  99         tsc >>= tsc_cputimer_shift;
 100
 101         return (tsc + tsc_cputimer.base);
 102 }
 103
 104 static sysclock_t
 105 tsc_cputimer_count_lfence(void)
 106 {
 107         cpu_lfence();
 108         return tsc_cputimer_count();
 109 }
 110
 111 static sysclock_t
 112 tsc_cputimer_count_mfence(void)
 113 {
 114         cpu_mfence();
 115         return tsc_cputimer_count();
 116 }
 117
 118 static uint64_t
 119 tsc_cpucounter_count_lfence(void)
 120 {
 121
 122         cpu_lfence();
 123         return (rdtsc());
 124 }
 125
 126 static uint64_t
 127 tsc_cpucounter_count_mfence(void)
 128 {
 129
 130         cpu_mfence();
 131         return (rdtsc());
 132 }
 133
 134 static void
 135 tsc_cputimer_register(void)
 136 {
 137         uint64_t freq;
 138         int enable = 1;
 139
 140 #ifdef _KERNEL_VIRTUAL
 141         if (!allow_tsc_timer)
 142                 return;
 143 #endif
 144
 145         if (!tsc_mpsync) {
 146 #ifndef _KERNEL_VIRTUAL
 147                 if (tsc_invariant) {
 148                         /* Per-cpu cpucounter still works. */
 149                         goto regcnt;
 150                 }
 151 #endif
 152                 return;
 153         }
 154
 155         TUNABLE_INT_FETCH("hw.tsc_cputimer_enable", &enable);
 156         if (!enable)
 157                 return;
 158
 159         freq = tsc_frequency;
 160         while (freq > TSC_CPUTIMER_FREQMAX) {
 161                 freq >>= 1;
 162                 ++tsc_cputimer_shift;
 163         }
 164         kprintf("TSC: cputimer freq %ju, shift %d\n",
 165             (uintmax_t)freq, tsc_cputimer_shift);
 166
 167         tsc_cputimer.freq = freq;
 168
 169         if (cpu_vendor_id == CPU_VENDOR_INTEL)
 170                 tsc_cputimer.count = tsc_cputimer_count_lfence;
 171         else
 172                 tsc_cputimer.count = tsc_cputimer_count_mfence; /* safe bet */
 173
 174         cputimer_register(&tsc_cputimer);
 175         cputimer_select(&tsc_cputimer, 0);
 176
 177         tsc_cpucounter.flags |= CPUCOUNTER_FLAG_MPSYNC;
 178 #ifndef _KERNEL_VIRTUAL
 179 regcnt:
 180 #endif
 181         tsc_cpucounter.freq = tsc_frequency;
 182         if (cpu_vendor_id == CPU_VENDOR_INTEL) {
 183                 tsc_cpucounter.count =
 184                     tsc_cpucounter_count_lfence;
 185         } else {
 186                 tsc_cpucounter.count =
 187                     tsc_cpucounter_count_mfence; /* safe bet */
 188         }
 189         cpucounter_register(&tsc_cpucounter);
 190 }
 191 SYSINIT(tsc_cputimer_reg, SI_BOOT2_POST_SMP, SI_ORDER_FIRST,
 192         tsc_cputimer_register, NULL);