Commit | Line | Data |
---|---|---|
c8fe38ae MD |
1 | /*- |
2 | * Copyright (c) 1990 The Regents of the University of California. | |
3 | * Copyright (c) 2008 The DragonFly Project. | |
4 | * All rights reserved. | |
5 | * | |
6 | * This code is derived from software contributed to Berkeley by | |
7 | * William Jolitz and Don Ahn. | |
8 | * | |
9 | * Redistribution and use in source and binary forms, with or without | |
10 | * modification, are permitted provided that the following conditions | |
11 | * are met: | |
12 | * 1. Redistributions of source code must retain the above copyright | |
13 | * notice, this list of conditions and the following disclaimer. | |
14 | * 2. Redistributions in binary form must reproduce the above copyright | |
15 | * notice, this list of conditions and the following disclaimer in the | |
16 | * documentation and/or other materials provided with the distribution. | |
17 | * 3. All advertising materials mentioning features or use of this software | |
18 | * must display the following acknowledgement: | |
19 | * This product includes software developed by the University of | |
20 | * California, Berkeley and its contributors. | |
21 | * 4. Neither the name of the University nor the names of its contributors | |
22 | * may be used to endorse or promote products derived from this software | |
23 | * without specific prior written permission. | |
24 | * | |
25 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND | |
26 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |
27 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |
28 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE | |
29 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |
30 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | |
31 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | |
32 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | |
33 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | |
34 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |
35 | * SUCH DAMAGE. | |
36 | * | |
37 | * from: @(#)clock.c 7.2 (Berkeley) 5/12/91 | |
38 | * $FreeBSD: src/sys/i386/isa/clock.c,v 1.149.2.6 2002/11/02 04:41:50 iwasaki Exp $ | |
c8fe38ae MD |
39 | */ |
40 | ||
41 | /* | |
42 | * Routines to handle clock hardware. | |
43 | */ | |
44 | ||
45 | /* | |
46 | * inittodr, settodr and support routines written | |
47 | * by Christoph Robitschko <chmr@edvz.tu-graz.ac.at> | |
48 | * | |
49 | * reintroduced and updated by Chris Stenton <chris@gnome.co.uk> 8/10/94 | |
50 | */ | |
51 | ||
40672791 | 52 | #if 0 |
40672791 SZ |
53 | #include "opt_clock.h" |
54 | #endif | |
c8fe38ae MD |
55 | |
56 | #include <sys/param.h> | |
57 | #include <sys/systm.h> | |
58 | #include <sys/eventhandler.h> | |
59 | #include <sys/time.h> | |
60 | #include <sys/kernel.h> | |
61 | #include <sys/bus.h> | |
c8fe38ae MD |
62 | #include <sys/sysctl.h> |
63 | #include <sys/cons.h> | |
ce7866b8 | 64 | #include <sys/kbio.h> |
c8fe38ae MD |
65 | #include <sys/systimer.h> |
66 | #include <sys/globaldata.h> | |
c8fe38ae | 67 | #include <sys/machintr.h> |
1b505979 | 68 | #include <sys/interrupt.h> |
c8fe38ae | 69 | |
ce7866b8 MD |
70 | #include <sys/thread2.h> |
71 | ||
c8fe38ae | 72 | #include <machine/clock.h> |
c8fe38ae MD |
73 | #include <machine/cputypes.h> |
74 | #include <machine/frame.h> | |
75 | #include <machine/ipl.h> | |
76 | #include <machine/limits.h> | |
77 | #include <machine/md_var.h> | |
78 | #include <machine/psl.h> | |
79 | #include <machine/segments.h> | |
80 | #include <machine/smp.h> | |
81 | #include <machine/specialreg.h> | |
57a9c56b | 82 | #include <machine/intr_machdep.h> |
c8fe38ae | 83 | |
ed4d621d | 84 | #include <machine_base/apic/ioapic.h> |
6b809ec7 | 85 | #include <machine_base/apic/ioapic_abi.h> |
c8fe38ae | 86 | #include <machine_base/icu/icu.h> |
0855a2af | 87 | #include <bus/isa/isa.h> |
c8fe38ae MD |
88 | #include <bus/isa/rtc.h> |
89 | #include <machine_base/isa/timerreg.h> | |
90 | ||
c8fe38ae MD |
91 | static void i8254_restore(void); |
92 | static void resettodr_on_shutdown(void *arg __unused); | |
93 | ||
94 | /* | |
95 | * 32-bit time_t's can't reach leap years before 1904 or after 2036, so we | |
96 | * can use a simple formula for leap years. | |
97 | */ | |
98 | #define LEAPYEAR(y) ((u_int)(y) % 4 == 0) | |
99 | #define DAYSPERYEAR (31+28+31+30+31+30+31+31+30+31+30+31) | |
100 | ||
101 | #ifndef TIMER_FREQ | |
102 | #define TIMER_FREQ 1193182 | |
103 | #endif | |
104 | ||
105 | static uint8_t i8254_walltimer_sel; | |
106 | static uint16_t i8254_walltimer_cntr; | |
107 | ||
108 | int adjkerntz; /* local offset from GMT in seconds */ | |
109 | int disable_rtc_set; /* disable resettodr() if != 0 */ | |
c8fe38ae | 110 | int tsc_present; |
5a81b19f | 111 | int tsc_invariant; |
dda44f1e | 112 | int tsc_mpsync; |
c8fe38ae MD |
113 | int64_t tsc_frequency; |
114 | int tsc_is_broken; | |
115 | int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */ | |
116 | int timer0_running; | |
117 | enum tstate { RELEASED, ACQUIRED }; | |
118 | enum tstate timer0_state; | |
119 | enum tstate timer1_state; | |
120 | enum tstate timer2_state; | |
121 | ||
122 | static int beeping = 0; | |
123 | static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31}; | |
124 | static u_char rtc_statusa = RTCSA_DIVIDER | RTCSA_NOPROF; | |
125 | static u_char rtc_statusb = RTCSB_24HR | RTCSB_PINTR; | |
126 | static int rtc_loaded; | |
127 | ||
128 | static int i8254_cputimer_div; | |
129 | ||
40672791 | 130 | static int i8254_nointr; |
4d517764 | 131 | static int i8254_intr_disable = 1; |
40672791 SZ |
132 | TUNABLE_INT("hw.i8254.intr_disable", &i8254_intr_disable); |
133 | ||
c8fe38ae MD |
134 | static struct callout sysbeepstop_ch; |
135 | ||
136 | static sysclock_t i8254_cputimer_count(void); | |
137 | static void i8254_cputimer_construct(struct cputimer *cputimer, sysclock_t last); | |
138 | static void i8254_cputimer_destruct(struct cputimer *cputimer); | |
139 | ||
140 | static struct cputimer i8254_cputimer = { | |
141 | SLIST_ENTRY_INITIALIZER, | |
142 | "i8254", | |
143 | CPUTIMER_PRI_8254, | |
144 | 0, | |
145 | i8254_cputimer_count, | |
146 | cputimer_default_fromhz, | |
147 | cputimer_default_fromus, | |
148 | i8254_cputimer_construct, | |
149 | i8254_cputimer_destruct, | |
150 | TIMER_FREQ, | |
151 | 0, 0, 0 | |
152 | }; | |
153 | ||
59878316 SZ |
154 | static sysclock_t tsc_cputimer_count_mfence(void); |
155 | static sysclock_t tsc_cputimer_count_lfence(void); | |
8d23b56c SZ |
156 | static void tsc_cputimer_construct(struct cputimer *, sysclock_t); |
157 | ||
158 | static struct cputimer tsc_cputimer = { | |
159 | SLIST_ENTRY_INITIALIZER, | |
160 | "TSC", | |
161 | CPUTIMER_PRI_TSC, | |
162 | CPUTIMER_TSC, | |
59878316 | 163 | tsc_cputimer_count_mfence, /* safe bet */ |
8d23b56c SZ |
164 | cputimer_default_fromhz, |
165 | cputimer_default_fromus, | |
166 | tsc_cputimer_construct, | |
167 | cputimer_default_destruct, | |
168 | 0, | |
169 | 0, 0, 0 | |
170 | }; | |
171 | ||
40672791 SZ |
172 | static void i8254_intr_reload(struct cputimer_intr *, sysclock_t); |
173 | static void i8254_intr_config(struct cputimer_intr *, const struct cputimer *); | |
174 | static void i8254_intr_initclock(struct cputimer_intr *, boolean_t); | |
175 | ||
176 | static struct cputimer_intr i8254_cputimer_intr = { | |
177 | .freq = TIMER_FREQ, | |
178 | .reload = i8254_intr_reload, | |
179 | .enable = cputimer_intr_default_enable, | |
180 | .config = i8254_intr_config, | |
181 | .restart = cputimer_intr_default_restart, | |
182 | .pmfixup = cputimer_intr_default_pmfixup, | |
183 | .initclock = i8254_intr_initclock, | |
184 | .next = SLIST_ENTRY_INITIALIZER, | |
185 | .name = "i8254", | |
186 | .type = CPUTIMER_INTR_8254, | |
187 | .prio = CPUTIMER_INTR_PRIO_8254, | |
188 | .caps = CPUTIMER_INTR_CAP_PS | |
189 | }; | |
190 | ||
c8fe38ae MD |
191 | /* |
192 | * timer0 clock interrupt. Timer0 is in one-shot mode and has stopped | |
193 | * counting as of this interrupt. We use timer1 in free-running mode (not | |
194 | * generating any interrupts) as our main counter. Each cpu has timeouts | |
195 | * pending. | |
196 | * | |
197 | * This code is INTR_MPSAFE and may be called without the BGL held. | |
198 | */ | |
199 | static void | |
200 | clkintr(void *dummy, void *frame_arg) | |
201 | { | |
202 | static sysclock_t sysclock_count; /* NOTE! Must be static */ | |
203 | struct globaldata *gd = mycpu; | |
c8fe38ae MD |
204 | struct globaldata *gscan; |
205 | int n; | |
c8fe38ae MD |
206 | |
207 | /* | |
208 | * SWSTROBE mode is a one-shot, the timer is no longer running | |
209 | */ | |
210 | timer0_running = 0; | |
211 | ||
212 | /* | |
213 | * XXX the dispatcher needs work. right now we call systimer_intr() | |
214 | * directly or via IPI for any cpu with systimers queued, which is | |
215 | * usually *ALL* of them. We need to use the LAPIC timer for this. | |
216 | */ | |
217 | sysclock_count = sys_cputimer->count(); | |
c8fe38ae MD |
218 | for (n = 0; n < ncpus; ++n) { |
219 | gscan = globaldata_find(n); | |
220 | if (TAILQ_FIRST(&gscan->gd_systimerq) == NULL) | |
221 | continue; | |
222 | if (gscan != gd) { | |
223 | lwkt_send_ipiq3(gscan, (ipifunc3_t)systimer_intr, | |
96d52ac8 | 224 | &sysclock_count, 1); |
c8fe38ae MD |
225 | } else { |
226 | systimer_intr(&sysclock_count, 0, frame_arg); | |
227 | } | |
228 | } | |
c8fe38ae MD |
229 | } |
230 | ||
231 | ||
232 | /* | |
233 | * NOTE! not MP safe. | |
234 | */ | |
235 | int | |
236 | acquire_timer2(int mode) | |
237 | { | |
238 | if (timer2_state != RELEASED) | |
239 | return (-1); | |
240 | timer2_state = ACQUIRED; | |
241 | ||
242 | /* | |
243 | * This access to the timer registers is as atomic as possible | |
244 | * because it is a single instruction. We could do better if we | |
245 | * knew the rate. | |
246 | */ | |
247 | outb(TIMER_MODE, TIMER_SEL2 | (mode & 0x3f)); | |
248 | return (0); | |
249 | } | |
250 | ||
251 | int | |
252 | release_timer2(void) | |
253 | { | |
254 | if (timer2_state != ACQUIRED) | |
255 | return (-1); | |
256 | outb(TIMER_MODE, TIMER_SEL2 | TIMER_SQWAVE | TIMER_16BIT); | |
257 | timer2_state = RELEASED; | |
258 | return (0); | |
259 | } | |
260 | ||
c8fe38ae MD |
261 | #include "opt_ddb.h" |
262 | #ifdef DDB | |
263 | #include <ddb/ddb.h> | |
264 | ||
265 | DB_SHOW_COMMAND(rtc, rtc) | |
266 | { | |
267 | kprintf("%02x/%02x/%02x %02x:%02x:%02x, A = %02x, B = %02x, C = %02x\n", | |
268 | rtcin(RTC_YEAR), rtcin(RTC_MONTH), rtcin(RTC_DAY), | |
269 | rtcin(RTC_HRS), rtcin(RTC_MIN), rtcin(RTC_SEC), | |
270 | rtcin(RTC_STATUSA), rtcin(RTC_STATUSB), rtcin(RTC_INTR)); | |
271 | } | |
272 | #endif /* DDB */ | |
273 | ||
274 | /* | |
275 | * Return the current cpu timer count as a 32 bit integer. | |
276 | */ | |
277 | static | |
278 | sysclock_t | |
279 | i8254_cputimer_count(void) | |
280 | { | |
e28c8ef4 SW |
281 | static uint16_t cputimer_last; |
282 | uint16_t count; | |
c8fe38ae MD |
283 | sysclock_t ret; |
284 | ||
285 | clock_lock(); | |
286 | outb(TIMER_MODE, i8254_walltimer_sel | TIMER_LATCH); | |
e28c8ef4 SW |
287 | count = (uint8_t)inb(i8254_walltimer_cntr); /* get countdown */ |
288 | count |= ((uint8_t)inb(i8254_walltimer_cntr) << 8); | |
c8fe38ae MD |
289 | count = -count; /* -> countup */ |
290 | if (count < cputimer_last) /* rollover */ | |
291 | i8254_cputimer.base += 0x00010000; | |
292 | ret = i8254_cputimer.base | count; | |
293 | cputimer_last = count; | |
294 | clock_unlock(); | |
295 | return(ret); | |
296 | } | |
297 | ||
298 | /* | |
299 | * This function is called whenever the system timebase changes, allowing | |
300 | * us to calculate what is needed to convert a system timebase tick | |
301 | * into an 8254 tick for the interrupt timer. If we can convert to a | |
302 | * simple shift, multiplication, or division, we do so. Otherwise 64 | |
303 | * bit arithmatic is required every time the interrupt timer is reloaded. | |
304 | */ | |
40672791 SZ |
305 | static void |
306 | i8254_intr_config(struct cputimer_intr *cti, const struct cputimer *timer) | |
c8fe38ae MD |
307 | { |
308 | int freq; | |
309 | int div; | |
310 | ||
311 | /* | |
312 | * Will a simple divide do the trick? | |
313 | */ | |
40672791 SZ |
314 | div = (timer->freq + (cti->freq / 2)) / cti->freq; |
315 | freq = cti->freq * div; | |
c8fe38ae MD |
316 | |
317 | if (freq >= timer->freq - 1 && freq <= timer->freq + 1) | |
318 | i8254_cputimer_div = div; | |
319 | else | |
320 | i8254_cputimer_div = 0; | |
321 | } | |
322 | ||
323 | /* | |
324 | * Reload for the next timeout. It is possible for the reload value | |
325 | * to be 0 or negative, indicating that an immediate timer interrupt | |
326 | * is desired. For now make the minimum 2 ticks. | |
327 | * | |
328 | * We may have to convert from the system timebase to the 8254 timebase. | |
329 | */ | |
c5b8324c | 330 | static void |
40672791 | 331 | i8254_intr_reload(struct cputimer_intr *cti, sysclock_t reload) |
c8fe38ae | 332 | { |
e28c8ef4 | 333 | uint16_t count; |
c8fe38ae MD |
334 | |
335 | if (i8254_cputimer_div) | |
336 | reload /= i8254_cputimer_div; | |
337 | else | |
40672791 | 338 | reload = (int64_t)reload * cti->freq / sys_cputimer->freq; |
c8fe38ae MD |
339 | |
340 | if ((int)reload < 2) | |
341 | reload = 2; | |
342 | ||
343 | clock_lock(); | |
344 | if (timer0_running) { | |
345 | outb(TIMER_MODE, TIMER_SEL0 | TIMER_LATCH); /* count-down timer */ | |
e28c8ef4 SW |
346 | count = (uint8_t)inb(TIMER_CNTR0); /* lsb */ |
347 | count |= ((uint8_t)inb(TIMER_CNTR0) << 8); /* msb */ | |
c8fe38ae MD |
348 | if (reload < count) { |
349 | outb(TIMER_MODE, TIMER_SEL0 | TIMER_SWSTROBE | TIMER_16BIT); | |
e28c8ef4 SW |
350 | outb(TIMER_CNTR0, (uint8_t)reload); /* lsb */ |
351 | outb(TIMER_CNTR0, (uint8_t)(reload >> 8)); /* msb */ | |
c8fe38ae MD |
352 | } |
353 | } else { | |
354 | timer0_running = 1; | |
355 | if (reload > 0xFFFF) | |
356 | reload = 0; /* full count */ | |
357 | outb(TIMER_MODE, TIMER_SEL0 | TIMER_SWSTROBE | TIMER_16BIT); | |
e28c8ef4 SW |
358 | outb(TIMER_CNTR0, (uint8_t)reload); /* lsb */ |
359 | outb(TIMER_CNTR0, (uint8_t)(reload >> 8)); /* msb */ | |
c8fe38ae MD |
360 | } |
361 | clock_unlock(); | |
362 | } | |
363 | ||
364 | /* | |
365 | * DELAY(usec) - Spin for the specified number of microseconds. | |
366 | * DRIVERSLEEP(usec) - Spin for the specified number of microseconds, | |
367 | * but do a thread switch in the loop | |
368 | * | |
369 | * Relies on timer 1 counting down from (cputimer_freq / hz) | |
370 | * Note: timer had better have been programmed before this is first used! | |
371 | */ | |
372 | static void | |
373 | DODELAY(int n, int doswitch) | |
374 | { | |
8a224941 SZ |
375 | ssysclock_t delta, ticks_left; |
376 | sysclock_t prev_tick, tick; | |
c8fe38ae MD |
377 | |
378 | #ifdef DELAYDEBUG | |
379 | int getit_calls = 1; | |
380 | int n1; | |
381 | static int state = 0; | |
382 | ||
383 | if (state == 0) { | |
384 | state = 1; | |
385 | for (n1 = 1; n1 <= 10000000; n1 *= 10) | |
386 | DELAY(n1); | |
387 | state = 2; | |
388 | } | |
389 | if (state == 1) | |
390 | kprintf("DELAY(%d)...", n); | |
391 | #endif | |
392 | /* | |
393 | * Guard against the timer being uninitialized if we are called | |
394 | * early for console i/o. | |
395 | */ | |
396 | if (timer0_state == RELEASED) | |
397 | i8254_restore(); | |
398 | ||
399 | /* | |
400 | * Read the counter first, so that the rest of the setup overhead is | |
401 | * counted. Then calculate the number of hardware timer ticks | |
402 | * required, rounding up to be sure we delay at least the requested | |
403 | * number of microseconds. | |
404 | */ | |
405 | prev_tick = sys_cputimer->count(); | |
406 | ticks_left = ((u_int)n * (int64_t)sys_cputimer->freq + 999999) / | |
407 | 1000000; | |
408 | ||
409 | /* | |
410 | * Loop until done. | |
411 | */ | |
412 | while (ticks_left > 0) { | |
413 | tick = sys_cputimer->count(); | |
414 | #ifdef DELAYDEBUG | |
415 | ++getit_calls; | |
416 | #endif | |
417 | delta = tick - prev_tick; | |
418 | prev_tick = tick; | |
419 | if (delta < 0) | |
420 | delta = 0; | |
421 | ticks_left -= delta; | |
422 | if (doswitch && ticks_left > 0) | |
423 | lwkt_switch(); | |
c5724852 | 424 | cpu_pause(); |
c8fe38ae MD |
425 | } |
426 | #ifdef DELAYDEBUG | |
427 | if (state == 1) | |
428 | kprintf(" %d calls to getit() at %d usec each\n", | |
429 | getit_calls, (n + 5) / getit_calls); | |
430 | #endif | |
431 | } | |
432 | ||
77912481 MD |
433 | /* |
434 | * DELAY() never switches. | |
435 | */ | |
c8fe38ae MD |
436 | void |
437 | DELAY(int n) | |
438 | { | |
439 | DODELAY(n, 0); | |
440 | } | |
441 | ||
d8129ed3 MD |
442 | /* |
443 | * Returns non-zero if the specified time period has elapsed. Call | |
444 | * first with last_clock set to 0. | |
445 | */ | |
446 | int | |
447 | CHECKTIMEOUT(TOTALDELAY *tdd) | |
448 | { | |
449 | sysclock_t delta; | |
450 | int us; | |
451 | ||
452 | if (tdd->started == 0) { | |
453 | if (timer0_state == RELEASED) | |
454 | i8254_restore(); | |
455 | tdd->last_clock = sys_cputimer->count(); | |
456 | tdd->started = 1; | |
457 | return(0); | |
458 | } | |
459 | delta = sys_cputimer->count() - tdd->last_clock; | |
460 | us = (u_int64_t)delta * (u_int64_t)1000000 / | |
461 | (u_int64_t)sys_cputimer->freq; | |
462 | tdd->last_clock += (u_int64_t)us * (u_int64_t)sys_cputimer->freq / | |
463 | 1000000; | |
464 | tdd->us -= us; | |
465 | return (tdd->us < 0); | |
466 | } | |
467 | ||
468 | ||
77912481 MD |
469 | /* |
470 | * DRIVERSLEEP() does not switch if called with a spinlock held or | |
471 | * from a hard interrupt. | |
472 | */ | |
c8fe38ae MD |
473 | void |
474 | DRIVERSLEEP(int usec) | |
475 | { | |
476 | globaldata_t gd = mycpu; | |
477 | ||
0846e4ce | 478 | if (gd->gd_intr_nesting_level || gd->gd_spinlocks) { |
c8fe38ae MD |
479 | DODELAY(usec, 0); |
480 | } else { | |
481 | DODELAY(usec, 1); | |
482 | } | |
483 | } | |
484 | ||
485 | static void | |
486 | sysbeepstop(void *chan) | |
487 | { | |
488 | outb(IO_PPI, inb(IO_PPI)&0xFC); /* disable counter2 output to speaker */ | |
489 | beeping = 0; | |
490 | release_timer2(); | |
491 | } | |
492 | ||
493 | int | |
494 | sysbeep(int pitch, int period) | |
495 | { | |
496 | if (acquire_timer2(TIMER_SQWAVE|TIMER_16BIT)) | |
497 | return(-1); | |
7caeaffe MD |
498 | if (sysbeep_enable == 0) |
499 | return(-1); | |
c8fe38ae MD |
500 | /* |
501 | * Nobody else is using timer2, we do not need the clock lock | |
502 | */ | |
503 | outb(TIMER_CNTR2, pitch); | |
504 | outb(TIMER_CNTR2, (pitch>>8)); | |
505 | if (!beeping) { | |
506 | /* enable counter2 output to speaker */ | |
507 | outb(IO_PPI, inb(IO_PPI) | 3); | |
508 | beeping = period; | |
509 | callout_reset(&sysbeepstop_ch, period, sysbeepstop, NULL); | |
510 | } | |
511 | return (0); | |
512 | } | |
513 | ||
514 | /* | |
515 | * RTC support routines | |
516 | */ | |
517 | ||
518 | int | |
519 | rtcin(int reg) | |
520 | { | |
521 | u_char val; | |
522 | ||
523 | crit_enter(); | |
524 | outb(IO_RTC, reg); | |
525 | inb(0x84); | |
526 | val = inb(IO_RTC + 1); | |
527 | inb(0x84); | |
528 | crit_exit(); | |
529 | return (val); | |
530 | } | |
531 | ||
532 | static __inline void | |
533 | writertc(u_char reg, u_char val) | |
534 | { | |
535 | crit_enter(); | |
536 | inb(0x84); | |
537 | outb(IO_RTC, reg); | |
538 | inb(0x84); | |
539 | outb(IO_RTC + 1, val); | |
540 | inb(0x84); /* XXX work around wrong order in rtcin() */ | |
541 | crit_exit(); | |
542 | } | |
543 | ||
544 | static __inline int | |
545 | readrtc(int port) | |
546 | { | |
547 | return(bcd2bin(rtcin(port))); | |
548 | } | |
549 | ||
550 | static u_int | |
551 | calibrate_clocks(void) | |
552 | { | |
553 | u_int64_t old_tsc; | |
8a224941 SZ |
554 | u_int tot_count; |
555 | sysclock_t count, prev_count; | |
c8fe38ae MD |
556 | int sec, start_sec, timeout; |
557 | ||
558 | if (bootverbose) | |
5a81b19f | 559 | kprintf("Calibrating clock(s) ...\n"); |
c8fe38ae MD |
560 | if (!(rtcin(RTC_STATUSD) & RTCSD_PWR)) |
561 | goto fail; | |
562 | timeout = 100000000; | |
563 | ||
564 | /* Read the mc146818A seconds counter. */ | |
565 | for (;;) { | |
566 | if (!(rtcin(RTC_STATUSA) & RTCSA_TUP)) { | |
567 | sec = rtcin(RTC_SEC); | |
568 | break; | |
569 | } | |
570 | if (--timeout == 0) | |
571 | goto fail; | |
572 | } | |
573 | ||
574 | /* Wait for the mC146818A seconds counter to change. */ | |
575 | start_sec = sec; | |
576 | for (;;) { | |
577 | if (!(rtcin(RTC_STATUSA) & RTCSA_TUP)) { | |
578 | sec = rtcin(RTC_SEC); | |
579 | if (sec != start_sec) | |
580 | break; | |
581 | } | |
582 | if (--timeout == 0) | |
583 | goto fail; | |
584 | } | |
585 | ||
586 | /* Start keeping track of the i8254 counter. */ | |
587 | prev_count = sys_cputimer->count(); | |
588 | tot_count = 0; | |
589 | ||
590 | if (tsc_present) | |
591 | old_tsc = rdtsc(); | |
592 | else | |
593 | old_tsc = 0; /* shut up gcc */ | |
594 | ||
595 | /* | |
596 | * Wait for the mc146818A seconds counter to change. Read the i8254 | |
597 | * counter for each iteration since this is convenient and only | |
598 | * costs a few usec of inaccuracy. The timing of the final reads | |
599 | * of the counters almost matches the timing of the initial reads, | |
600 | * so the main cause of inaccuracy is the varying latency from | |
601 | * inside getit() or rtcin(RTC_STATUSA) to the beginning of the | |
602 | * rtcin(RTC_SEC) that returns a changed seconds count. The | |
603 | * maximum inaccuracy from this cause is < 10 usec on 486's. | |
604 | */ | |
605 | start_sec = sec; | |
606 | for (;;) { | |
607 | if (!(rtcin(RTC_STATUSA) & RTCSA_TUP)) | |
608 | sec = rtcin(RTC_SEC); | |
609 | count = sys_cputimer->count(); | |
610 | tot_count += (int)(count - prev_count); | |
611 | prev_count = count; | |
612 | if (sec != start_sec) | |
613 | break; | |
614 | if (--timeout == 0) | |
615 | goto fail; | |
616 | } | |
617 | ||
618 | /* | |
619 | * Read the cpu cycle counter. The timing considerations are | |
620 | * similar to those for the i8254 clock. | |
621 | */ | |
622 | if (tsc_present) { | |
623 | tsc_frequency = rdtsc() - old_tsc; | |
624 | } | |
625 | ||
5a81b19f SZ |
626 | if (tsc_present) { |
627 | kprintf("TSC%s clock: %llu Hz, ", | |
628 | tsc_invariant ? " invariant" : "", | |
629 | (long long)tsc_frequency); | |
630 | } | |
c8fe38ae MD |
631 | kprintf("i8254 clock: %u Hz\n", tot_count); |
632 | return (tot_count); | |
633 | ||
634 | fail: | |
635 | kprintf("failed, using default i8254 clock of %u Hz\n", | |
636 | i8254_cputimer.freq); | |
637 | return (i8254_cputimer.freq); | |
638 | } | |
639 | ||
640 | static void | |
641 | i8254_restore(void) | |
642 | { | |
643 | timer0_state = ACQUIRED; | |
644 | ||
645 | clock_lock(); | |
646 | ||
647 | /* | |
648 | * Timer0 is our fine-grained variable clock interrupt | |
649 | */ | |
650 | outb(TIMER_MODE, TIMER_SEL0 | TIMER_SWSTROBE | TIMER_16BIT); | |
651 | outb(TIMER_CNTR0, 2); /* lsb */ | |
652 | outb(TIMER_CNTR0, 0); /* msb */ | |
653 | clock_unlock(); | |
654 | ||
40672791 SZ |
655 | if (!i8254_nointr) { |
656 | cputimer_intr_register(&i8254_cputimer_intr); | |
657 | cputimer_intr_select(&i8254_cputimer_intr, 0); | |
658 | } | |
659 | ||
c8fe38ae MD |
660 | /* |
661 | * Timer1 or timer2 is our free-running clock, but only if another | |
662 | * has not been selected. | |
663 | */ | |
664 | cputimer_register(&i8254_cputimer); | |
665 | cputimer_select(&i8254_cputimer, 0); | |
666 | } | |
667 | ||
668 | static void | |
669 | i8254_cputimer_construct(struct cputimer *timer, sysclock_t oldclock) | |
670 | { | |
671 | int which; | |
672 | ||
673 | /* | |
674 | * Should we use timer 1 or timer 2 ? | |
675 | */ | |
676 | which = 0; | |
677 | TUNABLE_INT_FETCH("hw.i8254.walltimer", &which); | |
678 | if (which != 1 && which != 2) | |
679 | which = 2; | |
680 | ||
681 | switch(which) { | |
682 | case 1: | |
683 | timer->name = "i8254_timer1"; | |
684 | timer->type = CPUTIMER_8254_SEL1; | |
685 | i8254_walltimer_sel = TIMER_SEL1; | |
686 | i8254_walltimer_cntr = TIMER_CNTR1; | |
687 | timer1_state = ACQUIRED; | |
688 | break; | |
689 | case 2: | |
690 | timer->name = "i8254_timer2"; | |
691 | timer->type = CPUTIMER_8254_SEL2; | |
692 | i8254_walltimer_sel = TIMER_SEL2; | |
693 | i8254_walltimer_cntr = TIMER_CNTR2; | |
694 | timer2_state = ACQUIRED; | |
695 | break; | |
696 | } | |
697 | ||
698 | timer->base = (oldclock + 0xFFFF) & ~0xFFFF; | |
699 | ||
700 | clock_lock(); | |
701 | outb(TIMER_MODE, i8254_walltimer_sel | TIMER_RATEGEN | TIMER_16BIT); | |
702 | outb(i8254_walltimer_cntr, 0); /* lsb */ | |
703 | outb(i8254_walltimer_cntr, 0); /* msb */ | |
704 | outb(IO_PPI, inb(IO_PPI) | 1); /* bit 0: enable gate, bit 1: spkr */ | |
705 | clock_unlock(); | |
706 | } | |
707 | ||
708 | static void | |
709 | i8254_cputimer_destruct(struct cputimer *timer) | |
710 | { | |
711 | switch(timer->type) { | |
712 | case CPUTIMER_8254_SEL1: | |
713 | timer1_state = RELEASED; | |
714 | break; | |
715 | case CPUTIMER_8254_SEL2: | |
716 | timer2_state = RELEASED; | |
717 | break; | |
718 | default: | |
719 | break; | |
720 | } | |
721 | timer->type = 0; | |
722 | } | |
723 | ||
724 | static void | |
725 | rtc_restore(void) | |
726 | { | |
727 | /* Restore all of the RTC's "status" (actually, control) registers. */ | |
728 | writertc(RTC_STATUSB, RTCSB_24HR); | |
729 | writertc(RTC_STATUSA, rtc_statusa); | |
730 | writertc(RTC_STATUSB, rtc_statusb); | |
731 | } | |
732 | ||
733 | /* | |
734 | * Restore all the timers. | |
735 | * | |
736 | * This function is called to resynchronize our core timekeeping after a | |
737 | * long halt, e.g. from apm_default_resume() and friends. It is also | |
738 | * called if after a BIOS call we have detected munging of the 8254. | |
739 | * It is necessary because cputimer_count() counter's delta may have grown | |
740 | * too large for nanouptime() and friends to handle, or (in the case of 8254 | |
741 | * munging) might cause the SYSTIMER code to prematurely trigger. | |
742 | */ | |
743 | void | |
744 | timer_restore(void) | |
745 | { | |
746 | crit_enter(); | |
747 | i8254_restore(); /* restore timer_freq and hz */ | |
748 | rtc_restore(); /* reenable RTC interrupts */ | |
749 | crit_exit(); | |
750 | } | |
751 | ||
752 | /* | |
753 | * Initialize 8254 timer 0 early so that it can be used in DELAY(). | |
754 | */ | |
755 | void | |
756 | startrtclock(void) | |
757 | { | |
758 | u_int delta, freq; | |
759 | ||
760 | /* | |
761 | * Can we use the TSC? | |
1997b4c2 MD |
762 | * |
763 | * NOTE: If running under qemu, probably a good idea to force the | |
764 | * TSC because we are not likely to detect it as being | |
765 | * invariant or mpsyncd if you don't. This will greatly | |
766 | * reduce SMP contention. | |
c8fe38ae | 767 | */ |
5a81b19f | 768 | if (cpu_feature & CPUID_TSC) { |
c8fe38ae | 769 | tsc_present = 1; |
1997b4c2 MD |
770 | TUNABLE_INT_FETCH("hw.tsc_cputimer_force", &tsc_invariant); |
771 | ||
5a81b19f SZ |
772 | if ((cpu_vendor_id == CPU_VENDOR_INTEL || |
773 | cpu_vendor_id == CPU_VENDOR_AMD) && | |
774 | cpu_exthigh >= 0x80000007) { | |
775 | u_int regs[4]; | |
776 | ||
777 | do_cpuid(0x80000007, regs); | |
778 | if (regs[3] & 0x100) | |
779 | tsc_invariant = 1; | |
780 | } | |
781 | } else { | |
c8fe38ae | 782 | tsc_present = 0; |
5a81b19f | 783 | } |
c8fe38ae MD |
784 | |
785 | /* | |
786 | * Initial RTC state, don't do anything unexpected | |
787 | */ | |
788 | writertc(RTC_STATUSA, rtc_statusa); | |
789 | writertc(RTC_STATUSB, RTCSB_24HR); | |
790 | ||
791 | /* | |
792 | * Set the 8254 timer0 in TIMER_SWSTROBE mode and cause it to | |
793 | * generate an interrupt, which we will ignore for now. | |
794 | * | |
795 | * Set the 8254 timer1 in TIMER_RATEGEN mode and load 0x0000 | |
796 | * (so it counts a full 2^16 and repeats). We will use this timer | |
797 | * for our counting. | |
798 | */ | |
799 | i8254_restore(); | |
800 | freq = calibrate_clocks(); | |
801 | #ifdef CLK_CALIBRATION_LOOP | |
802 | if (bootverbose) { | |
ce7866b8 MD |
803 | int c; |
804 | ||
805 | cnpoll(TRUE); | |
806 | kprintf("Press a key on the console to " | |
807 | "abort clock calibration\n"); | |
808 | while ((c = cncheckc()) == -1 || c == NOKEY) | |
c8fe38ae | 809 | calibrate_clocks(); |
ce7866b8 | 810 | cnpoll(FALSE); |
c8fe38ae MD |
811 | } |
812 | #endif | |
813 | ||
814 | /* | |
815 | * Use the calibrated i8254 frequency if it seems reasonable. | |
816 | * Otherwise use the default, and don't use the calibrated i586 | |
817 | * frequency. | |
818 | */ | |
819 | delta = freq > i8254_cputimer.freq ? | |
820 | freq - i8254_cputimer.freq : i8254_cputimer.freq - freq; | |
821 | if (delta < i8254_cputimer.freq / 100) { | |
822 | #ifndef CLK_USE_I8254_CALIBRATION | |
823 | if (bootverbose) | |
824 | kprintf( | |
825 | "CLK_USE_I8254_CALIBRATION not specified - using default frequency\n"); | |
826 | freq = i8254_cputimer.freq; | |
827 | #endif | |
40672791 SZ |
828 | /* |
829 | * NOTE: | |
830 | * Interrupt timer's freq must be adjusted | |
831 | * before we change the cuptimer's frequency. | |
832 | */ | |
833 | i8254_cputimer_intr.freq = freq; | |
c8fe38ae MD |
834 | cputimer_set_frequency(&i8254_cputimer, freq); |
835 | } else { | |
836 | if (bootverbose) | |
837 | kprintf( | |
838 | "%d Hz differs from default of %d Hz by more than 1%%\n", | |
839 | freq, i8254_cputimer.freq); | |
840 | tsc_frequency = 0; | |
841 | } | |
842 | ||
843 | #ifndef CLK_USE_TSC_CALIBRATION | |
844 | if (tsc_frequency != 0) { | |
845 | if (bootverbose) | |
846 | kprintf( | |
847 | "CLK_USE_TSC_CALIBRATION not specified - using old calibration method\n"); | |
848 | tsc_frequency = 0; | |
849 | } | |
850 | #endif | |
851 | if (tsc_present && tsc_frequency == 0) { | |
852 | /* | |
853 | * Calibration of the i586 clock relative to the mc146818A | |
854 | * clock failed. Do a less accurate calibration relative | |
855 | * to the i8254 clock. | |
856 | */ | |
857 | u_int64_t old_tsc = rdtsc(); | |
858 | ||
859 | DELAY(1000000); | |
860 | tsc_frequency = rdtsc() - old_tsc; | |
861 | #ifdef CLK_USE_TSC_CALIBRATION | |
862 | if (bootverbose) { | |
863 | kprintf("TSC clock: %llu Hz (Method B)\n", | |
864 | tsc_frequency); | |
865 | } | |
866 | #endif | |
867 | } | |
868 | ||
869 | EVENTHANDLER_REGISTER(shutdown_post_sync, resettodr_on_shutdown, NULL, SHUTDOWN_PRI_LAST); | |
c8fe38ae MD |
870 | } |
871 | ||
872 | /* | |
873 | * Sync the time of day back to the RTC on shutdown, but only if | |
874 | * we have already loaded it and have not crashed. | |
875 | */ | |
876 | static void | |
877 | resettodr_on_shutdown(void *arg __unused) | |
878 | { | |
879 | if (rtc_loaded && panicstr == NULL) { | |
880 | resettodr(); | |
881 | } | |
882 | } | |
883 | ||
884 | /* | |
885 | * Initialize the time of day register, based on the time base which is, e.g. | |
886 | * from a filesystem. | |
887 | */ | |
888 | void | |
889 | inittodr(time_t base) | |
890 | { | |
891 | unsigned long sec, days; | |
c8fe38ae MD |
892 | int year, month; |
893 | int y, m; | |
894 | struct timespec ts; | |
895 | ||
896 | if (base) { | |
897 | ts.tv_sec = base; | |
898 | ts.tv_nsec = 0; | |
899 | set_timeofday(&ts); | |
900 | } | |
901 | ||
902 | /* Look if we have a RTC present and the time is valid */ | |
903 | if (!(rtcin(RTC_STATUSD) & RTCSD_PWR)) | |
904 | goto wrong_time; | |
905 | ||
906 | /* wait for time update to complete */ | |
907 | /* If RTCSA_TUP is zero, we have at least 244us before next update */ | |
908 | crit_enter(); | |
909 | while (rtcin(RTC_STATUSA) & RTCSA_TUP) { | |
910 | crit_exit(); | |
911 | crit_enter(); | |
912 | } | |
913 | ||
914 | days = 0; | |
915 | #ifdef USE_RTC_CENTURY | |
916 | year = readrtc(RTC_YEAR) + readrtc(RTC_CENTURY) * 100; | |
917 | #else | |
918 | year = readrtc(RTC_YEAR) + 1900; | |
919 | if (year < 1970) | |
920 | year += 100; | |
921 | #endif | |
922 | if (year < 1970) { | |
923 | crit_exit(); | |
924 | goto wrong_time; | |
925 | } | |
926 | month = readrtc(RTC_MONTH); | |
927 | for (m = 1; m < month; m++) | |
928 | days += daysinmonth[m-1]; | |
929 | if ((month > 2) && LEAPYEAR(year)) | |
930 | days ++; | |
931 | days += readrtc(RTC_DAY) - 1; | |
c8fe38ae MD |
932 | for (y = 1970; y < year; y++) |
933 | days += DAYSPERYEAR + LEAPYEAR(y); | |
934 | sec = ((( days * 24 + | |
935 | readrtc(RTC_HRS)) * 60 + | |
936 | readrtc(RTC_MIN)) * 60 + | |
937 | readrtc(RTC_SEC)); | |
938 | /* sec now contains the number of seconds, since Jan 1 1970, | |
939 | in the local time zone */ | |
940 | ||
941 | sec += tz.tz_minuteswest * 60 + (wall_cmos_clock ? adjkerntz : 0); | |
942 | ||
cec73927 | 943 | y = (int)(time_second - sec); |
c8fe38ae MD |
944 | if (y <= -2 || y >= 2) { |
945 | /* badly off, adjust it */ | |
946 | ts.tv_sec = sec; | |
947 | ts.tv_nsec = 0; | |
948 | set_timeofday(&ts); | |
949 | } | |
950 | rtc_loaded = 1; | |
951 | crit_exit(); | |
952 | return; | |
953 | ||
954 | wrong_time: | |
955 | kprintf("Invalid time in real time clock.\n"); | |
956 | kprintf("Check and reset the date immediately!\n"); | |
957 | } | |
958 | ||
959 | /* | |
960 | * Write system time back to RTC | |
961 | */ | |
962 | void | |
963 | resettodr(void) | |
964 | { | |
965 | struct timeval tv; | |
966 | unsigned long tm; | |
967 | int m; | |
968 | int y; | |
969 | ||
970 | if (disable_rtc_set) | |
971 | return; | |
972 | ||
973 | microtime(&tv); | |
974 | tm = tv.tv_sec; | |
975 | ||
976 | crit_enter(); | |
977 | /* Disable RTC updates and interrupts. */ | |
978 | writertc(RTC_STATUSB, RTCSB_HALT | RTCSB_24HR); | |
979 | ||
980 | /* Calculate local time to put in RTC */ | |
981 | ||
982 | tm -= tz.tz_minuteswest * 60 + (wall_cmos_clock ? adjkerntz : 0); | |
983 | ||
984 | writertc(RTC_SEC, bin2bcd(tm%60)); tm /= 60; /* Write back Seconds */ | |
985 | writertc(RTC_MIN, bin2bcd(tm%60)); tm /= 60; /* Write back Minutes */ | |
986 | writertc(RTC_HRS, bin2bcd(tm%24)); tm /= 24; /* Write back Hours */ | |
987 | ||
988 | /* We have now the days since 01-01-1970 in tm */ | |
989 | writertc(RTC_WDAY, (tm+4)%7); /* Write back Weekday */ | |
990 | for (y = 1970, m = DAYSPERYEAR + LEAPYEAR(y); | |
991 | tm >= m; | |
992 | y++, m = DAYSPERYEAR + LEAPYEAR(y)) | |
993 | tm -= m; | |
994 | ||
995 | /* Now we have the years in y and the day-of-the-year in tm */ | |
996 | writertc(RTC_YEAR, bin2bcd(y%100)); /* Write back Year */ | |
997 | #ifdef USE_RTC_CENTURY | |
998 | writertc(RTC_CENTURY, bin2bcd(y/100)); /* ... and Century */ | |
999 | #endif | |
1000 | for (m = 0; ; m++) { | |
1001 | int ml; | |
1002 | ||
1003 | ml = daysinmonth[m]; | |
1004 | if (m == 1 && LEAPYEAR(y)) | |
1005 | ml++; | |
1006 | if (tm < ml) | |
1007 | break; | |
1008 | tm -= ml; | |
1009 | } | |
1010 | ||
1011 | writertc(RTC_MONTH, bin2bcd(m + 1)); /* Write back Month */ | |
1012 | writertc(RTC_DAY, bin2bcd(tm + 1)); /* Write back Month Day */ | |
1013 | ||
1014 | /* Reenable RTC updates and interrupts. */ | |
1015 | writertc(RTC_STATUSB, rtc_statusb); | |
1016 | crit_exit(); | |
1017 | } | |
1018 | ||
6b809ec7 SZ |
1019 | static int |
1020 | i8254_ioapic_trial(int irq, struct cputimer_intr *cti) | |
1021 | { | |
1022 | sysclock_t base; | |
1023 | long lastcnt; | |
1024 | ||
1025 | /* | |
1026 | * Following code assumes the 8254 is the cpu timer, | |
1027 | * so make sure it is. | |
1028 | */ | |
1029 | KKASSERT(sys_cputimer == &i8254_cputimer); | |
1030 | KKASSERT(cti == &i8254_cputimer_intr); | |
1031 | ||
c83c147e | 1032 | lastcnt = get_interrupt_counter(irq, mycpuid); |
6b809ec7 SZ |
1033 | |
1034 | /* | |
1035 | * Force an 8254 Timer0 interrupt and wait 1/100s for | |
1036 | * it to happen, then see if we got it. | |
1037 | */ | |
1038 | kprintf("IOAPIC: testing 8254 interrupt delivery\n"); | |
1039 | ||
1040 | i8254_intr_reload(cti, 2); | |
1041 | base = sys_cputimer->count(); | |
1042 | while (sys_cputimer->count() - base < sys_cputimer->freq / 100) | |
1043 | ; /* nothing */ | |
1044 | ||
c83c147e | 1045 | if (get_interrupt_counter(irq, mycpuid) - lastcnt == 0) |
6b809ec7 SZ |
1046 | return ENOENT; |
1047 | return 0; | |
1048 | } | |
1049 | ||
40672791 SZ |
1050 | /* |
1051 | * Start both clocks running. DragonFly note: the stat clock is no longer | |
1052 | * used. Instead, 8254 based systimers are used for all major clock | |
d426f67a | 1053 | * interrupts. |
40672791 SZ |
1054 | */ |
1055 | static void | |
1056 | i8254_intr_initclock(struct cputimer_intr *cti, boolean_t selected) | |
1057 | { | |
c79ae131 | 1058 | void *clkdesc = NULL; |
6b809ec7 | 1059 | int irq = 0, mixed_mode = 0, error; |
c8fe38ae | 1060 | |
6355d931 | 1061 | KKASSERT(mycpuid == 0); |
bf0ecf68 | 1062 | callout_init_mp(&sysbeepstop_ch); |
adc34348 | 1063 | |
6b809ec7 SZ |
1064 | if (!selected && i8254_intr_disable) |
1065 | goto nointr; | |
40672791 | 1066 | |
d426f67a SZ |
1067 | /* |
1068 | * The stat interrupt mask is different without the | |
1069 | * statistics clock. Also, don't set the interrupt | |
1070 | * flag which would normally cause the RTC to generate | |
1071 | * interrupts. | |
1072 | */ | |
1073 | rtc_statusb = RTCSB_24HR; | |
adc34348 | 1074 | |
da23a592 | 1075 | /* Finish initializing 8254 timer 0. */ |
f45bfca0 | 1076 | if (ioapic_enable) { |
86d692fe | 1077 | irq = machintr_legacy_intr_find(0, INTR_TRIGGER_EDGE, |
6b809ec7 SZ |
1078 | INTR_POLARITY_HIGH); |
1079 | if (irq < 0) { | |
1080 | mixed_mode_setup: | |
027bbbfe | 1081 | error = ioapic_conf_legacy_extint(0); |
6b809ec7 | 1082 | if (!error) { |
86d692fe | 1083 | irq = machintr_legacy_intr_find(0, |
027bbbfe | 1084 | INTR_TRIGGER_EDGE, INTR_POLARITY_HIGH); |
6b809ec7 SZ |
1085 | if (irq < 0) |
1086 | error = ENOENT; | |
1087 | } | |
1088 | ||
1089 | if (error) { | |
1090 | if (!selected) { | |
1091 | kprintf("IOAPIC: setup mixed mode for " | |
1092 | "irq 0 failed: %d\n", error); | |
1093 | goto nointr; | |
1094 | } else { | |
1095 | panic("IOAPIC: setup mixed mode for " | |
1096 | "irq 0 failed: %d\n", error); | |
1097 | } | |
1098 | } | |
1099 | mixed_mode = 1; | |
1100 | } | |
1101 | clkdesc = register_int(irq, clkintr, NULL, "clk", | |
1102 | NULL, | |
1103 | INTR_EXCL | INTR_CLOCK | | |
1104 | INTR_NOPOLL | INTR_MPSAFE | | |
6355d931 | 1105 | INTR_NOENTROPY, 0); |
7a603b36 | 1106 | } else { |
9a4bd8f3 SZ |
1107 | register_int(0, clkintr, NULL, "clk", NULL, |
1108 | INTR_EXCL | INTR_CLOCK | | |
1109 | INTR_NOPOLL | INTR_MPSAFE | | |
6355d931 | 1110 | INTR_NOENTROPY, 0); |
7a603b36 | 1111 | } |
adc34348 SZ |
1112 | |
1113 | /* Initialize RTC. */ | |
1114 | writertc(RTC_STATUSA, rtc_statusa); | |
1115 | writertc(RTC_STATUSB, RTCSB_24HR); | |
1116 | ||
f45bfca0 | 1117 | if (ioapic_enable) { |
7a603b36 SZ |
1118 | error = i8254_ioapic_trial(irq, cti); |
1119 | if (error) { | |
1120 | if (mixed_mode) { | |
1121 | if (!selected) { | |
1122 | kprintf("IOAPIC: mixed mode for irq %d " | |
1123 | "trial failed: %d\n", | |
1124 | irq, error); | |
1125 | goto nointr; | |
1126 | } else { | |
1127 | panic("IOAPIC: mixed mode for irq %d " | |
1128 | "trial failed: %d\n", irq, error); | |
1129 | } | |
6b809ec7 | 1130 | } else { |
7a603b36 SZ |
1131 | kprintf("IOAPIC: warning 8254 is not connected " |
1132 | "to the correct pin, try mixed mode\n"); | |
6355d931 | 1133 | unregister_int(clkdesc, 0); |
7a603b36 | 1134 | goto mixed_mode_setup; |
6b809ec7 | 1135 | } |
6b809ec7 SZ |
1136 | } |
1137 | } | |
6b809ec7 SZ |
1138 | return; |
1139 | ||
1140 | nointr: | |
1141 | i8254_nointr = 1; /* don't try to register again */ | |
1142 | cputimer_intr_deregister(cti); | |
c8fe38ae | 1143 | } |
c8fe38ae | 1144 | |
c8fe38ae MD |
1145 | void |
1146 | setstatclockrate(int newhz) | |
1147 | { | |
1148 | if (newhz == RTC_PROFRATE) | |
1149 | rtc_statusa = RTCSA_DIVIDER | RTCSA_PROF; | |
1150 | else | |
1151 | rtc_statusa = RTCSA_DIVIDER | RTCSA_NOPROF; | |
1152 | writertc(RTC_STATUSA, rtc_statusa); | |
1153 | } | |
1154 | ||
1155 | #if 0 | |
1156 | static unsigned | |
1157 | tsc_get_timecount(struct timecounter *tc) | |
1158 | { | |
1159 | return (rdtsc()); | |
1160 | } | |
1161 | #endif | |
1162 | ||
1163 | #ifdef KERN_TIMESTAMP | |
1164 | #define KERN_TIMESTAMP_SIZE 16384 | |
1165 | static u_long tsc[KERN_TIMESTAMP_SIZE] ; | |
1166 | SYSCTL_OPAQUE(_debug, OID_AUTO, timestamp, CTLFLAG_RD, tsc, | |
1167 | sizeof(tsc), "LU", "Kernel timestamps"); | |
1168 | void | |
1169 | _TSTMP(u_int32_t x) | |
1170 | { | |
1171 | static int i; | |
1172 | ||
1173 | tsc[i] = (u_int32_t)rdtsc(); | |
1174 | tsc[i+1] = x; | |
1175 | i = i + 2; | |
1176 | if (i >= KERN_TIMESTAMP_SIZE) | |
1177 | i = 0; | |
1178 | tsc[i] = 0; /* mark last entry */ | |
1179 | } | |
1180 | #endif /* KERN_TIMESTAMP */ | |
1181 | ||
1182 | /* | |
1183 | * | |
1184 | */ | |
1185 | ||
1186 | static int | |
1187 | hw_i8254_timestamp(SYSCTL_HANDLER_ARGS) | |
1188 | { | |
1189 | sysclock_t count; | |
e28c8ef4 | 1190 | uint64_t tscval; |
c8fe38ae MD |
1191 | char buf[32]; |
1192 | ||
1193 | crit_enter(); | |
1194 | if (sys_cputimer == &i8254_cputimer) | |
1195 | count = sys_cputimer->count(); | |
1196 | else | |
1197 | count = 0; | |
1198 | if (tsc_present) | |
1199 | tscval = rdtsc(); | |
1200 | else | |
1201 | tscval = 0; | |
1202 | crit_exit(); | |
1203 | ksnprintf(buf, sizeof(buf), "%08x %016llx", count, (long long)tscval); | |
1204 | return(SYSCTL_OUT(req, buf, strlen(buf) + 1)); | |
1205 | } | |
1206 | ||
00ec69c7 SZ |
1207 | struct tsc_mpsync_arg { |
1208 | volatile uint64_t tsc_target; | |
1209 | volatile int tsc_mpsync; | |
1210 | }; | |
1211 | ||
1212 | struct tsc_mpsync_thr { | |
1213 | volatile int tsc_done_cnt; | |
1214 | volatile int tsc_mpsync_cnt; | |
1215 | }; | |
dda44f1e SZ |
1216 | |
1217 | static void | |
00ec69c7 | 1218 | tsc_mpsync_test_remote(void *xarg) |
dda44f1e | 1219 | { |
00ec69c7 | 1220 | struct tsc_mpsync_arg *arg = xarg; |
dda44f1e SZ |
1221 | uint64_t tsc; |
1222 | ||
ea9728ca | 1223 | tsc = rdtsc_ordered(); |
00ec69c7 SZ |
1224 | if (tsc < arg->tsc_target) |
1225 | arg->tsc_mpsync = 0; | |
dda44f1e SZ |
1226 | } |
1227 | ||
1228 | static void | |
00ec69c7 | 1229 | tsc_mpsync_test_loop(struct tsc_mpsync_arg *arg) |
dda44f1e SZ |
1230 | { |
1231 | struct globaldata *gd = mycpu; | |
1232 | uint64_t test_end, test_begin; | |
1233 | u_int i; | |
1234 | ||
00ec69c7 SZ |
1235 | if (bootverbose) { |
1236 | kprintf("cpu%d: TSC testing MP synchronization ...\n", | |
1237 | gd->gd_cpuid); | |
1238 | } | |
1239 | ||
ea9728ca | 1240 | test_begin = rdtsc_ordered(); |
00ec69c7 SZ |
1241 | /* Run test for 100ms */ |
1242 | test_end = test_begin + (tsc_frequency / 10); | |
1243 | ||
1244 | arg->tsc_mpsync = 1; | |
1245 | arg->tsc_target = test_begin; | |
1246 | ||
1247 | #define TSC_TEST_TRYMAX 1000000 /* Make sure we could stop */ | |
1248 | #define TSC_TEST_TRYMIN 50000 | |
1249 | ||
1250 | for (i = 0; i < TSC_TEST_TRYMAX; ++i) { | |
1251 | struct lwkt_cpusync cs; | |
1252 | ||
1253 | crit_enter(); | |
1254 | lwkt_cpusync_init(&cs, gd->gd_other_cpus, | |
1255 | tsc_mpsync_test_remote, arg); | |
1256 | lwkt_cpusync_interlock(&cs); | |
ea9728ca | 1257 | arg->tsc_target = rdtsc_ordered(); |
00ec69c7 SZ |
1258 | cpu_mfence(); |
1259 | lwkt_cpusync_deinterlock(&cs); | |
1260 | crit_exit(); | |
1261 | ||
1262 | if (!arg->tsc_mpsync) { | |
1263 | kprintf("cpu%d: TSC is not MP synchronized @%u\n", | |
1264 | gd->gd_cpuid, i); | |
1265 | break; | |
1266 | } | |
1267 | if (arg->tsc_target > test_end && i >= TSC_TEST_TRYMIN) | |
1268 | break; | |
1269 | } | |
1270 | ||
1271 | #undef TSC_TEST_TRYMIN | |
1272 | #undef TSC_TEST_TRYMAX | |
1273 | ||
1274 | if (arg->tsc_target == test_begin) { | |
1275 | kprintf("cpu%d: TSC does not tick?!\n", gd->gd_cpuid); | |
1276 | /* XXX disable TSC? */ | |
1277 | tsc_invariant = 0; | |
1278 | arg->tsc_mpsync = 0; | |
1279 | return; | |
1280 | } | |
1281 | ||
1282 | if (arg->tsc_mpsync && bootverbose) { | |
1283 | kprintf("cpu%d: TSC is MP synchronized after %u tries\n", | |
1284 | gd->gd_cpuid, i); | |
1285 | } | |
1286 | } | |
1287 | ||
1288 | static void | |
1289 | tsc_mpsync_ap_thread(void *xthr) | |
1290 | { | |
1291 | struct tsc_mpsync_thr *thr = xthr; | |
1292 | struct tsc_mpsync_arg arg; | |
1293 | ||
1294 | tsc_mpsync_test_loop(&arg); | |
1295 | if (arg.tsc_mpsync) { | |
1296 | atomic_add_int(&thr->tsc_mpsync_cnt, 1); | |
1297 | cpu_sfence(); | |
1298 | } | |
1299 | atomic_add_int(&thr->tsc_done_cnt, 1); | |
1300 | ||
1301 | lwkt_exit(); | |
1302 | } | |
1303 | ||
1304 | static void | |
1305 | tsc_mpsync_test(void) | |
1306 | { | |
1307 | struct tsc_mpsync_arg arg; | |
1308 | ||
dda44f1e SZ |
1309 | if (!tsc_invariant) { |
1310 | /* Not even invariant TSC */ | |
1311 | return; | |
1312 | } | |
1313 | ||
1314 | if (ncpus == 1) { | |
1315 | /* Only one CPU */ | |
1316 | tsc_mpsync = 1; | |
1317 | return; | |
1318 | } | |
1319 | ||
1997b4c2 MD |
1320 | /* |
1321 | * Forcing can be used w/qemu to reduce contention | |
1322 | */ | |
1323 | TUNABLE_INT_FETCH("hw.tsc_cputimer_force", &tsc_mpsync); | |
1324 | if (tsc_mpsync) { | |
1325 | kprintf("TSC as cputimer forced\n"); | |
1326 | return; | |
1327 | } | |
1328 | ||
dda44f1e SZ |
1329 | if (cpu_vendor_id != CPU_VENDOR_INTEL) { |
1330 | /* XXX only Intel works */ | |
1331 | return; | |
1332 | } | |
1333 | ||
1334 | kprintf("TSC testing MP synchronization ...\n"); | |
dda44f1e | 1335 | |
00ec69c7 SZ |
1336 | tsc_mpsync_test_loop(&arg); |
1337 | if (arg.tsc_mpsync) { | |
1338 | struct tsc_mpsync_thr thr; | |
1339 | int cpu; | |
dda44f1e | 1340 | |
00ec69c7 SZ |
1341 | /* |
1342 | * Test TSC MP synchronization on APs. | |
1343 | */ | |
dda44f1e | 1344 | |
00ec69c7 SZ |
1345 | thr.tsc_done_cnt = 1; |
1346 | thr.tsc_mpsync_cnt = 1; | |
dda44f1e | 1347 | |
00ec69c7 SZ |
1348 | for (cpu = 0; cpu < ncpus; ++cpu) { |
1349 | if (cpu == mycpuid) | |
1350 | continue; | |
dda44f1e | 1351 | |
00ec69c7 SZ |
1352 | lwkt_create(tsc_mpsync_ap_thread, &thr, NULL, |
1353 | NULL, 0, cpu, "tsc mpsync %d", cpu); | |
dda44f1e | 1354 | } |
dda44f1e | 1355 | |
00ec69c7 SZ |
1356 | while (thr.tsc_done_cnt != ncpus) { |
1357 | cpu_pause(); | |
1358 | cpu_lfence(); | |
dda44f1e | 1359 | } |
00ec69c7 SZ |
1360 | if (thr.tsc_mpsync_cnt == ncpus) |
1361 | tsc_mpsync = 1; | |
dda44f1e | 1362 | } |
00ec69c7 SZ |
1363 | |
1364 | if (tsc_mpsync) | |
1365 | kprintf("TSC is MP synchronized\n"); | |
1366 | else | |
1367 | kprintf("TSC is not MP synchronized\n"); | |
dda44f1e SZ |
1368 | } |
1369 | SYSINIT(tsc_mpsync, SI_BOOT2_FINISH_SMP, SI_ORDER_ANY, tsc_mpsync_test, NULL); | |
1370 | ||
8d23b56c SZ |
1371 | #define TSC_CPUTIMER_FREQMAX 128000000 /* 128Mhz */ |
1372 | ||
1373 | static int tsc_cputimer_shift; | |
1374 | ||
1375 | static void | |
1376 | tsc_cputimer_construct(struct cputimer *timer, sysclock_t oldclock) | |
1377 | { | |
1378 | timer->base = 0; | |
59878316 | 1379 | timer->base = oldclock - timer->count(); |
8d23b56c SZ |
1380 | } |
1381 | ||
59878316 | 1382 | static __inline sysclock_t |
8d23b56c SZ |
1383 | tsc_cputimer_count(void) |
1384 | { | |
1385 | uint64_t tsc; | |
1386 | ||
1387 | tsc = rdtsc(); | |
1388 | tsc >>= tsc_cputimer_shift; | |
1389 | ||
1390 | return (tsc + tsc_cputimer.base); | |
1391 | } | |
1392 | ||
59878316 SZ |
1393 | static sysclock_t |
1394 | tsc_cputimer_count_lfence(void) | |
1395 | { | |
1396 | cpu_lfence(); | |
1397 | return tsc_cputimer_count(); | |
1398 | } | |
1399 | ||
1400 | static sysclock_t | |
1401 | tsc_cputimer_count_mfence(void) | |
1402 | { | |
1403 | cpu_mfence(); | |
1404 | return tsc_cputimer_count(); | |
1405 | } | |
1406 | ||
8d23b56c SZ |
1407 | static void |
1408 | tsc_cputimer_register(void) | |
1409 | { | |
1410 | uint64_t freq; | |
1411 | int enable = 1; | |
1412 | ||
1413 | if (!tsc_mpsync) | |
1414 | return; | |
1415 | ||
1416 | TUNABLE_INT_FETCH("hw.tsc_cputimer_enable", &enable); | |
1417 | if (!enable) | |
1418 | return; | |
1419 | ||
1420 | freq = tsc_frequency; | |
1421 | while (freq > TSC_CPUTIMER_FREQMAX) { | |
1422 | freq >>= 1; | |
1423 | ++tsc_cputimer_shift; | |
1424 | } | |
1425 | kprintf("TSC: cputimer freq %ju, shift %d\n", | |
1426 | (uintmax_t)freq, tsc_cputimer_shift); | |
1427 | ||
1428 | tsc_cputimer.freq = freq; | |
1429 | ||
59878316 SZ |
1430 | if (cpu_vendor_id == CPU_VENDOR_INTEL) |
1431 | tsc_cputimer.count = tsc_cputimer_count_lfence; | |
1432 | else | |
1433 | tsc_cputimer.count = tsc_cputimer_count_mfence; /* safe bet */ | |
1434 | ||
8d23b56c SZ |
1435 | cputimer_register(&tsc_cputimer); |
1436 | cputimer_select(&tsc_cputimer, 0); | |
1437 | } | |
1997b4c2 MD |
1438 | SYSINIT(tsc_cputimer_reg, SI_BOOT2_POST_SMP, SI_ORDER_FIRST, |
1439 | tsc_cputimer_register, NULL); | |
8d23b56c | 1440 | |
c8fe38ae MD |
1441 | SYSCTL_NODE(_hw, OID_AUTO, i8254, CTLFLAG_RW, 0, "I8254"); |
1442 | SYSCTL_UINT(_hw_i8254, OID_AUTO, freq, CTLFLAG_RD, &i8254_cputimer.freq, 0, | |
1443 | "frequency"); | |
1444 | SYSCTL_PROC(_hw_i8254, OID_AUTO, timestamp, CTLTYPE_STRING|CTLFLAG_RD, | |
1445 | 0, 0, hw_i8254_timestamp, "A", ""); | |
1446 | ||
1447 | SYSCTL_INT(_hw, OID_AUTO, tsc_present, CTLFLAG_RD, | |
1448 | &tsc_present, 0, "TSC Available"); | |
5a81b19f SZ |
1449 | SYSCTL_INT(_hw, OID_AUTO, tsc_invariant, CTLFLAG_RD, |
1450 | &tsc_invariant, 0, "Invariant TSC"); | |
dda44f1e SZ |
1451 | SYSCTL_INT(_hw, OID_AUTO, tsc_mpsync, CTLFLAG_RD, |
1452 | &tsc_mpsync, 0, "TSC is synchronized across CPUs"); | |
c8fe38ae MD |
1453 | SYSCTL_QUAD(_hw, OID_AUTO, tsc_frequency, CTLFLAG_RD, |
1454 | &tsc_frequency, 0, "TSC Frequency"); |