i386: Remove more old IOAPIC code
[dragonfly.git] / sys / platform / pc32 / apic / mpapic.c
CommitLineData
984263bc
MD
1/*
2 * Copyright (c) 1996, by Steve Passe
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. The name of the developer may NOT be used to endorse or promote products
11 * derived from this software without specific prior written permission.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 * $FreeBSD: src/sys/i386/i386/mpapic.c,v 1.37.2.7 2003/01/25 02:31:47 peter Exp $
26 */
27
28#include <sys/param.h>
29#include <sys/systm.h>
b12a1521 30#include <sys/kernel.h>
23b08e03 31#include <sys/bus.h>
e0918665 32#include <sys/machintr.h>
72740893 33#include <machine/globaldata.h>
984263bc 34#include <machine/smp.h>
90e8a35b 35#include <machine/cputypes.h>
d595a6c0 36#include <machine/md_var.h>
ad52b37b 37#include <machine/pmap.h>
a9295349 38#include <machine_base/apic/mpapic.h>
929c940f 39#include <machine_base/apic/ioapic_abi.h>
984263bc 40#include <machine/segments.h>
96728c05 41#include <sys/thread2.h>
984263bc 42
87cf6827 43#include <machine/intr_machdep.h>
984263bc 44
68b90e82
SZ
45#define IOAPIC_COUNT_MAX 16
46#define IOAPIC_ID_MASK (IOAPIC_COUNT_MAX - 1)
47
ad52b37b
SZ
48/* XXX */
49extern pt_entry_t *SMPpt;
50
984263bc
MD
51/* EISA Edge/Level trigger control registers */
52#define ELCR0 0x4d0 /* eisa irq 0-7 */
53#define ELCR1 0x4d1 /* eisa irq 8-15 */
54
ebf4f417
SZ
55struct ioapic_info {
56 int io_idx;
57 int io_apic_id;
58 void *io_addr;
59 int io_npin;
60 int io_gsi_base;
61
62 TAILQ_ENTRY(ioapic_info) io_link;
63};
64TAILQ_HEAD(ioapic_info_list, ioapic_info);
65
ae80be10
SZ
66struct ioapic_intsrc {
67 int int_gsi;
68 enum intr_trigger int_trig;
69 enum intr_polarity int_pola;
70};
71
ebf4f417
SZ
72struct ioapic_conf {
73 struct ioapic_info_list ioc_list;
ae80be10 74 struct ioapic_intsrc ioc_intsrc[16]; /* XXX magic number */
ebf4f417
SZ
75};
76
b52c8db0 77static void lapic_timer_calibrate(void);
086575e9 78static void lapic_timer_set_divisor(int);
a9e511df 79static void lapic_timer_fixup_handler(void *);
76c58571 80static void lapic_timer_restart_handler(void *);
c5b8324c 81
78ea5a2a
SZ
82void lapic_timer_process(void);
83void lapic_timer_process_frame(struct intrframe *);
c5b8324c 84
ef612539 85static int lapic_timer_enable = 1;
c5b8324c 86TUNABLE_INT("hw.lapic_timer_enable", &lapic_timer_enable);
b52c8db0 87
ef612539
SZ
88static void lapic_timer_intr_reload(struct cputimer_intr *, sysclock_t);
89static void lapic_timer_intr_enable(struct cputimer_intr *);
90static void lapic_timer_intr_restart(struct cputimer_intr *);
91static void lapic_timer_intr_pmfixup(struct cputimer_intr *);
92
68b90e82
SZ
93static int lapic_unused_apic_id(int);
94
23b08e03 95static void ioapic_setup(const struct ioapic_info *);
68b90e82 96static int ioapic_alloc_apic_id(int);
23b08e03
SZ
97static void ioapic_set_apic_id(const struct ioapic_info *);
98static void ioapic_gsi_setup(int);
99static const struct ioapic_info *
100 ioapic_gsi_search(int);
ecec8ddc
SZ
101static void ioapic_pin_prog(void *, int, int,
102 enum intr_trigger, enum intr_polarity, uint32_t);
23b08e03 103
ef612539
SZ
104static struct cputimer_intr lapic_cputimer_intr = {
105 .freq = 0,
106 .reload = lapic_timer_intr_reload,
107 .enable = lapic_timer_intr_enable,
108 .config = cputimer_intr_default_config,
109 .restart = lapic_timer_intr_restart,
110 .pmfixup = lapic_timer_intr_pmfixup,
111 .initclock = cputimer_intr_default_initclock,
112 .next = SLIST_ENTRY_INITIALIZER,
113 .name = "lapic",
114 .type = CPUTIMER_INTR_LAPIC,
115 .prio = CPUTIMER_INTR_PRIO_LAPIC,
116 .caps = CPUTIMER_INTR_CAP_NONE
117};
118
086575e9
SZ
119static int lapic_timer_divisor_idx = -1;
120static const uint32_t lapic_timer_divisors[] = {
121 APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16,
122 APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128, APIC_TDCR_1
123};
c157ff7a 124#define APIC_TIMER_NDIVISORS (int)(NELEM(lapic_timer_divisors))
086575e9 125
c5c405ff 126int lapic_id_max;
086575e9 127
ebf4f417
SZ
128static struct ioapic_conf ioapic_conf;
129
984263bc 130/*
d99d4acb 131 * Enable LAPIC, configure interrupts.
984263bc
MD
132 */
133void
5ddeabb9 134lapic_init(boolean_t bsp)
984263bc 135{
78ea5a2a 136 uint32_t timer;
984263bc
MD
137 u_int temp;
138
9d6bf2df 139 /*
dbfb3a5a
SZ
140 * Install vectors
141 *
142 * Since IDT is shared between BSP and APs, these vectors
143 * only need to be installed once; we do it on BSP.
144 */
145 if (bsp) {
146 /* Install a 'Spurious INTerrupt' vector */
147 setidt(XSPURIOUSINT_OFFSET, Xspuriousint,
148 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
149
150 /* Install an inter-CPU IPI for TLB invalidation */
151 setidt(XINVLTLB_OFFSET, Xinvltlb,
152 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
153
154 /* Install an inter-CPU IPI for IPIQ messaging */
155 setidt(XIPIQ_OFFSET, Xipiq,
156 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
157
158 /* Install a timer vector */
159 setidt(XTIMER_OFFSET, Xtimer,
160 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
161
162 /* Install an inter-CPU IPI for CPU stop/restart */
163 setidt(XCPUSTOP_OFFSET, Xcpustop,
164 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
165 }
166
167 /*
d99d4acb 168 * Setup LINT0 as ExtINT on the BSP. This is theoretically an
97359a5b
MD
169 * aggregate interrupt input from the 8259. The INTA cycle
170 * will be routed to the external controller (the 8259) which
171 * is expected to supply the vector.
172 *
173 * Must be setup edge triggered, active high.
174 *
4d08e038
SZ
175 * Disable LINT0 on BSP, if I/O APIC is enabled.
176 *
d99d4acb 177 * Disable LINT0 on the APs. It doesn't matter what delivery
97359a5b 178 * mode we use because we leave it masked.
9d6bf2df 179 */
984263bc 180 temp = lapic.lvt_lint0;
9d6bf2df
MD
181 temp &= ~(APIC_LVT_MASKED | APIC_LVT_TRIG_MASK |
182 APIC_LVT_POLARITY_MASK | APIC_LVT_DM_MASK);
4d08e038 183 if (bsp) {
9d6bf2df 184 temp |= APIC_LVT_DM_EXTINT;
4d08e038
SZ
185 if (apic_io_enable)
186 temp |= APIC_LVT_MASKED;
187 } else {
97359a5b 188 temp |= APIC_LVT_DM_FIXED | APIC_LVT_MASKED;
4d08e038 189 }
984263bc
MD
190 lapic.lvt_lint0 = temp;
191
9d6bf2df 192 /*
4d08e038
SZ
193 * Setup LINT1 as NMI.
194 *
195 * Must be setup edge trigger, active high.
196 *
197 * Enable LINT1 on BSP, if I/O APIC is enabled.
198 *
199 * Disable LINT1 on the APs.
9d6bf2df 200 */
984263bc 201 temp = lapic.lvt_lint1;
9d6bf2df
MD
202 temp &= ~(APIC_LVT_MASKED | APIC_LVT_TRIG_MASK |
203 APIC_LVT_POLARITY_MASK | APIC_LVT_DM_MASK);
204 temp |= APIC_LVT_MASKED | APIC_LVT_DM_NMI;
4d08e038
SZ
205 if (bsp && apic_io_enable)
206 temp &= ~APIC_LVT_MASKED;
984263bc
MD
207 lapic.lvt_lint1 = temp;
208
d9eea1a5 209 /*
d99d4acb 210 * Mask the LAPIC error interrupt, LAPIC performance counter
78ea5a2a 211 * interrupt.
c6a1aabe
MD
212 */
213 lapic.lvt_error = lapic.lvt_error | APIC_LVT_MASKED;
214 lapic.lvt_pcint = lapic.lvt_pcint | APIC_LVT_MASKED;
78ea5a2a 215
d99d4acb
SZ
216 /*
217 * Set LAPIC timer vector and mask the LAPIC timer interrupt.
218 */
78ea5a2a
SZ
219 timer = lapic.lvt_timer;
220 timer &= ~APIC_LVTT_VECTOR;
221 timer |= XTIMER_OFFSET;
222 timer |= APIC_LVTT_MASKED;
223 lapic.lvt_timer = timer;
c6a1aabe
MD
224
225 /*
d9eea1a5
MD
226 * Set the Task Priority Register as needed. At the moment allow
227 * interrupts on all cpus (the APs will remain CLId until they are
228 * ready to deal). We could disable all but IPIs by setting
84bf7d5a 229 * temp |= TPR_IPI for cpu != 0.
d9eea1a5 230 */
984263bc
MD
231 temp = lapic.tpr;
232 temp &= ~APIC_TPR_PRIO; /* clear priority field */
30c5f287
MN
233#ifdef SMP /* APIC-IO */
234if (!apic_io_enable) {
235#endif
97359a5b
MD
236 /*
237 * If we are NOT running the IO APICs, the LAPIC will only be used
238 * for IPIs. Set the TPR to prevent any unintentional interrupts.
239 */
84bf7d5a 240 temp |= TPR_IPI;
30c5f287
MN
241#ifdef SMP /* APIC-IO */
242}
97359a5b 243#endif
8a8d5d85 244
984263bc
MD
245 lapic.tpr = temp;
246
97359a5b 247 /*
d99d4acb 248 * Enable the LAPIC
97359a5b 249 */
984263bc 250 temp = lapic.svr;
d99d4acb 251 temp |= APIC_SVR_ENABLE; /* enable the LAPIC */
97359a5b 252 temp &= ~APIC_SVR_FOCUS_DISABLE; /* enable lopri focus processor */
984263bc 253
9d6bf2df
MD
254 /*
255 * Set the spurious interrupt vector. The low 4 bits of the vector
256 * must be 1111.
257 */
258 if ((XSPURIOUSINT_OFFSET & 0x0F) != 0x0F)
984263bc 259 panic("bad XSPURIOUSINT_OFFSET: 0x%08x", XSPURIOUSINT_OFFSET);
9d6bf2df
MD
260 temp &= ~APIC_SVR_VECTOR;
261 temp |= XSPURIOUSINT_OFFSET;
984263bc 262
984263bc
MD
263 lapic.svr = temp;
264
0b692e79
MD
265 /*
266 * Pump out a few EOIs to clean out interrupts that got through
267 * before we were able to set the TPR.
268 */
269 lapic.eoi = 0;
270 lapic.eoi = 0;
271 lapic.eoi = 0;
272
c5b8324c 273 if (bsp) {
b52c8db0 274 lapic_timer_calibrate();
ef612539
SZ
275 if (lapic_timer_enable) {
276 cputimer_intr_register(&lapic_cputimer_intr);
277 cputimer_intr_select(&lapic_cputimer_intr, 0);
278 }
c5b8324c 279 } else {
086575e9 280 lapic_timer_set_divisor(lapic_timer_divisor_idx);
c5b8324c 281 }
b52c8db0 282
984263bc
MD
283 if (bootverbose)
284 apic_dump("apic_initialize()");
285}
286
b52c8db0
SZ
287static void
288lapic_timer_set_divisor(int divisor_idx)
289{
290 KKASSERT(divisor_idx >= 0 && divisor_idx < APIC_TIMER_NDIVISORS);
291 lapic.dcr_timer = lapic_timer_divisors[divisor_idx];
292}
293
294static void
295lapic_timer_oneshot(u_int count)
296{
297 uint32_t value;
298
299 value = lapic.lvt_timer;
300 value &= ~APIC_LVTT_PERIODIC;
301 lapic.lvt_timer = value;
302 lapic.icr_timer = count;
303}
304
305static void
6198c499
SZ
306lapic_timer_oneshot_quick(u_int count)
307{
308 lapic.icr_timer = count;
309}
310
311static void
b52c8db0
SZ
312lapic_timer_calibrate(void)
313{
47bdf646 314 sysclock_t value;
b52c8db0
SZ
315
316 /* Try to calibrate the local APIC timer. */
317 for (lapic_timer_divisor_idx = 0;
318 lapic_timer_divisor_idx < APIC_TIMER_NDIVISORS;
319 lapic_timer_divisor_idx++) {
320 lapic_timer_set_divisor(lapic_timer_divisor_idx);
321 lapic_timer_oneshot(APIC_TIMER_MAX_COUNT);
322 DELAY(2000000);
323 value = APIC_TIMER_MAX_COUNT - lapic.ccr_timer;
324 if (value != APIC_TIMER_MAX_COUNT)
325 break;
326 }
327 if (lapic_timer_divisor_idx >= APIC_TIMER_NDIVISORS)
328 panic("lapic: no proper timer divisor?!\n");
ef612539 329 lapic_cputimer_intr.freq = value / 2;
b52c8db0 330
47bdf646 331 kprintf("lapic: divisor index %d, frequency %u Hz\n",
ef612539 332 lapic_timer_divisor_idx, lapic_cputimer_intr.freq);
b52c8db0
SZ
333}
334
c5b8324c
SZ
335static void
336lapic_timer_process_oncpu(struct globaldata *gd, struct intrframe *frame)
337{
338 sysclock_t count;
339
340 gd->gd_timer_running = 0;
341
342 count = sys_cputimer->count();
343 if (TAILQ_FIRST(&gd->gd_systimerq) != NULL)
344 systimer_intr(&count, 0, frame);
345}
346
78ea5a2a
SZ
347void
348lapic_timer_process(void)
349{
ae48d6cd 350 lapic_timer_process_oncpu(mycpu, NULL);
78ea5a2a
SZ
351}
352
353void
354lapic_timer_process_frame(struct intrframe *frame)
355{
ae48d6cd 356 lapic_timer_process_oncpu(mycpu, frame);
b12a1521
SZ
357}
358
c5b8324c 359static void
ef612539 360lapic_timer_intr_reload(struct cputimer_intr *cti, sysclock_t reload)
c5b8324c
SZ
361{
362 struct globaldata *gd = mycpu;
363
ef612539 364 reload = (int64_t)reload * cti->freq / sys_cputimer->freq;
c5b8324c
SZ
365 if (reload < 2)
366 reload = 2;
367
368 if (gd->gd_timer_running) {
369 if (reload < lapic.ccr_timer)
370 lapic_timer_oneshot_quick(reload);
371 } else {
372 gd->gd_timer_running = 1;
373 lapic_timer_oneshot_quick(reload);
374 }
375}
376
ef612539
SZ
377static void
378lapic_timer_intr_enable(struct cputimer_intr *cti __unused)
6198c499
SZ
379{
380 uint32_t timer;
381
382 timer = lapic.lvt_timer;
383 timer &= ~(APIC_LVTT_MASKED | APIC_LVTT_PERIODIC);
384 lapic.lvt_timer = timer;
a9e511df
SZ
385
386 lapic_timer_fixup_handler(NULL);
387}
388
389static void
76c58571 390lapic_timer_fixup_handler(void *arg)
a9e511df 391{
76c58571
SZ
392 int *started = arg;
393
394 if (started != NULL)
395 *started = 0;
396
90e8a35b 397 if (cpu_vendor_id == CPU_VENDOR_AMD) {
a9e511df
SZ
398 /*
399 * Detect the presence of C1E capability mostly on latest
400 * dual-cores (or future) k8 family. This feature renders
401 * the local APIC timer dead, so we disable it by reading
402 * the Interrupt Pending Message register and clearing both
403 * C1eOnCmpHalt (bit 28) and SmiOnCmpHalt (bit 27).
404 *
405 * Reference:
406 * "BIOS and Kernel Developer's Guide for AMD NPT
407 * Family 0Fh Processors"
408 * #32559 revision 3.00
409 */
410 if ((cpu_id & 0x00000f00) == 0x00000f00 &&
411 (cpu_id & 0x0fff0000) >= 0x00040000) {
412 uint64_t msr;
413
414 msr = rdmsr(0xc0010055);
415 if (msr & 0x18000000) {
416 struct globaldata *gd = mycpu;
417
418 kprintf("cpu%d: AMD C1E detected\n",
419 gd->gd_cpuid);
420 wrmsr(0xc0010055, msr & ~0x18000000ULL);
421
422 /*
423 * We are kinda stalled;
424 * kick start again.
425 */
426 gd->gd_timer_running = 1;
427 lapic_timer_oneshot_quick(2);
76c58571
SZ
428
429 if (started != NULL)
430 *started = 1;
a9e511df
SZ
431 }
432 }
433 }
434}
435
76c58571
SZ
436static void
437lapic_timer_restart_handler(void *dummy __unused)
438{
439 int started;
440
441 lapic_timer_fixup_handler(&started);
442 if (!started) {
443 struct globaldata *gd = mycpu;
444
445 gd->gd_timer_running = 1;
446 lapic_timer_oneshot_quick(2);
447 }
448}
449
a9e511df
SZ
450/*
451 * This function is called only by ACPI-CA code currently:
452 * - AMD C1E fixup. AMD C1E only seems to happen after ACPI
453 * module controls PM. So once ACPI-CA is attached, we try
454 * to apply the fixup to prevent LAPIC timer from hanging.
455 */
ef612539
SZ
456static void
457lapic_timer_intr_pmfixup(struct cputimer_intr *cti __unused)
a9e511df 458{
ef612539
SZ
459 lwkt_send_ipiq_mask(smp_active_mask,
460 lapic_timer_fixup_handler, NULL);
6198c499
SZ
461}
462
ef612539
SZ
463static void
464lapic_timer_intr_restart(struct cputimer_intr *cti __unused)
76c58571 465{
76c58571
SZ
466 lwkt_send_ipiq_mask(smp_active_mask, lapic_timer_restart_handler, NULL);
467}
468
b52c8db0 469
984263bc
MD
470/*
471 * dump contents of local APIC registers
472 */
473void
474apic_dump(char* str)
475{
26be20a0
SW
476 kprintf("SMP: CPU%d %s:\n", mycpu->gd_cpuid, str);
477 kprintf(" lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n",
984263bc
MD
478 lapic.lvt_lint0, lapic.lvt_lint1, lapic.tpr, lapic.svr);
479}
480
984263bc
MD
481/*
482 * Inter Processor Interrupt functions.
483 */
484
984263bc
MD
485/*
486 * Send APIC IPI 'vector' to 'destType' via 'deliveryMode'.
487 *
488 * destType is 1 of: APIC_DEST_SELF, APIC_DEST_ALLISELF, APIC_DEST_ALLESELF
489 * vector is any valid SYSTEM INT vector
490 * delivery_mode is 1 of: APIC_DELMODE_FIXED, APIC_DELMODE_LOWPRIO
96728c05
MD
491 *
492 * A backlog of requests can create a deadlock between cpus. To avoid this
493 * we have to be able to accept IPIs at the same time we are trying to send
494 * them. The critical section prevents us from attempting to send additional
495 * IPIs reentrantly, but also prevents IPIQ processing so we have to call
496 * lwkt_process_ipiq() manually. It's rather messy and expensive for this
497 * to occur but fortunately it does not happen too often.
984263bc 498 */
984263bc
MD
499int
500apic_ipi(int dest_type, int vector, int delivery_mode)
501{
502 u_long icr_lo;
503
96728c05
MD
504 crit_enter();
505 if ((lapic.icr_lo & APIC_DELSTAT_MASK) != 0) {
506 unsigned int eflags = read_eflags();
507 cpu_enable_intr();
cfaeae2a 508 DEBUG_PUSH_INFO("apic_ipi");
96728c05
MD
509 while ((lapic.icr_lo & APIC_DELSTAT_MASK) != 0) {
510 lwkt_process_ipiq();
511 }
cfaeae2a 512 DEBUG_POP_INFO();
96728c05 513 write_eflags(eflags);
984263bc 514 }
984263bc 515
9d6bf2df 516 icr_lo = (lapic.icr_lo & APIC_ICRLO_RESV_MASK) | dest_type |
96728c05 517 delivery_mode | vector;
984263bc 518 lapic.icr_lo = icr_lo;
96728c05 519 crit_exit();
984263bc
MD
520 return 0;
521}
522
41a01a4d
MD
523void
524single_apic_ipi(int cpu, int vector, int delivery_mode)
984263bc
MD
525{
526 u_long icr_lo;
527 u_long icr_hi;
984263bc 528
41a01a4d 529 crit_enter();
96728c05
MD
530 if ((lapic.icr_lo & APIC_DELSTAT_MASK) != 0) {
531 unsigned int eflags = read_eflags();
532 cpu_enable_intr();
cfaeae2a 533 DEBUG_PUSH_INFO("single_apic_ipi");
96728c05
MD
534 while ((lapic.icr_lo & APIC_DELSTAT_MASK) != 0) {
535 lwkt_process_ipiq();
536 }
cfaeae2a 537 DEBUG_POP_INFO();
96728c05 538 write_eflags(eflags);
984263bc 539 }
984263bc
MD
540 icr_hi = lapic.icr_hi & ~APIC_ID_MASK;
541 icr_hi |= (CPU_TO_ID(cpu) << 24);
542 lapic.icr_hi = icr_hi;
543
b2f93ae9 544 /* build ICR_LOW */
9d6bf2df 545 icr_lo = (lapic.icr_lo & APIC_ICRLO_RESV_MASK)
984263bc
MD
546 | APIC_DEST_DESTFLD | delivery_mode | vector;
547
548 /* write APIC ICR */
549 lapic.icr_lo = icr_lo;
41a01a4d 550 crit_exit();
984263bc
MD
551}
552
41a01a4d
MD
553#if 0
554
555/*
556 * Returns 0 if the apic is busy, 1 if we were able to queue the request.
557 *
558 * NOT WORKING YET! The code as-is may end up not queueing an IPI at all
559 * to the target, and the scheduler does not 'poll' for IPI messages.
560 */
561int
562single_apic_ipi_passive(int cpu, int vector, int delivery_mode)
563{
564 u_long icr_lo;
565 u_long icr_hi;
566
567 crit_enter();
568 if ((lapic.icr_lo & APIC_DELSTAT_MASK) != 0) {
569 crit_exit();
570 return(0);
571 }
572 icr_hi = lapic.icr_hi & ~APIC_ID_MASK;
573 icr_hi |= (CPU_TO_ID(cpu) << 24);
574 lapic.icr_hi = icr_hi;
575
576 /* build IRC_LOW */
577 icr_lo = (lapic.icr_lo & APIC_RESV2_MASK)
578 | APIC_DEST_DESTFLD | delivery_mode | vector;
579
580 /* write APIC ICR */
581 lapic.icr_lo = icr_lo;
582 crit_exit();
583 return(1);
584}
585
586#endif
587
984263bc
MD
588/*
589 * Send APIC IPI 'vector' to 'target's via 'delivery_mode'.
590 *
96728c05
MD
591 * target is a bitmask of destination cpus. Vector is any
592 * valid system INT vector. Delivery mode may be either
593 * APIC_DELMODE_FIXED or APIC_DELMODE_LOWPRIO.
984263bc 594 */
41a01a4d 595void
da23a592 596selected_apic_ipi(cpumask_t target, int vector, int delivery_mode)
984263bc 597{
96728c05
MD
598 crit_enter();
599 while (target) {
da23a592
MD
600 int n = BSFCPUMASK(target);
601 target &= ~CPUMASK(n);
41a01a4d 602 single_apic_ipi(n, vector, delivery_mode);
96728c05
MD
603 }
604 crit_exit();
984263bc 605}
984263bc 606
984263bc
MD
607/*
608 * Timer code, in development...
609 * - suggested by rgrimes@gndrsh.aac.dev.com
610 */
bb467734
MD
611int
612get_apic_timer_frequency(void)
613{
614 return(lapic_cputimer_intr.freq);
615}
984263bc 616
984263bc
MD
617/*
618 * Load a 'downcount time' in uSeconds.
619 */
620void
2942ed63 621set_apic_timer(int us)
984263bc 622{
2942ed63 623 u_int count;
984263bc
MD
624
625 /*
2942ed63
SZ
626 * When we reach here, lapic timer's frequency
627 * must have been calculated as well as the
628 * divisor (lapic.dcr_timer is setup during the
629 * divisor calculation).
984263bc 630 */
ef612539 631 KKASSERT(lapic_cputimer_intr.freq != 0 &&
2942ed63
SZ
632 lapic_timer_divisor_idx >= 0);
633
ef612539 634 count = ((us * (int64_t)lapic_cputimer_intr.freq) + 999999) / 1000000;
2942ed63 635 lapic_timer_oneshot(count);
984263bc
MD
636}
637
638
639/*
640 * Read remaining time in timer.
641 */
642int
643read_apic_timer(void)
644{
645#if 0
646 /** XXX FIXME: we need to return the actual remaining time,
647 * for now we just return the remaining count.
648 */
649#else
650 return lapic.ccr_timer;
651#endif
652}
653
654
655/*
656 * Spin-style delay, set delay time in uS, spin till it drains.
657 */
658void
659u_sleep(int count)
660{
661 set_apic_timer(count);
662 while (read_apic_timer())
663 /* spin */ ;
664}
ad52b37b 665
68b90e82
SZ
666static int
667lapic_unused_apic_id(int start)
668{
669 int i;
670
671 for (i = start; i < NAPICID; ++i) {
672 if (ID_TO_CPU(i) == -1)
673 return i;
674 }
675 return NAPICID;
676}
677
ad52b37b 678void
84cc808b 679lapic_map(vm_offset_t lapic_addr)
ad52b37b
SZ
680{
681 /* Local apic is mapped on last page */
682 SMPpt[NPTEPG - 1] = (pt_entry_t)(PG_V | PG_RW | PG_N |
683 pmap_get_pgeflag() | (lapic_addr & PG_FRAME));
684
d557216f 685 kprintf("lapic: at %p\n", (void *)lapic_addr);
ad52b37b 686}
281d9482
SZ
687
688static TAILQ_HEAD(, lapic_enumerator) lapic_enumerators =
689 TAILQ_HEAD_INITIALIZER(lapic_enumerators);
690
691void
692lapic_config(void)
693{
694 struct lapic_enumerator *e;
68b90e82
SZ
695 int error, i;
696
697 for (i = 0; i < NAPICID; ++i)
698 ID_TO_CPU(i) = -1;
281d9482
SZ
699
700 TAILQ_FOREACH(e, &lapic_enumerators, lapic_link) {
701 error = e->lapic_probe(e);
702 if (!error)
703 break;
704 }
705 if (e == NULL)
706 panic("can't config lapic\n");
707
708 e->lapic_enumerate(e);
709}
710
711void
712lapic_enumerator_register(struct lapic_enumerator *ne)
713{
714 struct lapic_enumerator *e;
715
716 TAILQ_FOREACH(e, &lapic_enumerators, lapic_link) {
717 if (e->lapic_prio < ne->lapic_prio) {
718 TAILQ_INSERT_BEFORE(e, ne, lapic_link);
719 return;
720 }
721 }
722 TAILQ_INSERT_TAIL(&lapic_enumerators, ne, lapic_link);
723}
65b2387f
SZ
724
725static TAILQ_HEAD(, ioapic_enumerator) ioapic_enumerators =
726 TAILQ_HEAD_INITIALIZER(ioapic_enumerators);
727
728void
729ioapic_config(void)
730{
6ac31e9d
SZ
731 struct ioapic_info *info;
732 int start_apic_id = 0;
65b2387f 733 struct ioapic_enumerator *e;
ebf4f417 734 int error, i;
e0918665 735 u_long ef = 0;
ebf4f417
SZ
736
737 TAILQ_INIT(&ioapic_conf.ioc_list);
738 /* XXX magic number */
739 for (i = 0; i < 16; ++i)
ae80be10 740 ioapic_conf.ioc_intsrc[i].int_gsi = -1;
65b2387f
SZ
741
742 TAILQ_FOREACH(e, &ioapic_enumerators, ioapic_link) {
743 error = e->ioapic_probe(e);
744 if (!error)
745 break;
746 }
747 if (e == NULL) {
748#ifdef notyet
749 panic("can't config I/O APIC\n");
750#else
751 kprintf("no I/O APIC\n");
752 return;
753#endif
754 }
755
6ac31e9d 756 crit_enter();
e0918665 757
6ac31e9d
SZ
758 ef = read_eflags();
759 cpu_disable_intr();
e0918665 760
6ac31e9d
SZ
761 /*
762 * Switch to I/O APIC MachIntrABI and reconfigure
763 * the default IDT entries.
764 */
765 MachIntrABI = MachIntrABI_IOAPIC;
766 MachIntrABI.setdefault();
e0918665 767
65b2387f 768 e->ioapic_enumerate(e);
0471bb0e 769
6ac31e9d
SZ
770 /*
771 * Setup index
772 */
773 i = 0;
774 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link)
775 info->io_idx = i++;
68b90e82 776
6ac31e9d
SZ
777 if (i > IOAPIC_COUNT_MAX) /* XXX magic number */
778 panic("ioapic_config: more than 16 I/O APIC\n");
68b90e82 779
6ac31e9d
SZ
780 /*
781 * Setup APIC ID
782 */
783 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) {
784 int apic_id;
68b90e82 785
6ac31e9d
SZ
786 apic_id = ioapic_alloc_apic_id(start_apic_id);
787 if (apic_id == NAPICID) {
788 kprintf("IOAPIC: can't alloc APIC ID for "
789 "%dth I/O APIC\n", info->io_idx);
790 break;
68b90e82 791 }
6ac31e9d 792 info->io_apic_id = apic_id;
68b90e82 793
6ac31e9d
SZ
794 start_apic_id = apic_id + 1;
795 }
796 if (info != NULL) {
68b90e82 797 /*
6ac31e9d
SZ
798 * xAPIC allows I/O APIC's APIC ID to be same
799 * as the LAPIC's APIC ID
68b90e82 800 */
6ac31e9d
SZ
801 kprintf("IOAPIC: use xAPIC model to alloc APIC ID "
802 "for I/O APIC\n");
803
804 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link)
805 info->io_apic_id = info->io_idx;
806 }
807
808 /*
809 * Warning about any GSI holes
810 */
811 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) {
812 const struct ioapic_info *prev_info;
813
814 prev_info = TAILQ_PREV(info, ioapic_info_list, io_link);
815 if (prev_info != NULL) {
816 if (info->io_gsi_base !=
817 prev_info->io_gsi_base + prev_info->io_npin) {
818 kprintf("IOAPIC: warning gsi hole "
819 "[%d, %d]\n",
820 prev_info->io_gsi_base +
821 prev_info->io_npin,
822 info->io_gsi_base - 1);
ebf4f417
SZ
823 }
824 }
6ac31e9d 825 }
23b08e03 826
6ac31e9d
SZ
827 if (bootverbose) {
828 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) {
829 kprintf("IOAPIC: idx %d, apic id %d, "
830 "gsi base %d, npin %d\n",
831 info->io_idx,
832 info->io_apic_id,
833 info->io_gsi_base,
834 info->io_npin);
68b90e82 835 }
6ac31e9d 836 }
68b90e82 837
6ac31e9d
SZ
838 /*
839 * Setup all I/O APIC
840 */
841 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link)
842 ioapic_setup(info);
843 ioapic_abi_fixup_irqmap();
23b08e03 844
6ac31e9d 845 write_eflags(ef);
e0918665 846
6ac31e9d 847 MachIntrABI.cleanup();
e0918665 848
6ac31e9d 849 crit_exit();
65b2387f
SZ
850}
851
852void
853ioapic_enumerator_register(struct ioapic_enumerator *ne)
854{
855 struct ioapic_enumerator *e;
856
857 TAILQ_FOREACH(e, &ioapic_enumerators, ioapic_link) {
858 if (e->ioapic_prio < ne->ioapic_prio) {
859 TAILQ_INSERT_BEFORE(e, ne, ioapic_link);
860 return;
861 }
862 }
863 TAILQ_INSERT_TAIL(&ioapic_enumerators, ne, ioapic_link);
864}
ebf4f417
SZ
865
866void
867ioapic_add(void *addr, int gsi_base, int npin)
868{
869 struct ioapic_info *info, *ninfo;
870 int gsi_end;
871
872 gsi_end = gsi_base + npin - 1;
873 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) {
874 if ((gsi_base >= info->io_gsi_base &&
875 gsi_base < info->io_gsi_base + info->io_npin) ||
876 (gsi_end >= info->io_gsi_base &&
877 gsi_end < info->io_gsi_base + info->io_npin)) {
878 panic("ioapic_add: overlapped gsi, base %d npin %d, "
879 "hit base %d, npin %d\n", gsi_base, npin,
880 info->io_gsi_base, info->io_npin);
881 }
882 if (info->io_addr == addr)
883 panic("ioapic_add: duplicated addr %p\n", addr);
884 }
885
886 ninfo = kmalloc(sizeof(*ninfo), M_DEVBUF, M_WAITOK | M_ZERO);
887 ninfo->io_addr = addr;
888 ninfo->io_npin = npin;
889 ninfo->io_gsi_base = gsi_base;
68b90e82 890 ninfo->io_apic_id = -1;
ebf4f417
SZ
891
892 /*
893 * Create IOAPIC list in ascending order of GSI base
894 */
895 TAILQ_FOREACH_REVERSE(info, &ioapic_conf.ioc_list,
896 ioapic_info_list, io_link) {
897 if (ninfo->io_gsi_base > info->io_gsi_base) {
898 TAILQ_INSERT_AFTER(&ioapic_conf.ioc_list,
899 info, ninfo, io_link);
900 break;
901 }
902 }
903 if (info == NULL)
904 TAILQ_INSERT_HEAD(&ioapic_conf.ioc_list, ninfo, io_link);
905}
512fb675
SZ
906
907void
ae80be10 908ioapic_intsrc(int irq, int gsi, enum intr_trigger trig, enum intr_polarity pola)
512fb675 909{
ae80be10
SZ
910 struct ioapic_intsrc *int_src;
911
512fb675 912 KKASSERT(irq < 16);
ae80be10 913 int_src = &ioapic_conf.ioc_intsrc[irq];
7eb6fb7e
SZ
914
915 if (gsi == 0) {
916 /* Don't allow mixed mode */
917 kprintf("IOAPIC: warning intsrc irq %d -> gsi 0\n", irq);
918 return;
919 }
920
ae80be10
SZ
921 if (int_src->int_gsi != -1) {
922 if (int_src->int_gsi != gsi) {
923 kprintf("IOAPIC: warning intsrc irq %d, gsi "
924 "%d -> %d\n", irq, int_src->int_gsi, gsi);
925 }
926 if (int_src->int_trig != trig) {
927 kprintf("IOAPIC: warning intsrc irq %d, trig "
4ecd5d4d
SZ
928 "%s -> %s\n", irq,
929 intr_str_trigger(int_src->int_trig),
930 intr_str_trigger(trig));
ae80be10
SZ
931 }
932 if (int_src->int_pola != pola) {
933 kprintf("IOAPIC: warning intsrc irq %d, pola "
934 "%s -> %s\n", irq,
4ecd5d4d
SZ
935 intr_str_polarity(int_src->int_pola),
936 intr_str_polarity(pola));
ae80be10 937 }
512fb675 938 }
ae80be10
SZ
939 int_src->int_gsi = gsi;
940 int_src->int_trig = trig;
941 int_src->int_pola = pola;
512fb675 942}
23b08e03
SZ
943
944static void
945ioapic_set_apic_id(const struct ioapic_info *info)
946{
947 uint32_t id;
68b90e82 948 int apic_id;
23b08e03
SZ
949
950 id = ioapic_read(info->io_addr, IOAPIC_ID);
951
952 id &= ~APIC_ID_MASK;
953 id |= (info->io_apic_id << 24);
954
955 ioapic_write(info->io_addr, IOAPIC_ID, id);
956
957 /*
958 * Re-read && test
959 */
960 id = ioapic_read(info->io_addr, IOAPIC_ID);
68b90e82
SZ
961 apic_id = (id & APIC_ID_MASK) >> 24;
962
963 /*
964 * I/O APIC ID is a 4bits field
965 */
966 if ((apic_id & IOAPIC_ID_MASK) !=
967 (info->io_apic_id & IOAPIC_ID_MASK)) {
968 panic("ioapic_set_apic_id: can't set apic id to %d, "
969 "currently set to %d\n", info->io_apic_id, apic_id);
23b08e03
SZ
970 }
971}
972
973static void
974ioapic_gsi_setup(int gsi)
975{
976 enum intr_trigger trig;
977 enum intr_polarity pola;
978 int irq;
979
ecec8ddc
SZ
980 if (gsi == 0) {
981 /* ExtINT */
7bceaa10 982 imen_lock();
ecec8ddc
SZ
983 ioapic_extpin_setup(ioapic_gsi_ioaddr(gsi),
984 ioapic_gsi_pin(gsi), 0);
7bceaa10 985 imen_unlock();
ecec8ddc
SZ
986 return;
987 }
988
5c26bcd6
MD
989 trig = 0; /* silence older gcc's */
990 pola = 0; /* silence older gcc's */
991
23b08e03 992 for (irq = 0; irq < 16; ++irq) {
ae80be10
SZ
993 const struct ioapic_intsrc *int_src =
994 &ioapic_conf.ioc_intsrc[irq];
995
996 if (gsi == int_src->int_gsi) {
997 trig = int_src->int_trig;
998 pola = int_src->int_pola;
23b08e03
SZ
999 break;
1000 }
1001 }
1002
1003 if (irq == 16) {
ecec8ddc 1004 if (gsi < 16) {
23b08e03
SZ
1005 trig = INTR_TRIGGER_EDGE;
1006 pola = INTR_POLARITY_HIGH;
1007 } else {
1008 trig = INTR_TRIGGER_LEVEL;
1009 pola = INTR_POLARITY_LOW;
1010 }
1011 irq = gsi;
1012 }
1013
23b08e03 1014 ioapic_abi_set_irqmap(irq, gsi, trig, pola);
23b08e03
SZ
1015}
1016
1017void *
1018ioapic_gsi_ioaddr(int gsi)
1019{
1020 const struct ioapic_info *info;
1021
1022 info = ioapic_gsi_search(gsi);
1023 return info->io_addr;
1024}
1025
1026int
1027ioapic_gsi_pin(int gsi)
1028{
1029 const struct ioapic_info *info;
1030
1031 info = ioapic_gsi_search(gsi);
1032 return gsi - info->io_gsi_base;
1033}
1034
1035static const struct ioapic_info *
1036ioapic_gsi_search(int gsi)
1037{
1038 const struct ioapic_info *info;
1039
1040 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) {
1041 if (gsi >= info->io_gsi_base &&
1042 gsi < info->io_gsi_base + info->io_npin)
1043 return info;
1044 }
1045 panic("ioapic_gsi_search: no I/O APIC\n");
1046}
1047
e90e7ac4
SZ
1048int
1049ioapic_gsi(int idx, int pin)
1050{
1051 const struct ioapic_info *info;
1052
1053 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) {
1054 if (info->io_idx == idx)
1055 break;
1056 }
1057 if (info == NULL)
1058 return -1;
1059 if (pin >= info->io_npin)
1060 return -1;
1061 return info->io_gsi_base + pin;
1062}
1063
ecec8ddc
SZ
1064void
1065ioapic_extpin_setup(void *addr, int pin, int vec)
1066{
ecec8ddc
SZ
1067 ioapic_pin_prog(addr, pin, vec,
1068 INTR_TRIGGER_CONFORM, INTR_POLARITY_CONFORM, IOART_DELEXINT);
ecec8ddc
SZ
1069}
1070
6b809ec7
SZ
1071int
1072ioapic_extpin_gsi(void)
1073{
1074 return 0;
1075}
1076
ecec8ddc
SZ
1077void
1078ioapic_pin_setup(void *addr, int pin, int vec,
1079 enum intr_trigger trig, enum intr_polarity pola)
1080{
1081 /*
1082 * Always clear an I/O APIC pin before [re]programming it. This is
1083 * particularly important if the pin is set up for a level interrupt
1084 * as the IOART_REM_IRR bit might be set. When we reprogram the
1085 * vector any EOI from pending ints on this pin could be lost and
1086 * IRR might never get reset.
1087 *
1088 * To fix this problem, clear the vector and make sure it is
1089 * programmed as an edge interrupt. This should theoretically
1090 * clear IRR so we can later, safely program it as a level
1091 * interrupt.
1092 */
ecec8ddc
SZ
1093 ioapic_pin_prog(addr, pin, vec, INTR_TRIGGER_EDGE, INTR_POLARITY_HIGH,
1094 IOART_DELFIXED);
1095 ioapic_pin_prog(addr, pin, vec, trig, pola, IOART_DELFIXED);
ecec8ddc
SZ
1096}
1097
1098static void
1099ioapic_pin_prog(void *addr, int pin, int vec,
1100 enum intr_trigger trig, enum intr_polarity pola, uint32_t del_mode)
1101{
1102 uint32_t flags, target;
1103 int select;
1104
1105 KKASSERT(del_mode == IOART_DELEXINT || del_mode == IOART_DELFIXED);
1106
1107 select = IOAPIC_REDTBL0 + (2 * pin);
1108
1109 flags = ioapic_read(addr, select) & IOART_RESV;
ddb99c82
SZ
1110 flags |= IOART_INTMSET | IOART_DESTPHY;
1111#ifdef foo
1112 flags |= del_mode;
1113#else
1114 /*
1115 * We only support limited I/O APIC mixed mode,
1116 * so even for ExtINT, we still use "fixed"
1117 * delivery mode.
1118 */
1119 flags |= IOART_DELFIXED;
1120#endif
ecec8ddc
SZ
1121
1122 if (del_mode == IOART_DELEXINT) {
1123 KKASSERT(trig == INTR_TRIGGER_CONFORM &&
1124 pola == INTR_POLARITY_CONFORM);
1125 flags |= IOART_TRGREDG | IOART_INTAHI;
1126 } else {
1127 switch (trig) {
1128 case INTR_TRIGGER_EDGE:
1129 flags |= IOART_TRGREDG;
1130 break;
1131
1132 case INTR_TRIGGER_LEVEL:
1133 flags |= IOART_TRGRLVL;
1134 break;
1135
1136 case INTR_TRIGGER_CONFORM:
1137 panic("ioapic_pin_prog: trig conform is not "
1138 "supported\n");
1139 }
1140 switch (pola) {
1141 case INTR_POLARITY_HIGH:
1142 flags |= IOART_INTAHI;
1143 break;
1144
1145 case INTR_POLARITY_LOW:
1146 flags |= IOART_INTALO;
1147 break;
1148
1149 case INTR_POLARITY_CONFORM:
1150 panic("ioapic_pin_prog: pola conform is not "
1151 "supported\n");
1152 }
1153 }
1154
1155 target = ioapic_read(addr, select + 1) & IOART_HI_DEST_RESV;
522f877c
SZ
1156 target |= (CPU_TO_ID(0) << IOART_HI_DEST_SHIFT) &
1157 IOART_HI_DEST_MASK;
ecec8ddc
SZ
1158
1159 ioapic_write(addr, select, flags | vec);
1160 ioapic_write(addr, select + 1, target);
1161}
1162
23b08e03
SZ
1163static void
1164ioapic_setup(const struct ioapic_info *info)
1165{
1166 int i;
1167
1168 ioapic_set_apic_id(info);
1169
1170 for (i = 0; i < info->io_npin; ++i)
1171 ioapic_gsi_setup(info->io_gsi_base + i);
1172}
68b90e82
SZ
1173
1174static int
1175ioapic_alloc_apic_id(int start)
1176{
1177 for (;;) {
1178 const struct ioapic_info *info;
1179 int apic_id, apic_id16;
1180
1181 apic_id = lapic_unused_apic_id(start);
1182 if (apic_id == NAPICID) {
1183 kprintf("IOAPIC: can't find unused APIC ID\n");
1184 return apic_id;
1185 }
1186 apic_id16 = apic_id & IOAPIC_ID_MASK;
1187
1188 /*
1189 * Check against other I/O APIC's APIC ID's lower 4bits.
1190 *
1191 * The new APIC ID will have to be different from others
1192 * in the lower 4bits, no matter whether xAPIC is used
1193 * or not.
1194 */
1195 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) {
1196 if (info->io_apic_id == -1) {
1197 info = NULL;
1198 break;
1199 }
1200 if ((info->io_apic_id & IOAPIC_ID_MASK) == apic_id16)
1201 break;
1202 }
1203 if (info == NULL)
1204 return apic_id;
1205
1206 kprintf("IOAPIC: APIC ID %d has same lower 4bits as "
1207 "%dth I/O APIC, keep searching...\n",
1208 apic_id, info->io_idx);
1209
1210 start = apic_id + 1;
1211 }
1212 panic("ioapic_unused_apic_id: never reached\n");
1213}