x86_64: Add MSI support for ICU MachIntrABI
[dragonfly.git] / sys / platform / pc64 / icu / icu_abi.c
CommitLineData
c8fe38ae
MD
1/*
2 * Copyright (c) 1991 The Regents of the University of California.
3 * Copyright (c) 2005,2008 The DragonFly Project.
4 * All rights reserved.
5 *
6 * This code is derived from software contributed to The DragonFly Project
7 * by Matthew Dillon <dillon@backplane.com>
8 *
9 * This code is derived from software contributed to Berkeley by
10 * William Jolitz.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 *
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in
20 * the documentation and/or other materials provided with the
21 * distribution.
22 * 3. Neither the name of The DragonFly Project nor the names of its
23 * contributors may be used to endorse or promote products derived
24 * from this software without specific, prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
29 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
30 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
31 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
32 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
33 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
34 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
35 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
36 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 * SUCH DAMAGE.
38 *
39 * $DragonFly: src/sys/platform/pc64/icu/icu_abi.c,v 1.1 2008/08/29 17:07:16 dillon Exp $
40 */
41
42#include <sys/param.h>
43#include <sys/systm.h>
44#include <sys/kernel.h>
45#include <sys/machintr.h>
46#include <sys/interrupt.h>
b3cf468e 47#include <sys/rman.h>
c8fe38ae
MD
48#include <sys/bus.h>
49
50#include <machine/segments.h>
51#include <machine/md_var.h>
57a9c56b 52#include <machine/intr_machdep.h>
c8fe38ae 53#include <machine/globaldata.h>
10db3cc6 54#include <machine/smp.h>
759fc533 55#include <machine/msi_var.h>
c8fe38ae
MD
56
57#include <sys/thread2.h>
58
7265a4fe 59#include <machine_base/icu/elcr_var.h>
9e0e3f85 60
7265a4fe
SZ
61#include <machine_base/icu/icu.h>
62#include <machine_base/icu/icu_ipl.h>
ed4d621d 63#include <machine_base/apic/ioapic.h>
c8fe38ae 64
c8fe38ae 65extern inthand_t
35e45e47
SZ
66 IDTVEC(icu_intr0), IDTVEC(icu_intr1),
67 IDTVEC(icu_intr2), IDTVEC(icu_intr3),
68 IDTVEC(icu_intr4), IDTVEC(icu_intr5),
69 IDTVEC(icu_intr6), IDTVEC(icu_intr7),
70 IDTVEC(icu_intr8), IDTVEC(icu_intr9),
71 IDTVEC(icu_intr10), IDTVEC(icu_intr11),
72 IDTVEC(icu_intr12), IDTVEC(icu_intr13),
73 IDTVEC(icu_intr14), IDTVEC(icu_intr15);
c8fe38ae 74
35e45e47
SZ
75static inthand_t *icu_intr[ICU_HWI_VECTORS] = {
76 &IDTVEC(icu_intr0), &IDTVEC(icu_intr1),
77 &IDTVEC(icu_intr2), &IDTVEC(icu_intr3),
78 &IDTVEC(icu_intr4), &IDTVEC(icu_intr5),
79 &IDTVEC(icu_intr6), &IDTVEC(icu_intr7),
80 &IDTVEC(icu_intr8), &IDTVEC(icu_intr9),
81 &IDTVEC(icu_intr10), &IDTVEC(icu_intr11),
82 &IDTVEC(icu_intr12), &IDTVEC(icu_intr13),
83 &IDTVEC(icu_intr14), &IDTVEC(icu_intr15)
c8fe38ae
MD
84};
85
a3dd9120
SZ
86static struct icu_irqmap {
87 int im_type; /* ICU_IMT_ */
88 enum intr_trigger im_trig;
759fc533 89 int im_msi_base;
b3cf468e 90} icu_irqmaps[MAXCPU][IDT_HWI_VECTORS];
a3dd9120 91
759fc533
SZ
92static struct lwkt_token icu_irqmap_tok =
93 LWKT_TOKEN_INITIALIZER(icu_irqmap_token);
94
a3dd9120
SZ
95#define ICU_IMT_UNUSED 0 /* KEEP THIS */
96#define ICU_IMT_RESERVED 1
97#define ICU_IMT_LINE 2
474ba684 98#define ICU_IMT_SYSCALL 3
759fc533 99#define ICU_IMT_MSI 4
a3dd9120 100
b3cf468e
SZ
101#define ICU_IMT_ISHWI(map) ((map)->im_type != ICU_IMT_RESERVED && \
102 (map)->im_type != ICU_IMT_SYSCALL)
103
10db3cc6
SZ
104extern void ICU_INTREN(int);
105extern void ICU_INTRDIS(int);
106
85bcaa51
SZ
107extern int imcr_present;
108
1967234a
SZ
109static void icu_abi_intr_enable(int);
110static void icu_abi_intr_disable(int);
f416026e
SZ
111static void icu_abi_intr_setup(int, int);
112static void icu_abi_intr_teardown(int);
aea76754 113static void icu_abi_intr_config(int, enum intr_trigger, enum intr_polarity);
a05c798c 114static int icu_abi_intr_cpuid(int);
aea76754 115
759fc533
SZ
116static int icu_abi_msi_alloc(int [], int, int);
117static void icu_abi_msi_release(const int [], int, int);
118static void icu_abi_msi_map(int, uint64_t *, uint32_t *, int);
119
aea76754
SZ
120static void icu_abi_finalize(void);
121static void icu_abi_cleanup(void);
122static void icu_abi_setdefault(void);
123static void icu_abi_stabilize(void);
124static void icu_abi_initmap(void);
b3cf468e 125static void icu_abi_rman_setup(struct rman *);
10db3cc6 126
faaf4131 127struct machintr_abi MachIntrABI_ICU = {
339478ac 128 MACHINTR_ICU,
1967234a
SZ
129 .intr_disable = icu_abi_intr_disable,
130 .intr_enable = icu_abi_intr_enable,
f416026e
SZ
131 .intr_setup = icu_abi_intr_setup,
132 .intr_teardown = icu_abi_intr_teardown,
aea76754 133 .intr_config = icu_abi_intr_config,
a05c798c 134 .intr_cpuid = icu_abi_intr_cpuid,
35b2edcb 135
759fc533
SZ
136 .msi_alloc = icu_abi_msi_alloc,
137 .msi_release = icu_abi_msi_release,
138 .msi_map = icu_abi_msi_map,
139
aea76754
SZ
140 .finalize = icu_abi_finalize,
141 .cleanup = icu_abi_cleanup,
142 .setdefault = icu_abi_setdefault,
143 .stabilize = icu_abi_stabilize,
b3cf468e
SZ
144 .initmap = icu_abi_initmap,
145 .rman_setup = icu_abi_rman_setup
c8fe38ae
MD
146};
147
759fc533
SZ
148static int icu_abi_msi_start; /* NOTE: for testing only */
149
c8fe38ae
MD
150/*
151 * WARNING! SMP builds can use the ICU now so this code must be MP safe.
152 */
c8fe38ae 153
1967234a
SZ
154static void
155icu_abi_intr_enable(int irq)
156{
157 const struct icu_irqmap *map;
158
159 KASSERT(irq >= 0 && irq < IDT_HWI_VECTORS,
160 ("icu enable, invalid irq %d\n", irq));
161
162 map = &icu_irqmaps[mycpuid][irq];
163 KASSERT(ICU_IMT_ISHWI(map),
164 ("icu enable, not hwi irq %d, type %d, cpu%d\n",
165 irq, map->im_type, mycpuid));
759fc533 166 if (map->im_type != ICU_IMT_LINE)
1967234a 167 return;
1967234a
SZ
168
169 ICU_INTREN(irq);
170}
171
172static void
173icu_abi_intr_disable(int irq)
174{
175 const struct icu_irqmap *map;
176
177 KASSERT(irq >= 0 && irq < IDT_HWI_VECTORS,
178 ("icu disable, invalid irq %d\n", irq));
179
180 map = &icu_irqmaps[mycpuid][irq];
181 KASSERT(ICU_IMT_ISHWI(map),
182 ("icu disable, not hwi irq %d, type %d, cpu%d\n",
183 irq, map->im_type, mycpuid));
759fc533 184 if (map->im_type != ICU_IMT_LINE)
1967234a 185 return;
1967234a
SZ
186
187 ICU_INTRDIS(irq);
188}
189
c8fe38ae
MD
190/*
191 * Called before interrupts are physically enabled
192 */
193static void
aea76754 194icu_abi_stabilize(void)
c8fe38ae 195{
339478ac
SZ
196 int intr;
197
7bf5fa56 198 for (intr = 0; intr < ICU_HWI_VECTORS; ++intr)
1967234a
SZ
199 ICU_INTRDIS(intr);
200 ICU_INTREN(ICU_IRQ_SLAVE);
7bf5fa56
SZ
201}
202
203/*
204 * Called after interrupts physically enabled but before the
205 * critical section is released.
206 */
207static void
aea76754 208icu_abi_cleanup(void)
7bf5fa56
SZ
209{
210 bzero(mdcpu->gd_ipending, sizeof(mdcpu->gd_ipending));
211}
212
213/*
214 * Called after stablize and cleanup; critical section is not
215 * held and interrupts are not physically disabled.
7bf5fa56
SZ
216 */
217static void
aea76754 218icu_abi_finalize(void)
7bf5fa56
SZ
219{
220 KKASSERT(MachIntrABI.type == MACHINTR_ICU);
f45bfca0 221 KKASSERT(!ioapic_enable);
339478ac 222
339478ac
SZ
223 /*
224 * If an IMCR is present, programming bit 0 disconnects the 8259
225 * from the BSP. The 8259 may still be connected to LINT0 on the
226 * BSP's LAPIC.
227 *
228 * If we are running SMP the LAPIC is active, try to use virtual
229 * wire mode so we can use other interrupt sources within the LAPIC
230 * in addition to the 8259.
231 */
9d758cc4 232 if (imcr_present) {
339478ac
SZ
233 outb(0x22, 0x70);
234 outb(0x23, 0x01);
7bf5fa56 235 }
c8fe38ae
MD
236}
237
f416026e
SZ
238static void
239icu_abi_intr_setup(int intr, int flags)
c8fe38ae 240{
1967234a 241 const struct icu_irqmap *map;
339478ac
SZ
242 register_t ef;
243
1967234a
SZ
244 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
245 ("icu setup, invalid irq %d\n", intr));
246
247 map = &icu_irqmaps[mycpuid][intr];
248 KASSERT(ICU_IMT_ISHWI(map),
249 ("icu setup, not hwi irq %d, type %d, cpu%d\n",
250 intr, map->im_type, mycpuid));
759fc533 251 if (map->im_type != ICU_IMT_LINE)
1967234a 252 return;
339478ac
SZ
253
254 ef = read_rflags();
255 cpu_disable_intr();
339478ac 256
26cf64b2 257 ICU_INTREN(intr);
339478ac 258
f416026e
SZ
259 write_rflags(ef);
260}
261
262static void
263icu_abi_intr_teardown(int intr)
264{
1967234a 265 const struct icu_irqmap *map;
f416026e
SZ
266 register_t ef;
267
1967234a
SZ
268 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
269 ("icu teardown, invalid irq %d\n", intr));
270
271 map = &icu_irqmaps[mycpuid][intr];
272 KASSERT(ICU_IMT_ISHWI(map),
273 ("icu teardown, not hwi irq %d, type %d, cpu%d\n",
274 intr, map->im_type, mycpuid));
759fc533 275 if (map->im_type != ICU_IMT_LINE)
1967234a 276 return;
f416026e
SZ
277
278 ef = read_rflags();
279 cpu_disable_intr();
280
26cf64b2 281 ICU_INTRDIS(intr);
339478ac 282
339478ac 283 write_rflags(ef);
c8fe38ae 284}
10db3cc6
SZ
285
286static void
aea76754 287icu_abi_setdefault(void)
10db3cc6
SZ
288{
289 int intr;
290
291 for (intr = 0; intr < ICU_HWI_VECTORS; ++intr) {
292 if (intr == ICU_IRQ_SLAVE)
293 continue;
8a06c6ee
SZ
294 setidt_global(IDT_OFFSET + intr, icu_intr[intr],
295 SDT_SYSIGT, SEL_KPL, 0);
10db3cc6
SZ
296 }
297}
a3dd9120
SZ
298
299static void
aea76754 300icu_abi_initmap(void)
a3dd9120 301{
b3cf468e
SZ
302 int cpu;
303
759fc533
SZ
304 kgetenv_int("hw.icu.msi_start", &icu_abi_msi_start);
305 icu_abi_msi_start &= ~0x1f; /* MUST be 32 aligned */
306
b3cf468e
SZ
307 /*
308 * NOTE: ncpus is not ready yet
309 */
310 for (cpu = 0; cpu < MAXCPU; ++cpu) {
311 int i;
312
313 if (cpu != 0) {
314 for (i = 0; i < ICU_HWI_VECTORS; ++i)
315 icu_irqmaps[cpu][i].im_type = ICU_IMT_RESERVED;
316 } else {
317 for (i = 0; i < ICU_HWI_VECTORS; ++i)
318 icu_irqmaps[cpu][i].im_type = ICU_IMT_LINE;
319 icu_irqmaps[cpu][ICU_IRQ_SLAVE].im_type =
320 ICU_IMT_RESERVED;
321
322 if (elcr_found) {
323 for (i = 0; i < ICU_HWI_VECTORS; ++i) {
324 icu_irqmaps[cpu][i].im_trig =
325 elcr_read_trigger(i);
326 }
327 } else {
328 /*
329 * NOTE: Trigger mode does not matter at all
330 */
331 for (i = 0; i < ICU_HWI_VECTORS; ++i) {
332 icu_irqmaps[cpu][i].im_trig =
333 INTR_TRIGGER_EDGE;
334 }
335 }
336 }
759fc533
SZ
337
338 for (i = 0; i < IDT_HWI_VECTORS; ++i)
339 icu_irqmaps[cpu][i].im_msi_base = -1;
340
b3cf468e
SZ
341 icu_irqmaps[cpu][IDT_OFFSET_SYSCALL - IDT_OFFSET].im_type =
342 ICU_IMT_SYSCALL;
a3dd9120
SZ
343 }
344}
d1ae7328
SZ
345
346static void
aea76754 347icu_abi_intr_config(int irq, enum intr_trigger trig,
d1ae7328
SZ
348 enum intr_polarity pola __unused)
349{
16437a92
SZ
350 struct icu_irqmap *map;
351
352 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL);
353
451af8d9 354 KKASSERT(irq >= 0 && irq < IDT_HWI_VECTORS);
b3cf468e 355 map = &icu_irqmaps[0][irq];
16437a92
SZ
356
357 KKASSERT(map->im_type == ICU_IMT_LINE);
358
359 /* TODO: Check whether it is configured or not */
360
361 if (trig == map->im_trig)
362 return;
363
364 if (bootverbose) {
365 kprintf("ICU: irq %d, %s -> %s\n", irq,
366 intr_str_trigger(map->im_trig),
367 intr_str_trigger(trig));
368 }
369 map->im_trig = trig;
370
371 if (!elcr_found) {
372 if (bootverbose)
373 kprintf("ICU: no ELCR, skip irq %d config\n", irq);
374 return;
375 }
376 elcr_write_trigger(irq, map->im_trig);
d1ae7328 377}
a05c798c
SZ
378
379static int
380icu_abi_intr_cpuid(int irq __unused)
381{
382 return 0;
383}
b3cf468e
SZ
384
385static void
386icu_abi_rman_setup(struct rman *rm)
387{
388 int start, end, i;
389
390 KASSERT(rm->rm_cpuid >= 0 && rm->rm_cpuid < MAXCPU,
391 ("invalid rman cpuid %d", rm->rm_cpuid));
392
393 start = end = -1;
394 for (i = 0; i < IDT_HWI_VECTORS; ++i) {
395 const struct icu_irqmap *map = &icu_irqmaps[rm->rm_cpuid][i];
396
397 if (start < 0) {
398 if (ICU_IMT_ISHWI(map))
399 start = end = i;
400 } else {
401 if (ICU_IMT_ISHWI(map)) {
402 end = i;
403 } else {
404 KKASSERT(end >= 0);
405 if (bootverbose) {
406 kprintf("ICU: rman cpu%d %d - %d\n",
407 rm->rm_cpuid, start, end);
408 }
409 if (rman_manage_region(rm, start, end)) {
410 panic("rman_manage_region"
411 "(cpu%d %d - %d)", rm->rm_cpuid,
412 start, end);
413 }
414 start = end = -1;
415 }
416 }
417 }
418 if (start >= 0) {
419 KKASSERT(end >= 0);
420 if (bootverbose) {
421 kprintf("ICU: rman cpu%d %d - %d\n",
422 rm->rm_cpuid, start, end);
423 }
424 if (rman_manage_region(rm, start, end)) {
425 panic("rman_manage_region(cpu%d %d - %d)",
426 rm->rm_cpuid, start, end);
427 }
428 }
429}
759fc533
SZ
430
431static int
432icu_abi_msi_alloc(int intrs[], int count, int cpuid)
433{
434 int i, error;
435
436 KASSERT(cpuid >= 0 && cpuid < ncpus,
437 ("invalid cpuid %d", cpuid));
438
439 KASSERT(count > 0 && count <= 32, ("invalid count %d\n", count));
440 KASSERT((count & (count - 1)) == 0,
441 ("count %d is not power of 2\n", count));
442
443 lwkt_gettoken(&icu_irqmap_tok);
444
445 /*
446 * NOTE:
447 * Since IDT_OFFSET is 32, which is the maximum valid 'count',
448 * we do not need to find out the first properly aligned
449 * interrupt vector.
450 */
451
452 error = EMSGSIZE;
453 for (i = icu_abi_msi_start; i < IDT_HWI_VECTORS; i += count) {
454 int j;
455
456 if (icu_irqmaps[cpuid][i].im_type != ICU_IMT_UNUSED)
457 continue;
458
459 for (j = 1; j < count; ++j) {
460 if (icu_irqmaps[cpuid][i + j].im_type != ICU_IMT_UNUSED)
461 break;
462 }
463 if (j != count)
464 continue;
465
466 for (j = 0; j < count; ++j) {
467 struct icu_irqmap *map;
468 int intr = i + j;
469
470 map = &icu_irqmaps[cpuid][intr];
471 KASSERT(map->im_msi_base < 0,
472 ("intr %d, stale MSI-base %d\n",
473 intr, map->im_msi_base));
474
475 map->im_type = ICU_IMT_MSI;
476 map->im_msi_base = i;
477
478 intrs[j] = intr;
479 msi_setup(intr, cpuid);
480
481 if (bootverbose) {
482 kprintf("alloc MSI intr %d on cpu%d\n",
483 intr, cpuid);
484 }
485
486 }
487 error = 0;
488 break;
489 }
490
491 lwkt_reltoken(&icu_irqmap_tok);
492
493 return error;
494}
495
496static void
497icu_abi_msi_release(const int intrs[], int count, int cpuid)
498{
499 int i, msi_base = -1, intr_next = -1, mask;
500
501 KASSERT(cpuid >= 0 && cpuid < ncpus,
502 ("invalid cpuid %d", cpuid));
503
504 KASSERT(count > 0 && count <= 32, ("invalid count %d\n", count));
505
506 mask = count - 1;
507 KASSERT((count & mask) == 0, ("count %d is not power of 2\n", count));
508
509 lwkt_gettoken(&icu_irqmap_tok);
510
511 for (i = 0; i < count; ++i) {
512 struct icu_irqmap *map;
513 int intr = intrs[i];
514
515 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
516 ("invalid intr %d\n", intr));
517
518 map = &icu_irqmaps[cpuid][intr];
519 KASSERT(map->im_type == ICU_IMT_MSI,
520 ("try release non-MSI intr %d, type %d\n",
521 intr, map->im_type));
522 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr,
523 ("intr %d, invalid MSI-base %d\n", intr, map->im_msi_base));
524 KASSERT((map->im_msi_base & mask) == 0,
525 ("intr %d, MSI-base %d is not proper aligned %d\n",
526 intr, map->im_msi_base, count));
527
528 if (msi_base < 0) {
529 msi_base = map->im_msi_base;
530 } else {
531 KASSERT(map->im_msi_base == msi_base,
532 ("intr %d, inconsistent MSI-base, "
533 "was %d, now %d\n",
534 intr, msi_base, map->im_msi_base));
535 }
536
537 if (intr_next < intr)
538 intr_next = intr;
539
540 map->im_type = ICU_IMT_UNUSED;
541 map->im_msi_base = -1;
542
543 if (bootverbose)
544 kprintf("release MSI intr %d on cpu%d\n", intr, cpuid);
545 }
546
547 KKASSERT(intr_next > 0);
548 KKASSERT(msi_base >= 0);
549
550 ++intr_next;
551 if (intr_next < IDT_HWI_VECTORS) {
552 const struct icu_irqmap *map = &icu_irqmaps[cpuid][intr_next];
553
554 if (map->im_type == ICU_IMT_MSI) {
555 KASSERT(map->im_msi_base != msi_base,
556 ("more than %d MSI was allocated\n", count));
557 }
558 }
559
560 lwkt_reltoken(&icu_irqmap_tok);
561}
562
563static void
564icu_abi_msi_map(int intr, uint64_t *addr, uint32_t *data, int cpuid)
565{
566 const struct icu_irqmap *map;
567
568 KASSERT(cpuid >= 0 && cpuid < ncpus,
569 ("invalid cpuid %d", cpuid));
570
571 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
572 ("invalid intr %d\n", intr));
573
574 lwkt_gettoken(&icu_irqmap_tok);
575
576 map = &icu_irqmaps[cpuid][intr];
577 KASSERT(map->im_type == ICU_IMT_MSI,
578 ("try map non-MSI intr %d, type %d\n", intr, map->im_type));
579 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr,
580 ("intr %d, invalid MSI-base %d\n", intr, map->im_msi_base));
581
582 msi_map(map->im_msi_base, addr, data, cpuid);
583
584 if (bootverbose)
585 kprintf("map MSI intr %d on cpu%d\n", intr, cpuid);
586
587 lwkt_reltoken(&icu_irqmap_tok);
588}