2 * Copyright (c) 1991 The Regents of the University of California.
3 * Copyright (c) 2005,2008 The DragonFly Project.
6 * This code is derived from software contributed to The DragonFly Project
7 * by Matthew Dillon <dillon@backplane.com>
9 * This code is derived from software contributed to Berkeley by
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in
20 * the documentation and/or other materials provided with the
22 * 3. Neither the name of The DragonFly Project nor the names of its
23 * contributors may be used to endorse or promote products derived
24 * from this software without specific, prior written permission.
26 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
29 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
30 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
31 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
32 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
33 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
34 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
35 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
36 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
40 #include <sys/param.h>
41 #include <sys/systm.h>
42 #include <sys/kernel.h>
43 #include <sys/machintr.h>
44 #include <sys/interrupt.h>
48 #include <machine/segments.h>
49 #include <machine/md_var.h>
50 #include <machine/intr_machdep.h>
51 #include <machine/globaldata.h>
52 #include <machine/smp.h>
53 #include <machine/msi_var.h>
55 #include <sys/thread2.h>
57 #include <machine_base/icu/elcr_var.h>
59 #include <machine_base/icu/icu.h>
60 #include <machine_base/icu/icu_ipl.h>
61 #include <machine_base/apic/ioapic.h>
64 IDTVEC(icu_intr0), IDTVEC(icu_intr1),
65 IDTVEC(icu_intr2), IDTVEC(icu_intr3),
66 IDTVEC(icu_intr4), IDTVEC(icu_intr5),
67 IDTVEC(icu_intr6), IDTVEC(icu_intr7),
68 IDTVEC(icu_intr8), IDTVEC(icu_intr9),
69 IDTVEC(icu_intr10), IDTVEC(icu_intr11),
70 IDTVEC(icu_intr12), IDTVEC(icu_intr13),
71 IDTVEC(icu_intr14), IDTVEC(icu_intr15);
73 static inthand_t *icu_intr[ICU_HWI_VECTORS] = {
74 &IDTVEC(icu_intr0), &IDTVEC(icu_intr1),
75 &IDTVEC(icu_intr2), &IDTVEC(icu_intr3),
76 &IDTVEC(icu_intr4), &IDTVEC(icu_intr5),
77 &IDTVEC(icu_intr6), &IDTVEC(icu_intr7),
78 &IDTVEC(icu_intr8), &IDTVEC(icu_intr9),
79 &IDTVEC(icu_intr10), &IDTVEC(icu_intr11),
80 &IDTVEC(icu_intr12), &IDTVEC(icu_intr13),
81 &IDTVEC(icu_intr14), &IDTVEC(icu_intr15)
84 static struct icu_irqmap {
85 int im_type; /* ICU_IMT_ */
86 enum intr_trigger im_trig;
88 } icu_irqmaps[MAXCPU][IDT_HWI_VECTORS];
90 static struct lwkt_token icu_irqmap_tok =
91 LWKT_TOKEN_INITIALIZER(icu_irqmap_token);
93 #define ICU_IMT_UNUSED 0 /* KEEP THIS */
94 #define ICU_IMT_RESERVED 1
95 #define ICU_IMT_LEGACY 2
96 #define ICU_IMT_SYSCALL 3
98 #define ICU_IMT_MSIX 5
100 #define ICU_IMT_ISHWI(map) ((map)->im_type != ICU_IMT_RESERVED && \
101 (map)->im_type != ICU_IMT_SYSCALL)
103 extern void ICU_INTREN(int);
104 extern void ICU_INTRDIS(int);
106 extern int imcr_present;
108 static void icu_abi_intr_enable(int);
109 static void icu_abi_intr_disable(int);
110 static void icu_abi_intr_setup(int, int);
111 static void icu_abi_intr_teardown(int);
113 static void icu_abi_legacy_intr_config(int, enum intr_trigger,
115 static int icu_abi_legacy_intr_cpuid(int);
117 static int icu_abi_msi_alloc(int [], int, int);
118 static void icu_abi_msi_release(const int [], int, int);
119 static void icu_abi_msi_map(int, uint64_t *, uint32_t *, int);
120 static int icu_abi_msix_alloc(int *, int);
121 static void icu_abi_msix_release(int, int);
123 static int icu_abi_msi_alloc_intern(int, const char *,
125 static void icu_abi_msi_release_intern(int, const char *,
126 const int [], int, int);
128 static void icu_abi_finalize(void);
129 static void icu_abi_cleanup(void);
130 static void icu_abi_setdefault(void);
131 static void icu_abi_stabilize(void);
132 static void icu_abi_initmap(void);
133 static void icu_abi_rman_setup(struct rman *);
135 struct machintr_abi MachIntrABI_ICU = {
137 .intr_disable = icu_abi_intr_disable,
138 .intr_enable = icu_abi_intr_enable,
139 .intr_setup = icu_abi_intr_setup,
140 .intr_teardown = icu_abi_intr_teardown,
142 .legacy_intr_config = icu_abi_legacy_intr_config,
143 .legacy_intr_cpuid = icu_abi_legacy_intr_cpuid,
145 .msi_alloc = icu_abi_msi_alloc,
146 .msi_release = icu_abi_msi_release,
147 .msi_map = icu_abi_msi_map,
148 .msix_alloc = icu_abi_msix_alloc,
149 .msix_release = icu_abi_msix_release,
151 .finalize = icu_abi_finalize,
152 .cleanup = icu_abi_cleanup,
153 .setdefault = icu_abi_setdefault,
154 .stabilize = icu_abi_stabilize,
155 .initmap = icu_abi_initmap,
156 .rman_setup = icu_abi_rman_setup
159 static int icu_abi_msi_start; /* NOTE: for testing only */
162 * WARNING! SMP builds can use the ICU now so this code must be MP safe.
166 icu_abi_intr_enable(int irq)
168 const struct icu_irqmap *map;
170 KASSERT(irq >= 0 && irq < IDT_HWI_VECTORS,
171 ("icu enable, invalid irq %d", irq));
173 map = &icu_irqmaps[mycpuid][irq];
174 KASSERT(ICU_IMT_ISHWI(map),
175 ("icu enable, not hwi irq %d, type %d, cpu%d",
176 irq, map->im_type, mycpuid));
177 if (map->im_type != ICU_IMT_LEGACY)
184 icu_abi_intr_disable(int irq)
186 const struct icu_irqmap *map;
188 KASSERT(irq >= 0 && irq < IDT_HWI_VECTORS,
189 ("icu disable, invalid irq %d", irq));
191 map = &icu_irqmaps[mycpuid][irq];
192 KASSERT(ICU_IMT_ISHWI(map),
193 ("icu disable, not hwi irq %d, type %d, cpu%d",
194 irq, map->im_type, mycpuid));
195 if (map->im_type != ICU_IMT_LEGACY)
202 * Called before interrupts are physically enabled
205 icu_abi_stabilize(void)
209 for (intr = 0; intr < ICU_HWI_VECTORS; ++intr)
211 ICU_INTREN(ICU_IRQ_SLAVE);
215 * Called after interrupts physically enabled but before the
216 * critical section is released.
219 icu_abi_cleanup(void)
221 bzero(mdcpu->gd_ipending, sizeof(mdcpu->gd_ipending));
225 * Called after stablize and cleanup; critical section is not
226 * held and interrupts are not physically disabled.
229 icu_abi_finalize(void)
231 KKASSERT(MachIntrABI.type == MACHINTR_ICU);
232 KKASSERT(!ioapic_enable);
235 * If an IMCR is present, programming bit 0 disconnects the 8259
236 * from the BSP. The 8259 may still be connected to LINT0 on the
239 * If we are running SMP the LAPIC is active, try to use virtual
240 * wire mode so we can use other interrupt sources within the LAPIC
241 * in addition to the 8259.
250 icu_abi_intr_setup(int intr, int flags)
252 const struct icu_irqmap *map;
255 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
256 ("icu setup, invalid irq %d", intr));
258 map = &icu_irqmaps[mycpuid][intr];
259 KASSERT(ICU_IMT_ISHWI(map),
260 ("icu setup, not hwi irq %d, type %d, cpu%d",
261 intr, map->im_type, mycpuid));
262 if (map->im_type != ICU_IMT_LEGACY)
274 icu_abi_intr_teardown(int intr)
276 const struct icu_irqmap *map;
279 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
280 ("icu teardown, invalid irq %d", intr));
282 map = &icu_irqmaps[mycpuid][intr];
283 KASSERT(ICU_IMT_ISHWI(map),
284 ("icu teardown, not hwi irq %d, type %d, cpu%d",
285 intr, map->im_type, mycpuid));
286 if (map->im_type != ICU_IMT_LEGACY)
298 icu_abi_setdefault(void)
302 for (intr = 0; intr < ICU_HWI_VECTORS; ++intr) {
303 if (intr == ICU_IRQ_SLAVE)
305 setidt_global(IDT_OFFSET + intr, icu_intr[intr],
306 SDT_SYSIGT, SEL_KPL, 0);
311 icu_abi_initmap(void)
315 kgetenv_int("hw.icu.msi_start", &icu_abi_msi_start);
316 icu_abi_msi_start &= ~0x1f; /* MUST be 32 aligned */
319 * NOTE: ncpus is not ready yet
321 for (cpu = 0; cpu < MAXCPU; ++cpu) {
325 for (i = 0; i < ICU_HWI_VECTORS; ++i)
326 icu_irqmaps[cpu][i].im_type = ICU_IMT_RESERVED;
328 for (i = 0; i < ICU_HWI_VECTORS; ++i)
329 icu_irqmaps[cpu][i].im_type = ICU_IMT_LEGACY;
330 icu_irqmaps[cpu][ICU_IRQ_SLAVE].im_type =
334 for (i = 0; i < ICU_HWI_VECTORS; ++i) {
335 icu_irqmaps[cpu][i].im_trig =
336 elcr_read_trigger(i);
340 * NOTE: Trigger mode does not matter at all
342 for (i = 0; i < ICU_HWI_VECTORS; ++i) {
343 icu_irqmaps[cpu][i].im_trig =
349 for (i = 0; i < IDT_HWI_VECTORS; ++i)
350 icu_irqmaps[cpu][i].im_msi_base = -1;
352 icu_irqmaps[cpu][IDT_OFFSET_SYSCALL - IDT_OFFSET].im_type =
358 icu_abi_legacy_intr_config(int irq, enum intr_trigger trig,
359 enum intr_polarity pola __unused)
361 struct icu_irqmap *map;
363 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL);
365 KKASSERT(irq >= 0 && irq < IDT_HWI_VECTORS);
366 map = &icu_irqmaps[0][irq];
368 KKASSERT(map->im_type == ICU_IMT_LEGACY);
370 /* TODO: Check whether it is configured or not */
372 if (trig == map->im_trig)
376 kprintf("ICU: irq %d, %s -> %s\n", irq,
377 intr_str_trigger(map->im_trig),
378 intr_str_trigger(trig));
384 kprintf("ICU: no ELCR, skip irq %d config\n", irq);
387 elcr_write_trigger(irq, map->im_trig);
391 icu_abi_legacy_intr_cpuid(int irq __unused)
397 icu_abi_rman_setup(struct rman *rm)
401 KASSERT(rm->rm_cpuid >= 0 && rm->rm_cpuid < MAXCPU,
402 ("invalid rman cpuid %d", rm->rm_cpuid));
405 for (i = 0; i < IDT_HWI_VECTORS; ++i) {
406 const struct icu_irqmap *map = &icu_irqmaps[rm->rm_cpuid][i];
409 if (ICU_IMT_ISHWI(map))
412 if (ICU_IMT_ISHWI(map)) {
417 kprintf("ICU: rman cpu%d %d - %d\n",
418 rm->rm_cpuid, start, end);
420 if (rman_manage_region(rm, start, end)) {
421 panic("rman_manage_region"
422 "(cpu%d %d - %d)", rm->rm_cpuid,
432 kprintf("ICU: rman cpu%d %d - %d\n",
433 rm->rm_cpuid, start, end);
435 if (rman_manage_region(rm, start, end)) {
436 panic("rman_manage_region(cpu%d %d - %d)",
437 rm->rm_cpuid, start, end);
443 icu_abi_msi_alloc_intern(int type, const char *desc,
444 int intrs[], int count, int cpuid)
448 KASSERT(cpuid >= 0 && cpuid < ncpus,
449 ("invalid cpuid %d", cpuid));
451 KASSERT(count > 0 && count <= 32, ("invalid count %d", count));
452 KASSERT((count & (count - 1)) == 0,
453 ("count %d is not power of 2", count));
455 lwkt_gettoken(&icu_irqmap_tok);
459 * Since IDT_OFFSET is 32, which is the maximum valid 'count',
460 * we do not need to find out the first properly aligned
465 for (i = icu_abi_msi_start; i < IDT_HWI_VECTORS; i += count) {
468 if (icu_irqmaps[cpuid][i].im_type != ICU_IMT_UNUSED)
471 for (j = 1; j < count; ++j) {
472 if (icu_irqmaps[cpuid][i + j].im_type != ICU_IMT_UNUSED)
478 for (j = 0; j < count; ++j) {
479 struct icu_irqmap *map;
482 map = &icu_irqmaps[cpuid][intr];
483 KASSERT(map->im_msi_base < 0,
484 ("intr %d, stale %s-base %d",
485 intr, desc, map->im_msi_base));
488 map->im_msi_base = i;
491 msi_setup(intr, cpuid);
494 kprintf("alloc %s intr %d on cpu%d\n",
502 lwkt_reltoken(&icu_irqmap_tok);
508 icu_abi_msi_release_intern(int type, const char *desc,
509 const int intrs[], int count, int cpuid)
511 int i, msi_base = -1, intr_next = -1, mask;
513 KASSERT(cpuid >= 0 && cpuid < ncpus,
514 ("invalid cpuid %d", cpuid));
516 KASSERT(count > 0 && count <= 32, ("invalid count %d", count));
519 KASSERT((count & mask) == 0, ("count %d is not power of 2", count));
521 lwkt_gettoken(&icu_irqmap_tok);
523 for (i = 0; i < count; ++i) {
524 struct icu_irqmap *map;
527 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
528 ("invalid intr %d", intr));
530 map = &icu_irqmaps[cpuid][intr];
531 KASSERT(map->im_type == type,
532 ("trying to release non-%s intr %d, type %d", desc,
533 intr, map->im_type));
534 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr,
535 ("intr %d, invalid %s-base %d", intr, desc,
537 KASSERT((map->im_msi_base & mask) == 0,
538 ("intr %d, %s-base %d is not properly aligned %d",
539 intr, desc, map->im_msi_base, count));
542 msi_base = map->im_msi_base;
544 KASSERT(map->im_msi_base == msi_base,
545 ("intr %d, inconsistent %s-base, "
547 intr, desc, msi_base, map->im_msi_base));
550 if (intr_next < intr)
553 map->im_type = ICU_IMT_UNUSED;
554 map->im_msi_base = -1;
557 kprintf("release %s intr %d on cpu%d\n",
562 KKASSERT(intr_next > 0);
563 KKASSERT(msi_base >= 0);
566 if (intr_next < IDT_HWI_VECTORS) {
567 const struct icu_irqmap *map = &icu_irqmaps[cpuid][intr_next];
569 if (map->im_type == type) {
570 KASSERT(map->im_msi_base != msi_base,
571 ("more than %d %s was allocated", count, desc));
575 lwkt_reltoken(&icu_irqmap_tok);
579 icu_abi_msi_alloc(int intrs[], int count, int cpuid)
581 return icu_abi_msi_alloc_intern(ICU_IMT_MSI, "MSI",
582 intrs, count, cpuid);
586 icu_abi_msi_release(const int intrs[], int count, int cpuid)
588 icu_abi_msi_release_intern(ICU_IMT_MSI, "MSI",
589 intrs, count, cpuid);
593 icu_abi_msix_alloc(int *intr, int cpuid)
595 return icu_abi_msi_alloc_intern(ICU_IMT_MSIX, "MSI-X",
600 icu_abi_msix_release(int intr, int cpuid)
602 icu_abi_msi_release_intern(ICU_IMT_MSIX, "MXI-X",
607 icu_abi_msi_map(int intr, uint64_t *addr, uint32_t *data, int cpuid)
609 const struct icu_irqmap *map;
611 KASSERT(cpuid >= 0 && cpuid < ncpus,
612 ("invalid cpuid %d", cpuid));
614 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
615 ("invalid intr %d", intr));
617 lwkt_gettoken(&icu_irqmap_tok);
619 map = &icu_irqmaps[cpuid][intr];
620 KASSERT(map->im_type == ICU_IMT_MSI ||
621 map->im_type == ICU_IMT_MSIX,
622 ("trying to map non-MSI/MSI-X intr %d, type %d", intr, map->im_type));
623 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr,
624 ("intr %d, invalid %s-base %d", intr,
625 map->im_type == ICU_IMT_MSI ? "MSI" : "MSI-X",
628 msi_map(map->im_msi_base, addr, data, cpuid);
631 kprintf("map %s intr %d on cpu%d\n",
632 map->im_type == ICU_IMT_MSI ? "MSI" : "MSI-X",
636 lwkt_reltoken(&icu_irqmap_tok);