MachIntr: Add two methods to find IRQ
[dragonfly.git] / sys / platform / pc32 / icu / icu_abi.c
CommitLineData
06f5be02
MD
1/*
2 * Copyright (c) 2005 The DragonFly Project. All rights reserved.
10ff1029
MD
3 * Copyright (c) 1991 The Regents of the University of California.
4 * All rights reserved.
06f5be02
MD
5 *
6 * This code is derived from software contributed to The DragonFly Project
7 * by Matthew Dillon <dillon@backplane.com>
10ff1029
MD
8 *
9 * This code is derived from software contributed to Berkeley by
10 * William Jolitz.
06f5be02
MD
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 *
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in
20 * the documentation and/or other materials provided with the
21 * distribution.
22 * 3. Neither the name of The DragonFly Project nor the names of its
23 * contributors may be used to endorse or promote products derived
24 * from this software without specific, prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
29 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
30 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
31 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
32 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
33 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
34 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
35 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
36 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 * SUCH DAMAGE.
06f5be02
MD
38 */
39
37e7efec
MD
40#include <sys/param.h>
41#include <sys/systm.h>
42#include <sys/kernel.h>
43#include <sys/machintr.h>
10ff1029 44#include <sys/interrupt.h>
14bd3265 45#include <sys/rman.h>
10ff1029
MD
46#include <sys/bus.h>
47
48#include <machine/segments.h>
49#include <machine/md_var.h>
87cf6827 50#include <machine/intr_machdep.h>
0b692e79 51#include <machine/globaldata.h>
10db3cc6 52#include <machine/smp.h>
2a39d30f 53#include <machine/msi_var.h>
0b692e79
MD
54
55#include <sys/thread2.h>
5f456c40 56
7265a4fe 57#include <machine_base/icu/elcr_var.h>
9e0e3f85 58
7265a4fe
SZ
59#include <machine_base/icu/icu.h>
60#include <machine_base/icu/icu_ipl.h>
ed4d621d 61#include <machine_base/apic/ioapic.h>
37e7efec 62
10ff1029 63extern inthand_t
081be8a5
SZ
64 IDTVEC(icu_intr0), IDTVEC(icu_intr1),
65 IDTVEC(icu_intr2), IDTVEC(icu_intr3),
66 IDTVEC(icu_intr4), IDTVEC(icu_intr5),
67 IDTVEC(icu_intr6), IDTVEC(icu_intr7),
68 IDTVEC(icu_intr8), IDTVEC(icu_intr9),
69 IDTVEC(icu_intr10), IDTVEC(icu_intr11),
70 IDTVEC(icu_intr12), IDTVEC(icu_intr13),
71 IDTVEC(icu_intr14), IDTVEC(icu_intr15);
10ff1029 72
081be8a5
SZ
73static inthand_t *icu_intr[ICU_HWI_VECTORS] = {
74 &IDTVEC(icu_intr0), &IDTVEC(icu_intr1),
75 &IDTVEC(icu_intr2), &IDTVEC(icu_intr3),
76 &IDTVEC(icu_intr4), &IDTVEC(icu_intr5),
77 &IDTVEC(icu_intr6), &IDTVEC(icu_intr7),
78 &IDTVEC(icu_intr8), &IDTVEC(icu_intr9),
79 &IDTVEC(icu_intr10), &IDTVEC(icu_intr11),
80 &IDTVEC(icu_intr12), &IDTVEC(icu_intr13),
81 &IDTVEC(icu_intr14), &IDTVEC(icu_intr15)
10ff1029
MD
82};
83
a3dd9120
SZ
84static struct icu_irqmap {
85 int im_type; /* ICU_IMT_ */
86 enum intr_trigger im_trig;
2a39d30f 87 int im_msi_base;
86d692fe 88 uint32_t im_flags;
14bd3265 89} icu_irqmaps[MAXCPU][IDT_HWI_VECTORS];
a3dd9120 90
2a39d30f
SZ
91static struct lwkt_token icu_irqmap_tok =
92 LWKT_TOKEN_INITIALIZER(icu_irqmap_token);
93
a3dd9120
SZ
94#define ICU_IMT_UNUSED 0 /* KEEP THIS */
95#define ICU_IMT_RESERVED 1
2a32da90 96#define ICU_IMT_LEGACY 2
474ba684 97#define ICU_IMT_SYSCALL 3
2a39d30f
SZ
98#define ICU_IMT_SHADOW 4
99#define ICU_IMT_MSI 5
6bf6f751 100#define ICU_IMT_MSIX 6
a3dd9120 101
14bd3265 102#define ICU_IMT_ISHWI(map) ((map)->im_type != ICU_IMT_RESERVED && \
2a39d30f
SZ
103 (map)->im_type != ICU_IMT_SYSCALL && \
104 (map)->im_type != ICU_IMT_SHADOW)
14bd3265 105
86d692fe
SZ
106#define ICU_IMF_CONF 0x1
107
10db3cc6
SZ
108extern void ICU_INTREN(int);
109extern void ICU_INTRDIS(int);
110
1d903de5
SZ
111extern int imcr_present;
112
d9a0e2b3
SZ
113static void icu_abi_intr_enable(int);
114static void icu_abi_intr_disable(int);
f416026e
SZ
115static void icu_abi_intr_setup(int, int);
116static void icu_abi_intr_teardown(int);
bec969af
SZ
117
118static void icu_abi_legacy_intr_config(int, enum intr_trigger,
119 enum intr_polarity);
120static int icu_abi_legacy_intr_cpuid(int);
86d692fe
SZ
121static int icu_abi_legacy_intr_find(int, enum intr_trigger,
122 enum intr_polarity);
123static int icu_abi_legacy_intr_find_bygsi(int, enum intr_trigger,
124 enum intr_polarity);
780a6eec 125
2a39d30f
SZ
126static int icu_abi_msi_alloc(int [], int, int);
127static void icu_abi_msi_release(const int [], int, int);
128static void icu_abi_msi_map(int, uint64_t *, uint32_t *, int);
6bf6f751
SZ
129static int icu_abi_msix_alloc(int *, int);
130static void icu_abi_msix_release(int, int);
131
132static int icu_abi_msi_alloc_intern(int, const char *,
133 int [], int, int);
134static void icu_abi_msi_release_intern(int, const char *,
135 const int [], int, int);
2a39d30f 136
780a6eec
SZ
137static void icu_abi_finalize(void);
138static void icu_abi_cleanup(void);
139static void icu_abi_setdefault(void);
140static void icu_abi_stabilize(void);
141static void icu_abi_initmap(void);
14bd3265 142static void icu_abi_rman_setup(struct rman *);
10db3cc6 143
30c5f287 144struct machintr_abi MachIntrABI_ICU = {
54e1df6b 145 MACHINTR_ICU,
d9a0e2b3
SZ
146 .intr_disable = icu_abi_intr_disable,
147 .intr_enable = icu_abi_intr_enable,
f416026e
SZ
148 .intr_setup = icu_abi_intr_setup,
149 .intr_teardown = icu_abi_intr_teardown,
bec969af
SZ
150
151 .legacy_intr_config = icu_abi_legacy_intr_config,
152 .legacy_intr_cpuid = icu_abi_legacy_intr_cpuid,
86d692fe
SZ
153 .legacy_intr_find = icu_abi_legacy_intr_find,
154 .legacy_intr_find_bygsi = icu_abi_legacy_intr_find_bygsi,
35b2edcb 155
2a39d30f
SZ
156 .msi_alloc = icu_abi_msi_alloc,
157 .msi_release = icu_abi_msi_release,
158 .msi_map = icu_abi_msi_map,
6bf6f751
SZ
159 .msix_alloc = icu_abi_msix_alloc,
160 .msix_release = icu_abi_msix_release,
2a39d30f 161
780a6eec
SZ
162 .finalize = icu_abi_finalize,
163 .cleanup = icu_abi_cleanup,
164 .setdefault = icu_abi_setdefault,
165 .stabilize = icu_abi_stabilize,
14bd3265
SZ
166 .initmap = icu_abi_initmap,
167 .rman_setup = icu_abi_rman_setup
37e7efec
MD
168};
169
2a39d30f
SZ
170static int icu_abi_msi_start; /* NOTE: for testing only */
171
97359a5b
MD
172/*
173 * WARNING! SMP builds can use the ICU now so this code must be MP safe.
174 */
37e7efec 175
d9a0e2b3
SZ
176static void
177icu_abi_intr_enable(int irq)
178{
179 const struct icu_irqmap *map;
180
181 KASSERT(irq >= 0 && irq < IDT_HWI_VECTORS,
ed20d0e3 182 ("icu enable, invalid irq %d", irq));
d9a0e2b3
SZ
183
184 map = &icu_irqmaps[mycpuid][irq];
185 KASSERT(ICU_IMT_ISHWI(map),
ed20d0e3 186 ("icu enable, not hwi irq %d, type %d, cpu%d",
d9a0e2b3 187 irq, map->im_type, mycpuid));
2a32da90 188 if (map->im_type != ICU_IMT_LEGACY)
d9a0e2b3 189 return;
d9a0e2b3
SZ
190
191 ICU_INTREN(irq);
192}
193
194static void
195icu_abi_intr_disable(int irq)
196{
197 const struct icu_irqmap *map;
198
199 KASSERT(irq >= 0 && irq < IDT_HWI_VECTORS,
ed20d0e3 200 ("icu disable, invalid irq %d", irq));
d9a0e2b3
SZ
201
202 map = &icu_irqmaps[mycpuid][irq];
203 KASSERT(ICU_IMT_ISHWI(map),
ed20d0e3 204 ("icu disable, not hwi irq %d, type %d, cpu%d",
d9a0e2b3 205 irq, map->im_type, mycpuid));
2a32da90 206 if (map->im_type != ICU_IMT_LEGACY)
d9a0e2b3 207 return;
d9a0e2b3
SZ
208
209 ICU_INTRDIS(irq);
210}
211
0b692e79
MD
212/*
213 * Called before interrupts are physically enabled
214 */
37e7efec 215static void
780a6eec 216icu_abi_stabilize(void)
37e7efec 217{
54e1df6b
SZ
218 int intr;
219
7bf5fa56 220 for (intr = 0; intr < ICU_HWI_VECTORS; ++intr)
d9a0e2b3
SZ
221 ICU_INTRDIS(intr);
222 ICU_INTREN(ICU_IRQ_SLAVE);
7bf5fa56
SZ
223}
224
225/*
226 * Called after interrupts physically enabled but before the
227 * critical section is released.
228 */
229static void
780a6eec 230icu_abi_cleanup(void)
7bf5fa56
SZ
231{
232 bzero(mdcpu->gd_ipending, sizeof(mdcpu->gd_ipending));
233}
234
235/*
236 * Called after stablize and cleanup; critical section is not
237 * held and interrupts are not physically disabled.
7bf5fa56
SZ
238 */
239static void
780a6eec 240icu_abi_finalize(void)
7bf5fa56
SZ
241{
242 KKASSERT(MachIntrABI.type == MACHINTR_ICU);
f45bfca0 243 KKASSERT(!ioapic_enable);
54e1df6b 244
54e1df6b
SZ
245 /*
246 * If an IMCR is present, programming bit 0 disconnects the 8259
247 * from the BSP. The 8259 may still be connected to LINT0 on the
248 * BSP's LAPIC.
249 *
250 * If we are running SMP the LAPIC is active, try to use virtual
251 * wire mode so we can use other interrupt sources within the LAPIC
252 * in addition to the 8259.
253 */
9d758cc4 254 if (imcr_present) {
54e1df6b
SZ
255 outb(0x22, 0x70);
256 outb(0x23, 0x01);
7bf5fa56 257 }
0b692e79
MD
258}
259
f416026e
SZ
260static void
261icu_abi_intr_setup(int intr, int flags __unused)
10ff1029 262{
d9a0e2b3 263 const struct icu_irqmap *map;
54e1df6b
SZ
264 u_long ef;
265
d9a0e2b3 266 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
ed20d0e3 267 ("icu setup, invalid irq %d", intr));
d9a0e2b3
SZ
268
269 map = &icu_irqmaps[mycpuid][intr];
270 KASSERT(ICU_IMT_ISHWI(map),
ed20d0e3 271 ("icu setup, not hwi irq %d, type %d, cpu%d",
d9a0e2b3 272 intr, map->im_type, mycpuid));
2a32da90 273 if (map->im_type != ICU_IMT_LEGACY)
d9a0e2b3 274 return;
54e1df6b
SZ
275
276 ef = read_eflags();
277 cpu_disable_intr();
54e1df6b 278
eaeca813 279 ICU_INTREN(intr);
54e1df6b 280
f416026e
SZ
281 write_eflags(ef);
282}
283
284static void
285icu_abi_intr_teardown(int intr)
286{
d9a0e2b3 287 const struct icu_irqmap *map;
f416026e
SZ
288 u_long ef;
289
d9a0e2b3 290 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
ed20d0e3 291 ("icu teardown, invalid irq %d", intr));
d9a0e2b3
SZ
292
293 map = &icu_irqmaps[mycpuid][intr];
294 KASSERT(ICU_IMT_ISHWI(map),
ed20d0e3 295 ("icu teardown, not hwi irq %d, type %d, cpu%d",
d9a0e2b3 296 intr, map->im_type, mycpuid));
2a32da90 297 if (map->im_type != ICU_IMT_LEGACY)
d9a0e2b3 298 return;
f416026e
SZ
299
300 ef = read_eflags();
301 cpu_disable_intr();
302
eaeca813 303 ICU_INTRDIS(intr);
54e1df6b 304
54e1df6b 305 write_eflags(ef);
10ff1029 306}
10db3cc6
SZ
307
308static void
780a6eec 309icu_abi_setdefault(void)
10db3cc6
SZ
310{
311 int intr;
312
313 for (intr = 0; intr < ICU_HWI_VECTORS; ++intr) {
314 if (intr == ICU_IRQ_SLAVE)
315 continue;
316 setidt(IDT_OFFSET + intr, icu_intr[intr], SDT_SYS386IGT,
317 SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
318 }
319}
a3dd9120
SZ
320
321static void
780a6eec 322icu_abi_initmap(void)
a3dd9120 323{
14bd3265
SZ
324 int cpu;
325
2a39d30f
SZ
326 kgetenv_int("hw.icu.msi_start", &icu_abi_msi_start);
327 icu_abi_msi_start &= ~0x1f; /* MUST be 32 aligned */
328
14bd3265
SZ
329 /*
330 * NOTE: ncpus is not ready yet
331 */
332 for (cpu = 0; cpu < MAXCPU; ++cpu) {
333 int i;
334
335 if (cpu != 0) {
336 for (i = 0; i < ICU_HWI_VECTORS; ++i)
337 icu_irqmaps[cpu][i].im_type = ICU_IMT_RESERVED;
338 } else {
339 for (i = 0; i < ICU_HWI_VECTORS; ++i)
2a32da90 340 icu_irqmaps[cpu][i].im_type = ICU_IMT_LEGACY;
14bd3265
SZ
341 icu_irqmaps[cpu][ICU_IRQ_SLAVE].im_type =
342 ICU_IMT_RESERVED;
343
344 if (elcr_found) {
345 for (i = 0; i < ICU_HWI_VECTORS; ++i) {
346 icu_irqmaps[cpu][i].im_trig =
347 elcr_read_trigger(i);
348 }
349 } else {
350 /*
351 * NOTE: Trigger mode does not matter at all
352 */
353 for (i = 0; i < ICU_HWI_VECTORS; ++i) {
354 icu_irqmaps[cpu][i].im_trig =
355 INTR_TRIGGER_EDGE;
356 }
357 }
358 }
2a39d30f
SZ
359
360 for (i = 0; i < IDT_HWI_VECTORS; ++i)
361 icu_irqmaps[cpu][i].im_msi_base = -1;
362
14bd3265
SZ
363 icu_irqmaps[cpu][IDT_OFFSET_SYSCALL - IDT_OFFSET].im_type =
364 ICU_IMT_SYSCALL;
a3dd9120
SZ
365 }
366}
d1ae7328
SZ
367
368static void
bec969af 369icu_abi_legacy_intr_config(int irq, enum intr_trigger trig,
d1ae7328
SZ
370 enum intr_polarity pola __unused)
371{
16437a92
SZ
372 struct icu_irqmap *map;
373
374 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL);
375
b2150df1 376 KKASSERT(irq >= 0 && irq < IDT_HWI_VECTORS);
14bd3265 377 map = &icu_irqmaps[0][irq];
16437a92 378
2a32da90 379 KKASSERT(map->im_type == ICU_IMT_LEGACY);
16437a92
SZ
380
381 /* TODO: Check whether it is configured or not */
86d692fe 382 map->im_flags |= ICU_IMF_CONF;
16437a92
SZ
383
384 if (trig == map->im_trig)
385 return;
386
387 if (bootverbose) {
388 kprintf("ICU: irq %d, %s -> %s\n", irq,
389 intr_str_trigger(map->im_trig),
390 intr_str_trigger(trig));
391 }
392 map->im_trig = trig;
393
394 if (!elcr_found) {
395 if (bootverbose)
396 kprintf("ICU: no ELCR, skip irq %d config\n", irq);
397 return;
398 }
399 elcr_write_trigger(irq, map->im_trig);
d1ae7328 400}
a05c798c
SZ
401
402static int
bec969af 403icu_abi_legacy_intr_cpuid(int irq __unused)
a05c798c
SZ
404{
405 return 0;
406}
14bd3265
SZ
407
408static void
409icu_abi_rman_setup(struct rman *rm)
410{
411 int start, end, i;
412
413 KASSERT(rm->rm_cpuid >= 0 && rm->rm_cpuid < MAXCPU,
414 ("invalid rman cpuid %d", rm->rm_cpuid));
415
416 start = end = -1;
417 for (i = 0; i < IDT_HWI_VECTORS; ++i) {
418 const struct icu_irqmap *map = &icu_irqmaps[rm->rm_cpuid][i];
419
420 if (start < 0) {
421 if (ICU_IMT_ISHWI(map))
422 start = end = i;
423 } else {
424 if (ICU_IMT_ISHWI(map)) {
425 end = i;
426 } else {
427 KKASSERT(end >= 0);
428 if (bootverbose) {
429 kprintf("ICU: rman cpu%d %d - %d\n",
430 rm->rm_cpuid, start, end);
431 }
432 if (rman_manage_region(rm, start, end)) {
433 panic("rman_manage_region"
434 "(cpu%d %d - %d)", rm->rm_cpuid,
435 start, end);
436 }
437 start = end = -1;
438 }
439 }
440 }
441 if (start >= 0) {
442 KKASSERT(end >= 0);
443 if (bootverbose) {
444 kprintf("ICU: rman cpu%d %d - %d\n",
445 rm->rm_cpuid, start, end);
446 }
447 if (rman_manage_region(rm, start, end)) {
448 panic("rman_manage_region(cpu%d %d - %d)",
449 rm->rm_cpuid, start, end);
450 }
451 }
452}
2a39d30f
SZ
453
454static int
6bf6f751
SZ
455icu_abi_msi_alloc_intern(int type, const char *desc,
456 int intrs[], int count, int cpuid)
2a39d30f
SZ
457{
458 int i, error;
459
460 KASSERT(cpuid >= 0 && cpuid < ncpus,
461 ("invalid cpuid %d", cpuid));
462
ed20d0e3 463 KASSERT(count > 0 && count <= 32, ("invalid count %d", count));
2a39d30f 464 KASSERT((count & (count - 1)) == 0,
ed20d0e3 465 ("count %d is not power of 2", count));
2a39d30f
SZ
466
467 lwkt_gettoken(&icu_irqmap_tok);
468
469 /*
470 * NOTE:
471 * Since IDT_OFFSET is 32, which is the maximum valid 'count',
472 * we do not need to find out the first properly aligned
473 * interrupt vector.
474 */
475
476 error = EMSGSIZE;
477 for (i = icu_abi_msi_start; i < IDT_HWI_VECTORS; i += count) {
478 int j;
479
480 if (icu_irqmaps[cpuid][i].im_type != ICU_IMT_UNUSED)
481 continue;
482
483 for (j = 1; j < count; ++j) {
484 if (icu_irqmaps[cpuid][i + j].im_type != ICU_IMT_UNUSED)
485 break;
486 }
487 if (j != count)
488 continue;
489
490 for (j = 0; j < count; ++j) {
491 int intr = i + j, cpu;
492
493 for (cpu = 0; cpu < ncpus; ++cpu) {
494 struct icu_irqmap *map;
495
496 map = &icu_irqmaps[cpu][intr];
497 KASSERT(map->im_msi_base < 0,
ed20d0e3 498 ("intr %d cpu%d, stale %s-base %d",
6bf6f751 499 intr, cpu, desc, map->im_msi_base));
2a39d30f 500 KASSERT(map->im_type == ICU_IMT_UNUSED,
ed20d0e3 501 ("intr %d cpu%d, already allocated",
2a39d30f
SZ
502 intr, cpu));
503
504 if (cpu == cpuid) {
6bf6f751 505 map->im_type = type;
2a39d30f
SZ
506 map->im_msi_base = i;
507 } else {
508 map->im_type = ICU_IMT_SHADOW;
509 }
510 }
511
512 intrs[j] = intr;
513 msi_setup(intr);
514
515 if (bootverbose) {
6bf6f751
SZ
516 kprintf("alloc %s intr %d on cpu%d\n",
517 desc, intr, cpuid);
2a39d30f
SZ
518 }
519 }
520 error = 0;
521 break;
522 }
523
524 lwkt_reltoken(&icu_irqmap_tok);
525
526 return error;
527}
528
529static void
6bf6f751
SZ
530icu_abi_msi_release_intern(int type, const char *desc,
531 const int intrs[], int count, int cpuid)
2a39d30f
SZ
532{
533 int i, msi_base = -1, intr_next = -1, mask;
534
535 KASSERT(cpuid >= 0 && cpuid < ncpus,
536 ("invalid cpuid %d", cpuid));
537
ed20d0e3 538 KASSERT(count > 0 && count <= 32, ("invalid count %d", count));
2a39d30f
SZ
539
540 mask = count - 1;
ed20d0e3 541 KASSERT((count & mask) == 0, ("count %d is not power of 2", count));
2a39d30f
SZ
542
543 lwkt_gettoken(&icu_irqmap_tok);
544
545 for (i = 0; i < count; ++i) {
546 int intr = intrs[i], cpu;
547
548 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
ed20d0e3 549 ("invalid intr %d", intr));
2a39d30f
SZ
550
551 for (cpu = 0; cpu < ncpus; ++cpu) {
552 struct icu_irqmap *map;
553
554 map = &icu_irqmaps[cpu][intr];
555
556 if (cpu == cpuid) {
6bf6f751 557 KASSERT(map->im_type == type,
ed20d0e3
SW
558 ("trying to release non-%s intr %d cpu%d, "
559 "type %d", desc, intr, cpu,
6bf6f751 560 map->im_type));
2a39d30f
SZ
561 KASSERT(map->im_msi_base >= 0 &&
562 map->im_msi_base <= intr,
ed20d0e3 563 ("intr %d cpu%d, invalid %s-base %d",
6bf6f751 564 intr, cpu, desc, map->im_msi_base));
2a39d30f 565 KASSERT((map->im_msi_base & mask) == 0,
6bf6f751 566 ("intr %d cpu%d, %s-base %d is "
ed20d0e3 567 "not properly aligned %d",
6bf6f751 568 intr, cpu, desc, map->im_msi_base, count));
2a39d30f
SZ
569
570 if (msi_base < 0) {
571 msi_base = map->im_msi_base;
572 } else {
573 KASSERT(map->im_msi_base == msi_base,
574 ("intr %d cpu%d, "
6bf6f751 575 "inconsistent %s-base, "
ed20d0e3 576 "was %d, now %d",
6bf6f751 577 intr, cpu, desc,
2a39d30f
SZ
578 msi_base, map->im_msi_base));
579 }
580 map->im_msi_base = -1;
581 } else {
582 KASSERT(map->im_type == ICU_IMT_SHADOW,
ed20d0e3
SW
583 ("trying to release non-%ssh intr %d cpu%d, "
584 "type %d", desc, intr, cpu,
6bf6f751 585 map->im_type));
2a39d30f 586 KASSERT(map->im_msi_base < 0,
ed20d0e3 587 ("intr %d cpu%d, invalid %ssh-base %d",
6bf6f751 588 intr, cpu, desc, map->im_msi_base));
2a39d30f
SZ
589 }
590 map->im_type = ICU_IMT_UNUSED;
591 }
592
593 if (intr_next < intr)
594 intr_next = intr;
595
6bf6f751
SZ
596 if (bootverbose) {
597 kprintf("release %s intr %d on cpu%d\n",
598 desc, intr, cpuid);
599 }
2a39d30f
SZ
600 }
601
602 KKASSERT(intr_next > 0);
603 KKASSERT(msi_base >= 0);
604
605 ++intr_next;
606 if (intr_next < IDT_HWI_VECTORS) {
607 int cpu;
608
609 for (cpu = 0; cpu < ncpus; ++cpu) {
610 const struct icu_irqmap *map =
611 &icu_irqmaps[cpu][intr_next];
612
6bf6f751 613 if (map->im_type == type) {
2a39d30f 614 KASSERT(map->im_msi_base != msi_base,
ed20d0e3 615 ("more than %d %s was allocated",
6bf6f751 616 count, desc));
2a39d30f
SZ
617 }
618 }
619 }
620
621 lwkt_reltoken(&icu_irqmap_tok);
622}
623
6bf6f751
SZ
624static int
625icu_abi_msi_alloc(int intrs[], int count, int cpuid)
626{
627 return icu_abi_msi_alloc_intern(ICU_IMT_MSI, "MSI",
628 intrs, count, cpuid);
629}
630
631static void
632icu_abi_msi_release(const int intrs[], int count, int cpuid)
633{
634 icu_abi_msi_release_intern(ICU_IMT_MSI, "MSI",
635 intrs, count, cpuid);
636}
637
638static int
639icu_abi_msix_alloc(int *intr, int cpuid)
640{
641 return icu_abi_msi_alloc_intern(ICU_IMT_MSIX, "MSI-X",
642 intr, 1, cpuid);
643}
644
645static void
646icu_abi_msix_release(int intr, int cpuid)
647{
648 icu_abi_msi_release_intern(ICU_IMT_MSIX, "MSI-X",
649 &intr, 1, cpuid);
650}
651
2a39d30f
SZ
652static void
653icu_abi_msi_map(int intr, uint64_t *addr, uint32_t *data, int cpuid)
654{
655 const struct icu_irqmap *map;
656
657 KASSERT(cpuid >= 0 && cpuid < ncpus,
658 ("invalid cpuid %d", cpuid));
659
660 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
ed20d0e3 661 ("invalid intr %d", intr));
2a39d30f
SZ
662
663 lwkt_gettoken(&icu_irqmap_tok);
664
665 map = &icu_irqmaps[cpuid][intr];
6bf6f751
SZ
666 KASSERT(map->im_type == ICU_IMT_MSI ||
667 map->im_type == ICU_IMT_MSIX,
ed20d0e3 668 ("trying to map non-MSI/MSI-X intr %d, type %d", intr, map->im_type));
2a39d30f 669 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr,
ed20d0e3 670 ("intr %d, invalid %s-base %d", intr,
6bf6f751
SZ
671 map->im_type == ICU_IMT_MSI ? "MSI" : "MSI-X",
672 map->im_msi_base));
2a39d30f
SZ
673
674 msi_map(map->im_msi_base, addr, data, cpuid);
675
6bf6f751
SZ
676 if (bootverbose) {
677 kprintf("map %s intr %d on cpu%d\n",
678 map->im_type == ICU_IMT_MSI ? "MSI" : "MSI-X",
679 intr, cpuid);
680 }
2a39d30f
SZ
681
682 lwkt_reltoken(&icu_irqmap_tok);
683}
86d692fe
SZ
684
685static int
686icu_abi_legacy_intr_find(int irq, enum intr_trigger trig,
687 enum intr_polarity pola __unused)
688{
689 const struct icu_irqmap *map;
690
691#ifdef INVARIANTS
692 if (trig == INTR_TRIGGER_CONFORM) {
693 KKASSERT(pola == INTR_POLARITY_CONFORM);
694 } else {
695 KKASSERT(trig == INTR_TRIGGER_EDGE ||
696 trig == INTR_TRIGGER_LEVEL);
697 KKASSERT(pola == INTR_POLARITY_HIGH ||
698 pola == INTR_POLARITY_LOW);
699 }
700#endif
701
702 if (irq < 0 || irq >= ICU_HWI_VECTORS)
703 return -1;
704
705 map = &icu_irqmaps[0][irq];
706 if (map->im_type == ICU_IMT_LEGACY) {
707 if ((map->im_flags & ICU_IMF_CONF) &&
708 trig != INTR_TRIGGER_CONFORM) {
709 if (map->im_trig != trig)
710 return -1;
711 }
712 return irq;
713 }
714 return -1;
715}
716
717static int
718icu_abi_legacy_intr_find_bygsi(int gsi, enum intr_trigger trig,
719 enum intr_polarity pola)
720{
721 /* GSI and IRQ has 1:1 mapping */
722 return icu_abi_legacy_intr_find(gsi, trig, pola);
723}