[dragonfly.git] / sys / platform / pc64 / icu / icu_abi.c

/*
 * Copyright (c) 1991 The Regents of the University of California.
 * Copyright (c) 2005,2008 The DragonFly Project.
 * All rights reserved.
 * 
 * This code is derived from software contributed to The DragonFly Project
 * by Matthew Dillon <dillon@backplane.com>
 *
 * This code is derived from software contributed to Berkeley by
 * William Jolitz.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name of The DragonFly Project nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific, prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 * 
 * $DragonFly: src/sys/platform/pc64/icu/icu_abi.c,v 1.1 2008/08/29 17:07:16 dillon Exp $
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/machintr.h>
#include <sys/interrupt.h>
#include <sys/rman.h>
#include <sys/bus.h>

#include <machine/segments.h>
#include <machine/md_var.h>
#include <machine/intr_machdep.h>
#include <machine/globaldata.h>
#include <machine/smp.h>
#include <machine/msi_var.h>

#include <sys/thread2.h>

#include <machine_base/icu/elcr_var.h>

#include <machine_base/icu/icu.h>
#include <machine_base/icu/icu_ipl.h>
#include <machine_base/apic/ioapic.h>

extern inthand_t
	IDTVEC(icu_intr0),	IDTVEC(icu_intr1),
	IDTVEC(icu_intr2),	IDTVEC(icu_intr3),
	IDTVEC(icu_intr4),	IDTVEC(icu_intr5),
	IDTVEC(icu_intr6),	IDTVEC(icu_intr7),
	IDTVEC(icu_intr8),	IDTVEC(icu_intr9),
	IDTVEC(icu_intr10),	IDTVEC(icu_intr11),
	IDTVEC(icu_intr12),	IDTVEC(icu_intr13),
	IDTVEC(icu_intr14),	IDTVEC(icu_intr15);

static inthand_t *icu_intr[ICU_HWI_VECTORS] = {
	&IDTVEC(icu_intr0),	&IDTVEC(icu_intr1),
	&IDTVEC(icu_intr2),	&IDTVEC(icu_intr3),
	&IDTVEC(icu_intr4),	&IDTVEC(icu_intr5),
	&IDTVEC(icu_intr6),	&IDTVEC(icu_intr7),
	&IDTVEC(icu_intr8),	&IDTVEC(icu_intr9),
	&IDTVEC(icu_intr10),	&IDTVEC(icu_intr11),
	&IDTVEC(icu_intr12),	&IDTVEC(icu_intr13),
	&IDTVEC(icu_intr14),	&IDTVEC(icu_intr15)
};

static struct icu_irqmap {
	int			im_type;	/* ICU_IMT_ */
	enum intr_trigger	im_trig;
	int			im_msi_base;
} icu_irqmaps[MAXCPU][IDT_HWI_VECTORS];

static struct lwkt_token icu_irqmap_tok =
	LWKT_TOKEN_INITIALIZER(icu_irqmap_token);

#define ICU_IMT_UNUSED		0	/* KEEP THIS */
#define ICU_IMT_RESERVED	1
#define ICU_IMT_LINE		2
#define ICU_IMT_SYSCALL		3
#define ICU_IMT_MSI		4

#define ICU_IMT_ISHWI(map)	((map)->im_type != ICU_IMT_RESERVED && \
				 (map)->im_type != ICU_IMT_SYSCALL)

extern void	ICU_INTREN(int);
extern void	ICU_INTRDIS(int);

extern int	imcr_present;

static void	icu_abi_intr_enable(int);
static void	icu_abi_intr_disable(int);
static void	icu_abi_intr_setup(int, int);
static void	icu_abi_intr_teardown(int);
static void	icu_abi_intr_config(int, enum intr_trigger, enum intr_polarity);
static int	icu_abi_intr_cpuid(int);

static int	icu_abi_msi_alloc(int [], int, int);
static void	icu_abi_msi_release(const int [], int, int);
static void	icu_abi_msi_map(int, uint64_t *, uint32_t *, int);

static void	icu_abi_finalize(void);
static void	icu_abi_cleanup(void);
static void	icu_abi_setdefault(void);
static void	icu_abi_stabilize(void);
static void	icu_abi_initmap(void);
static void	icu_abi_rman_setup(struct rman *);

struct machintr_abi MachIntrABI_ICU = {
	MACHINTR_ICU,
	.intr_disable	= icu_abi_intr_disable,
	.intr_enable	= icu_abi_intr_enable,
	.intr_setup	= icu_abi_intr_setup,
	.intr_teardown	= icu_abi_intr_teardown,
	.intr_config	= icu_abi_intr_config,
	.intr_cpuid	= icu_abi_intr_cpuid,

	.msi_alloc	= icu_abi_msi_alloc,
	.msi_release	= icu_abi_msi_release,
	.msi_map	= icu_abi_msi_map,

	.finalize	= icu_abi_finalize,
	.cleanup	= icu_abi_cleanup,
	.setdefault	= icu_abi_setdefault,
	.stabilize	= icu_abi_stabilize,
	.initmap	= icu_abi_initmap,
	.rman_setup	= icu_abi_rman_setup
};

static int	icu_abi_msi_start;	/* NOTE: for testing only */

/*
 * WARNING!  SMP builds can use the ICU now so this code must be MP safe.
 */

static void
icu_abi_intr_enable(int irq)
{
	const struct icu_irqmap *map;

	KASSERT(irq >= 0 && irq < IDT_HWI_VECTORS,
	    ("icu enable, invalid irq %d\n", irq));

	map = &icu_irqmaps[mycpuid][irq];
	KASSERT(ICU_IMT_ISHWI(map),
	    ("icu enable, not hwi irq %d, type %d, cpu%d\n",
	     irq, map->im_type, mycpuid));
	if (map->im_type != ICU_IMT_LINE)
		return;

	ICU_INTREN(irq);
}

static void
icu_abi_intr_disable(int irq)
{
	const struct icu_irqmap *map;

	KASSERT(irq >= 0 && irq < IDT_HWI_VECTORS,
	    ("icu disable, invalid irq %d\n", irq));

	map = &icu_irqmaps[mycpuid][irq];
	KASSERT(ICU_IMT_ISHWI(map),
	    ("icu disable, not hwi irq %d, type %d, cpu%d\n",
	     irq, map->im_type, mycpuid));
	if (map->im_type != ICU_IMT_LINE)
		return;

	ICU_INTRDIS(irq);
}

/*
 * Called before interrupts are physically enabled
 */
static void
icu_abi_stabilize(void)
{
	int intr;

	for (intr = 0; intr < ICU_HWI_VECTORS; ++intr)
		ICU_INTRDIS(intr);
	ICU_INTREN(ICU_IRQ_SLAVE);
}

/*
 * Called after interrupts physically enabled but before the
 * critical section is released.
 */
static void
icu_abi_cleanup(void)
{
	bzero(mdcpu->gd_ipending, sizeof(mdcpu->gd_ipending));
}

/*
 * Called after stablize and cleanup; critical section is not
 * held and interrupts are not physically disabled.
 */
static void
icu_abi_finalize(void)
{
	KKASSERT(MachIntrABI.type == MACHINTR_ICU);
	KKASSERT(!ioapic_enable);

	/*
	 * If an IMCR is present, programming bit 0 disconnects the 8259
	 * from the BSP.  The 8259 may still be connected to LINT0 on the
	 * BSP's LAPIC.
	 *
	 * If we are running SMP the LAPIC is active, try to use virtual
	 * wire mode so we can use other interrupt sources within the LAPIC
	 * in addition to the 8259.
	 */
	if (imcr_present) {
		outb(0x22, 0x70);
		outb(0x23, 0x01);
	}
}

static void
icu_abi_intr_setup(int intr, int flags)
{
	const struct icu_irqmap *map;
	register_t ef;

	KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
	    ("icu setup, invalid irq %d\n", intr));

	map = &icu_irqmaps[mycpuid][intr];
	KASSERT(ICU_IMT_ISHWI(map),
	    ("icu setup, not hwi irq %d, type %d, cpu%d\n",
	     intr, map->im_type, mycpuid));
	if (map->im_type != ICU_IMT_LINE)
		return;

	ef = read_rflags();
	cpu_disable_intr();

	ICU_INTREN(intr);

	write_rflags(ef);
}

static void
icu_abi_intr_teardown(int intr)
{
	const struct icu_irqmap *map;
	register_t ef;

	KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
	    ("icu teardown, invalid irq %d\n", intr));

	map = &icu_irqmaps[mycpuid][intr];
	KASSERT(ICU_IMT_ISHWI(map),
	    ("icu teardown, not hwi irq %d, type %d, cpu%d\n",
	     intr, map->im_type, mycpuid));
	if (map->im_type != ICU_IMT_LINE)
		return;

	ef = read_rflags();
	cpu_disable_intr();

	ICU_INTRDIS(intr);

	write_rflags(ef);
}

static void
icu_abi_setdefault(void)
{
	int intr;

	for (intr = 0; intr < ICU_HWI_VECTORS; ++intr) {
		if (intr == ICU_IRQ_SLAVE)
			continue;
		setidt_global(IDT_OFFSET + intr, icu_intr[intr],
		    SDT_SYSIGT, SEL_KPL, 0);
	}
}

static void
icu_abi_initmap(void)
{
	int cpu;

	kgetenv_int("hw.icu.msi_start", &icu_abi_msi_start);
	icu_abi_msi_start &= ~0x1f;	/* MUST be 32 aligned */

	/*
	 * NOTE: ncpus is not ready yet
	 */
	for (cpu = 0; cpu < MAXCPU; ++cpu) {
		int i;

		if (cpu != 0) {
			for (i = 0; i < ICU_HWI_VECTORS; ++i)
				icu_irqmaps[cpu][i].im_type = ICU_IMT_RESERVED;
		} else {
			for (i = 0; i < ICU_HWI_VECTORS; ++i)
				icu_irqmaps[cpu][i].im_type = ICU_IMT_LINE;
			icu_irqmaps[cpu][ICU_IRQ_SLAVE].im_type =
			    ICU_IMT_RESERVED;

			if (elcr_found) {
				for (i = 0; i < ICU_HWI_VECTORS; ++i) {
					icu_irqmaps[cpu][i].im_trig =
					    elcr_read_trigger(i);
				}
			} else {
				/*
				 * NOTE: Trigger mode does not matter at all
				 */
				for (i = 0; i < ICU_HWI_VECTORS; ++i) {
					icu_irqmaps[cpu][i].im_trig =
					    INTR_TRIGGER_EDGE;
				}
			}
		}

		for (i = 0; i < IDT_HWI_VECTORS; ++i)
			icu_irqmaps[cpu][i].im_msi_base = -1;

		icu_irqmaps[cpu][IDT_OFFSET_SYSCALL - IDT_OFFSET].im_type =
		    ICU_IMT_SYSCALL;
	}
}

static void
icu_abi_intr_config(int irq, enum intr_trigger trig,
    enum intr_polarity pola __unused)
{
	struct icu_irqmap *map;

	KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL);

	KKASSERT(irq >= 0 && irq < IDT_HWI_VECTORS);
	map = &icu_irqmaps[0][irq];

	KKASSERT(map->im_type == ICU_IMT_LINE);

	/* TODO: Check whether it is configured or not */

	if (trig == map->im_trig)
		return;

	if (bootverbose) {
		kprintf("ICU: irq %d, %s -> %s\n", irq,
			intr_str_trigger(map->im_trig),
			intr_str_trigger(trig));
	}
	map->im_trig = trig;

	if (!elcr_found) {
		if (bootverbose)
			kprintf("ICU: no ELCR, skip irq %d config\n", irq);
		return;
	}
	elcr_write_trigger(irq, map->im_trig);
}

static int
icu_abi_intr_cpuid(int irq __unused)
{
	return 0;
}

static void
icu_abi_rman_setup(struct rman *rm)
{
	int start, end, i;

	KASSERT(rm->rm_cpuid >= 0 && rm->rm_cpuid < MAXCPU,
	    ("invalid rman cpuid %d", rm->rm_cpuid));

	start = end = -1;
	for (i = 0; i < IDT_HWI_VECTORS; ++i) {
		const struct icu_irqmap *map = &icu_irqmaps[rm->rm_cpuid][i];

		if (start < 0) {
			if (ICU_IMT_ISHWI(map))
				start = end = i;
		} else {
			if (ICU_IMT_ISHWI(map)) {
				end = i;
			} else {
				KKASSERT(end >= 0);
				if (bootverbose) {
					kprintf("ICU: rman cpu%d %d - %d\n",
					    rm->rm_cpuid, start, end);
				}
				if (rman_manage_region(rm, start, end)) {
					panic("rman_manage_region"
					    "(cpu%d %d - %d)", rm->rm_cpuid,
					    start, end);
				}
				start = end = -1;
			}
		}
	}
	if (start >= 0) {
		KKASSERT(end >= 0);
		if (bootverbose) {
			kprintf("ICU: rman cpu%d %d - %d\n",
			    rm->rm_cpuid, start, end);
		}
		if (rman_manage_region(rm, start, end)) {
			panic("rman_manage_region(cpu%d %d - %d)",
			    rm->rm_cpuid, start, end);
		}
	}
}

static int
icu_abi_msi_alloc(int intrs[], int count, int cpuid)
{
	int i, error;

	KASSERT(cpuid >= 0 && cpuid < ncpus,
	    ("invalid cpuid %d", cpuid));

	KASSERT(count > 0 && count <= 32, ("invalid count %d\n", count));
	KASSERT((count & (count - 1)) == 0,
	    ("count %d is not power of 2\n", count));

	lwkt_gettoken(&icu_irqmap_tok);

	/*
	 * NOTE:
	 * Since IDT_OFFSET is 32, which is the maximum valid 'count',
	 * we do not need to find out the first properly aligned
	 * interrupt vector.
	 */

	error = EMSGSIZE;
	for (i = icu_abi_msi_start; i < IDT_HWI_VECTORS; i += count) {
		int j;

		if (icu_irqmaps[cpuid][i].im_type != ICU_IMT_UNUSED)
			continue;

		for (j = 1; j < count; ++j) {
			if (icu_irqmaps[cpuid][i + j].im_type != ICU_IMT_UNUSED)
				break;
		}
		if (j != count)
			continue;

		for (j = 0; j < count; ++j) {
			struct icu_irqmap *map;
			int intr = i + j;

			map = &icu_irqmaps[cpuid][intr];
			KASSERT(map->im_msi_base < 0,
			    ("intr %d, stale MSI-base %d\n",
			     intr, map->im_msi_base));

			map->im_type = ICU_IMT_MSI;
			map->im_msi_base = i;

			intrs[j] = intr;
			msi_setup(intr, cpuid);

			if (bootverbose) {
				kprintf("alloc MSI intr %d on cpu%d\n",
				    intr, cpuid);
			}

		}
		error = 0;
		break;
	}

	lwkt_reltoken(&icu_irqmap_tok);

	return error;
}

static void
icu_abi_msi_release(const int intrs[], int count, int cpuid)
{
	int i, msi_base = -1, intr_next = -1, mask;

	KASSERT(cpuid >= 0 && cpuid < ncpus,
	    ("invalid cpuid %d", cpuid));

	KASSERT(count > 0 && count <= 32, ("invalid count %d\n", count));

	mask = count - 1;
	KASSERT((count & mask) == 0, ("count %d is not power of 2\n", count));

	lwkt_gettoken(&icu_irqmap_tok);

	for (i = 0; i < count; ++i) {
		struct icu_irqmap *map;
		int intr = intrs[i];

		KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
		    ("invalid intr %d\n", intr));

		map = &icu_irqmaps[cpuid][intr];
		KASSERT(map->im_type == ICU_IMT_MSI,
		    ("try release non-MSI intr %d, type %d\n",
		     intr, map->im_type));
		KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr,
		    ("intr %d, invalid MSI-base %d\n", intr, map->im_msi_base));
		KASSERT((map->im_msi_base & mask) == 0,
		    ("intr %d, MSI-base %d is not proper aligned %d\n",
		     intr, map->im_msi_base, count));

		if (msi_base < 0) {
			msi_base = map->im_msi_base;
		} else {
			KASSERT(map->im_msi_base == msi_base,
			    ("intr %d, inconsistent MSI-base, "
			     "was %d, now %d\n",
			     intr, msi_base, map->im_msi_base));
		}

		if (intr_next < intr)
			intr_next = intr;

		map->im_type = ICU_IMT_UNUSED;
		map->im_msi_base = -1;

		if (bootverbose)
			kprintf("release MSI intr %d on cpu%d\n", intr, cpuid);
	}

	KKASSERT(intr_next > 0);
	KKASSERT(msi_base >= 0);

	++intr_next;
	if (intr_next < IDT_HWI_VECTORS) {
		const struct icu_irqmap *map = &icu_irqmaps[cpuid][intr_next];

		if (map->im_type == ICU_IMT_MSI) {
			KASSERT(map->im_msi_base != msi_base,
			    ("more than %d MSI was allocated\n", count));
		}
	}

	lwkt_reltoken(&icu_irqmap_tok);
}

static void
icu_abi_msi_map(int intr, uint64_t *addr, uint32_t *data, int cpuid)
{
	const struct icu_irqmap *map;

	KASSERT(cpuid >= 0 && cpuid < ncpus,
	    ("invalid cpuid %d", cpuid));

	KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
	    ("invalid intr %d\n", intr));

	lwkt_gettoken(&icu_irqmap_tok);

	map = &icu_irqmaps[cpuid][intr];
	KASSERT(map->im_type == ICU_IMT_MSI,
	    ("try map non-MSI intr %d, type %d\n", intr, map->im_type));
	KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr,
	    ("intr %d, invalid MSI-base %d\n", intr, map->im_msi_base));

	msi_map(map->im_msi_base, addr, data, cpuid);

	if (bootverbose)
		kprintf("map MSI intr %d on cpu%d\n", intr, cpuid);

	lwkt_reltoken(&icu_irqmap_tok);
}
Commit	Line	Data
c8fe38ae MD	1	/*
	2	* Copyright (c) 1991 The Regents of the University of California.
	3	* Copyright (c) 2005,2008 The DragonFly Project.
	4	* All rights reserved.
	5	*
	6	* This code is derived from software contributed to The DragonFly Project
	7	* by Matthew Dillon <dillon@backplane.com>
	8	*
	9	* This code is derived from software contributed to Berkeley by
	10	* William Jolitz.
	11	*
	12	* Redistribution and use in source and binary forms, with or without
	13	* modification, are permitted provided that the following conditions
	14	* are met:
	15	*
	16	* 1. Redistributions of source code must retain the above copyright
	17	* notice, this list of conditions and the following disclaimer.
	18	* 2. Redistributions in binary form must reproduce the above copyright
	19	* notice, this list of conditions and the following disclaimer in
	20	* the documentation and/or other materials provided with the
	21	* distribution.
	22	* 3. Neither the name of The DragonFly Project nor the names of its
	23	* contributors may be used to endorse or promote products derived
	24	* from this software without specific, prior written permission.
	25	*
	26	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	27	* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	28	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
	29	* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
	30	* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
	31	* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
	32	* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	33	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	34	* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	35	* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
	36	* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	37	* SUCH DAMAGE.
	38	*
	39	* $DragonFly: src/sys/platform/pc64/icu/icu_abi.c,v 1.1 2008/08/29 17:07:16 dillon Exp $
	40	*/
	41
	42	#include <sys/param.h>
	43	#include <sys/systm.h>
	44	#include <sys/kernel.h>
	45	#include <sys/machintr.h>
	46	#include <sys/interrupt.h>
b3cf468e	47	#include <sys/rman.h>
c8fe38ae MD	48	#include <sys/bus.h>
	49
	50	#include <machine/segments.h>
	51	#include <machine/md_var.h>
57a9c56b	52	#include <machine/intr_machdep.h>
c8fe38ae	53	#include <machine/globaldata.h>
10db3cc6	54	#include <machine/smp.h>
759fc533	55	#include <machine/msi_var.h>
c8fe38ae MD	56
	57	#include <sys/thread2.h>
	58
7265a4fe	59	#include <machine_base/icu/elcr_var.h>
9e0e3f85	60
7265a4fe SZ	61	#include <machine_base/icu/icu.h>
7265a4fe SZ	62	#include <machine_base/icu/icu_ipl.h>
ed4d621d	63	#include <machine_base/apic/ioapic.h>
c8fe38ae	64
c8fe38ae	65	extern inthand_t
35e45e47 SZ	66	IDTVEC(icu_intr0), IDTVEC(icu_intr1),
	67	IDTVEC(icu_intr2), IDTVEC(icu_intr3),
	68	IDTVEC(icu_intr4), IDTVEC(icu_intr5),
	69	IDTVEC(icu_intr6), IDTVEC(icu_intr7),
	70	IDTVEC(icu_intr8), IDTVEC(icu_intr9),
	71	IDTVEC(icu_intr10), IDTVEC(icu_intr11),
	72	IDTVEC(icu_intr12), IDTVEC(icu_intr13),
	73	IDTVEC(icu_intr14), IDTVEC(icu_intr15);
c8fe38ae	74
35e45e47 SZ	75	static inthand_t *icu_intr[ICU_HWI_VECTORS] = {
	76	&IDTVEC(icu_intr0), &IDTVEC(icu_intr1),
	77	&IDTVEC(icu_intr2), &IDTVEC(icu_intr3),
	78	&IDTVEC(icu_intr4), &IDTVEC(icu_intr5),
	79	&IDTVEC(icu_intr6), &IDTVEC(icu_intr7),
	80	&IDTVEC(icu_intr8), &IDTVEC(icu_intr9),
	81	&IDTVEC(icu_intr10), &IDTVEC(icu_intr11),
	82	&IDTVEC(icu_intr12), &IDTVEC(icu_intr13),
	83	&IDTVEC(icu_intr14), &IDTVEC(icu_intr15)
c8fe38ae MD	84	};
c8fe38ae MD	85
a3dd9120 SZ	86	static struct icu_irqmap {
	87	int im_type; /* ICU_IMT_ */
	88	enum intr_trigger im_trig;
759fc533	89	int im_msi_base;
b3cf468e	90	} icu_irqmaps[MAXCPU][IDT_HWI_VECTORS];
a3dd9120	91
759fc533 SZ	92	static struct lwkt_token icu_irqmap_tok =
	93	LWKT_TOKEN_INITIALIZER(icu_irqmap_token);
	94
a3dd9120 SZ	95	#define ICU_IMT_UNUSED 0 /* KEEP THIS */
	96	#define ICU_IMT_RESERVED 1
	97	#define ICU_IMT_LINE 2
474ba684	98	#define ICU_IMT_SYSCALL 3
759fc533	99	#define ICU_IMT_MSI 4
a3dd9120	100
b3cf468e SZ	101	#define ICU_IMT_ISHWI(map) ((map)->im_type != ICU_IMT_RESERVED && \
	102	(map)->im_type != ICU_IMT_SYSCALL)
	103
10db3cc6 SZ	104	extern void ICU_INTREN(int);
	105	extern void ICU_INTRDIS(int);
	106
85bcaa51 SZ	107	extern int imcr_present;
85bcaa51 SZ	108
1967234a SZ	109	static void icu_abi_intr_enable(int);
1967234a SZ	110	static void icu_abi_intr_disable(int);
f416026e SZ	111	static void icu_abi_intr_setup(int, int);
f416026e SZ	112	static void icu_abi_intr_teardown(int);
aea76754	113	static void icu_abi_intr_config(int, enum intr_trigger, enum intr_polarity);
a05c798c	114	static int icu_abi_intr_cpuid(int);
aea76754	115
759fc533 SZ	116	static int icu_abi_msi_alloc(int [], int, int);
	117	static void icu_abi_msi_release(const int [], int, int);
	118	static void icu_abi_msi_map(int, uint64_t , uint32_t , int);
	119
aea76754 SZ	120	static void icu_abi_finalize(void);
	121	static void icu_abi_cleanup(void);
	122	static void icu_abi_setdefault(void);
	123	static void icu_abi_stabilize(void);
	124	static void icu_abi_initmap(void);
b3cf468e	125	static void icu_abi_rman_setup(struct rman *);
10db3cc6	126
faaf4131	127	struct machintr_abi MachIntrABI_ICU = {
339478ac	128	MACHINTR_ICU,
1967234a SZ	129	.intr_disable = icu_abi_intr_disable,
1967234a SZ	130	.intr_enable = icu_abi_intr_enable,
f416026e SZ	131	.intr_setup = icu_abi_intr_setup,
f416026e SZ	132	.intr_teardown = icu_abi_intr_teardown,
aea76754	133	.intr_config = icu_abi_intr_config,
a05c798c	134	.intr_cpuid = icu_abi_intr_cpuid,
35b2edcb	135
759fc533 SZ	136	.msi_alloc = icu_abi_msi_alloc,
	137	.msi_release = icu_abi_msi_release,
	138	.msi_map = icu_abi_msi_map,
	139
aea76754 SZ	140	.finalize = icu_abi_finalize,
	141	.cleanup = icu_abi_cleanup,
	142	.setdefault = icu_abi_setdefault,
	143	.stabilize = icu_abi_stabilize,
b3cf468e SZ	144	.initmap = icu_abi_initmap,
b3cf468e SZ	145	.rman_setup = icu_abi_rman_setup
c8fe38ae MD	146	};
c8fe38ae MD	147
759fc533 SZ	148	static int icu_abi_msi_start; /* NOTE: for testing only */
759fc533 SZ	149
c8fe38ae MD	150	/*
	151	* WARNING! SMP builds can use the ICU now so this code must be MP safe.
	152	*/
c8fe38ae	153
1967234a SZ	154	static void
	155	icu_abi_intr_enable(int irq)
	156	{
	157	const struct icu_irqmap *map;
	158
	159	KASSERT(irq >= 0 && irq < IDT_HWI_VECTORS,
	160	("icu enable, invalid irq %d\n", irq));
	161
	162	map = &icu_irqmaps[mycpuid][irq];
	163	KASSERT(ICU_IMT_ISHWI(map),
	164	("icu enable, not hwi irq %d, type %d, cpu%d\n",
	165	irq, map->im_type, mycpuid));
759fc533	166	if (map->im_type != ICU_IMT_LINE)
1967234a	167	return;
1967234a SZ	168
	169	ICU_INTREN(irq);
	170	}
	171
	172	static void
	173	icu_abi_intr_disable(int irq)
	174	{
	175	const struct icu_irqmap *map;
	176
	177	KASSERT(irq >= 0 && irq < IDT_HWI_VECTORS,
	178	("icu disable, invalid irq %d\n", irq));
	179
	180	map = &icu_irqmaps[mycpuid][irq];
	181	KASSERT(ICU_IMT_ISHWI(map),
	182	("icu disable, not hwi irq %d, type %d, cpu%d\n",
	183	irq, map->im_type, mycpuid));
759fc533	184	if (map->im_type != ICU_IMT_LINE)
1967234a	185	return;
1967234a SZ	186
	187	ICU_INTRDIS(irq);
	188	}
	189
c8fe38ae MD	190	/*
	191	* Called before interrupts are physically enabled
	192	*/
	193	static void
aea76754	194	icu_abi_stabilize(void)
c8fe38ae	195	{
339478ac SZ	196	int intr;
339478ac SZ	197
7bf5fa56	198	for (intr = 0; intr < ICU_HWI_VECTORS; ++intr)
1967234a SZ	199	ICU_INTRDIS(intr);
1967234a SZ	200	ICU_INTREN(ICU_IRQ_SLAVE);
7bf5fa56 SZ	201	}
	202
	203	/*
	204	* Called after interrupts physically enabled but before the
	205	* critical section is released.
	206	*/
	207	static void
aea76754	208	icu_abi_cleanup(void)
7bf5fa56 SZ	209	{
	210	bzero(mdcpu->gd_ipending, sizeof(mdcpu->gd_ipending));
	211	}
	212
	213	/*
	214	* Called after stablize and cleanup; critical section is not
	215	* held and interrupts are not physically disabled.
7bf5fa56 SZ	216	*/
7bf5fa56 SZ	217	static void
aea76754	218	icu_abi_finalize(void)
7bf5fa56 SZ	219	{
7bf5fa56 SZ	220	KKASSERT(MachIntrABI.type == MACHINTR_ICU);
f45bfca0	221	KKASSERT(!ioapic_enable);
339478ac	222
339478ac SZ	223	/*
	224	* If an IMCR is present, programming bit 0 disconnects the 8259
	225	* from the BSP. The 8259 may still be connected to LINT0 on the
	226	* BSP's LAPIC.
	227	*
	228	* If we are running SMP the LAPIC is active, try to use virtual
	229	* wire mode so we can use other interrupt sources within the LAPIC
	230	* in addition to the 8259.
	231	*/
9d758cc4	232	if (imcr_present) {
339478ac SZ	233	outb(0x22, 0x70);
339478ac SZ	234	outb(0x23, 0x01);
7bf5fa56	235	}
c8fe38ae MD	236	}
c8fe38ae MD	237
f416026e SZ	238	static void
f416026e SZ	239	icu_abi_intr_setup(int intr, int flags)
c8fe38ae	240	{
1967234a	241	const struct icu_irqmap *map;
339478ac SZ	242	register_t ef;
339478ac SZ	243
1967234a SZ	244	KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
	245	("icu setup, invalid irq %d\n", intr));
	246
	247	map = &icu_irqmaps[mycpuid][intr];
	248	KASSERT(ICU_IMT_ISHWI(map),
	249	("icu setup, not hwi irq %d, type %d, cpu%d\n",
	250	intr, map->im_type, mycpuid));
759fc533	251	if (map->im_type != ICU_IMT_LINE)
1967234a	252	return;
339478ac SZ	253
	254	ef = read_rflags();
	255	cpu_disable_intr();
339478ac	256
26cf64b2	257	ICU_INTREN(intr);
339478ac	258
f416026e SZ	259	write_rflags(ef);
	260	}
	261
	262	static void
	263	icu_abi_intr_teardown(int intr)
	264	{
1967234a	265	const struct icu_irqmap *map;
f416026e SZ	266	register_t ef;
f416026e SZ	267
1967234a SZ	268	KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
	269	("icu teardown, invalid irq %d\n", intr));
	270
	271	map = &icu_irqmaps[mycpuid][intr];
	272	KASSERT(ICU_IMT_ISHWI(map),
	273	("icu teardown, not hwi irq %d, type %d, cpu%d\n",
	274	intr, map->im_type, mycpuid));
759fc533	275	if (map->im_type != ICU_IMT_LINE)
1967234a	276	return;
f416026e SZ	277
	278	ef = read_rflags();
	279	cpu_disable_intr();
	280
26cf64b2	281	ICU_INTRDIS(intr);
339478ac	282
339478ac	283	write_rflags(ef);
c8fe38ae	284	}
10db3cc6 SZ	285
10db3cc6 SZ	286	static void
aea76754	287	icu_abi_setdefault(void)
10db3cc6 SZ	288	{
	289	int intr;
	290
	291	for (intr = 0; intr < ICU_HWI_VECTORS; ++intr) {
	292	if (intr == ICU_IRQ_SLAVE)
	293	continue;
8a06c6ee SZ	294	setidt_global(IDT_OFFSET + intr, icu_intr[intr],
8a06c6ee SZ	295	SDT_SYSIGT, SEL_KPL, 0);
10db3cc6 SZ	296	}
10db3cc6 SZ	297	}
a3dd9120 SZ	298
a3dd9120 SZ	299	static void
aea76754	300	icu_abi_initmap(void)
a3dd9120	301	{
b3cf468e SZ	302	int cpu;
b3cf468e SZ	303
759fc533 SZ	304	kgetenv_int("hw.icu.msi_start", &icu_abi_msi_start);
	305	icu_abi_msi_start &= ~0x1f; /* MUST be 32 aligned */
	306
b3cf468e SZ	307	/*
	308	* NOTE: ncpus is not ready yet
	309	*/
	310	for (cpu = 0; cpu < MAXCPU; ++cpu) {
	311	int i;
	312
	313	if (cpu != 0) {
	314	for (i = 0; i < ICU_HWI_VECTORS; ++i)
	315	icu_irqmaps[cpu][i].im_type = ICU_IMT_RESERVED;
	316	} else {
	317	for (i = 0; i < ICU_HWI_VECTORS; ++i)
	318	icu_irqmaps[cpu][i].im_type = ICU_IMT_LINE;
	319	icu_irqmaps[cpu][ICU_IRQ_SLAVE].im_type =
	320	ICU_IMT_RESERVED;
	321
	322	if (elcr_found) {
	323	for (i = 0; i < ICU_HWI_VECTORS; ++i) {
	324	icu_irqmaps[cpu][i].im_trig =
	325	elcr_read_trigger(i);
	326	}
	327	} else {
	328	/*
	329	* NOTE: Trigger mode does not matter at all
	330	*/
	331	for (i = 0; i < ICU_HWI_VECTORS; ++i) {
	332	icu_irqmaps[cpu][i].im_trig =
	333	INTR_TRIGGER_EDGE;
	334	}
	335	}
	336	}
759fc533 SZ	337
	338	for (i = 0; i < IDT_HWI_VECTORS; ++i)
	339	icu_irqmaps[cpu][i].im_msi_base = -1;
	340
b3cf468e SZ	341	icu_irqmaps[cpu][IDT_OFFSET_SYSCALL - IDT_OFFSET].im_type =
b3cf468e SZ	342	ICU_IMT_SYSCALL;
a3dd9120 SZ	343	}
a3dd9120 SZ	344	}
d1ae7328 SZ	345
d1ae7328 SZ	346	static void
aea76754	347	icu_abi_intr_config(int irq, enum intr_trigger trig,
d1ae7328 SZ	348	enum intr_polarity pola __unused)
d1ae7328 SZ	349	{
16437a92 SZ	350	struct icu_irqmap *map;
	351
	352	KKASSERT(trig == INTR_TRIGGER_EDGE \|\| trig == INTR_TRIGGER_LEVEL);
	353
451af8d9	354	KKASSERT(irq >= 0 && irq < IDT_HWI_VECTORS);
b3cf468e	355	map = &icu_irqmaps[0][irq];
16437a92 SZ	356
	357	KKASSERT(map->im_type == ICU_IMT_LINE);
	358
	359	/* TODO: Check whether it is configured or not */
	360
	361	if (trig == map->im_trig)
	362	return;
	363
	364	if (bootverbose) {
	365	kprintf("ICU: irq %d, %s -> %s\n", irq,
	366	intr_str_trigger(map->im_trig),
	367	intr_str_trigger(trig));
	368	}
	369	map->im_trig = trig;
	370
	371	if (!elcr_found) {
	372	if (bootverbose)
	373	kprintf("ICU: no ELCR, skip irq %d config\n", irq);
	374	return;
	375	}
	376	elcr_write_trigger(irq, map->im_trig);
d1ae7328	377	}
a05c798c SZ	378
	379	static int
	380	icu_abi_intr_cpuid(int irq __unused)
	381	{
	382	return 0;
	383	}
b3cf468e SZ	384
	385	static void
	386	icu_abi_rman_setup(struct rman *rm)
	387	{
	388	int start, end, i;
	389
	390	KASSERT(rm->rm_cpuid >= 0 && rm->rm_cpuid < MAXCPU,
	391	("invalid rman cpuid %d", rm->rm_cpuid));
	392
	393	start = end = -1;
	394	for (i = 0; i < IDT_HWI_VECTORS; ++i) {
	395	const struct icu_irqmap *map = &icu_irqmaps[rm->rm_cpuid][i];
	396
	397	if (start < 0) {
	398	if (ICU_IMT_ISHWI(map))
	399	start = end = i;
	400	} else {
	401	if (ICU_IMT_ISHWI(map)) {
	402	end = i;
	403	} else {
	404	KKASSERT(end >= 0);
	405	if (bootverbose) {
	406	kprintf("ICU: rman cpu%d %d - %d\n",
	407	rm->rm_cpuid, start, end);
	408	}
	409	if (rman_manage_region(rm, start, end)) {
	410	panic("rman_manage_region"
	411	"(cpu%d %d - %d)", rm->rm_cpuid,
	412	start, end);
	413	}
	414	start = end = -1;
	415	}
	416	}
	417	}
	418	if (start >= 0) {
	419	KKASSERT(end >= 0);
	420	if (bootverbose) {
	421	kprintf("ICU: rman cpu%d %d - %d\n",
	422	rm->rm_cpuid, start, end);
	423	}
	424	if (rman_manage_region(rm, start, end)) {
	425	panic("rman_manage_region(cpu%d %d - %d)",
	426	rm->rm_cpuid, start, end);
	427	}
	428	}
	429	}
759fc533 SZ	430
	431	static int
	432	icu_abi_msi_alloc(int intrs[], int count, int cpuid)
	433	{
	434	int i, error;
	435
	436	KASSERT(cpuid >= 0 && cpuid < ncpus,
	437	("invalid cpuid %d", cpuid));
	438
	439	KASSERT(count > 0 && count <= 32, ("invalid count %d\n", count));
	440	KASSERT((count & (count - 1)) == 0,
	441	("count %d is not power of 2\n", count));
	442
	443	lwkt_gettoken(&icu_irqmap_tok);
	444
	445	/*
	446	* NOTE:
	447	* Since IDT_OFFSET is 32, which is the maximum valid 'count',
	448	* we do not need to find out the first properly aligned
	449	* interrupt vector.
	450	*/
	451
	452	error = EMSGSIZE;
	453	for (i = icu_abi_msi_start; i < IDT_HWI_VECTORS; i += count) {
	454	int j;
	455
	456	if (icu_irqmaps[cpuid][i].im_type != ICU_IMT_UNUSED)
	457	continue;
	458
	459	for (j = 1; j < count; ++j) {
	460	if (icu_irqmaps[cpuid][i + j].im_type != ICU_IMT_UNUSED)
	461	break;
	462	}
	463	if (j != count)
	464	continue;
	465
	466	for (j = 0; j < count; ++j) {
	467	struct icu_irqmap *map;
	468	int intr = i + j;
	469
	470	map = &icu_irqmaps[cpuid][intr];
	471	KASSERT(map->im_msi_base < 0,
	472	("intr %d, stale MSI-base %d\n",
	473	intr, map->im_msi_base));
	474
	475	map->im_type = ICU_IMT_MSI;
	476	map->im_msi_base = i;
	477
	478	intrs[j] = intr;
	479	msi_setup(intr, cpuid);
	480
	481	if (bootverbose) {
	482	kprintf("alloc MSI intr %d on cpu%d\n",
	483	intr, cpuid);
	484	}
	485
	486	}
	487	error = 0;
	488	break;
	489	}
	490
	491	lwkt_reltoken(&icu_irqmap_tok);
	492
	493	return error;
494	}
495
496	static void
497	icu_abi_msi_release(const int intrs[], int count, int cpuid)
498	{
499	int i, msi_base = -1, intr_next = -1, mask;
500
501	KASSERT(cpuid >= 0 && cpuid < ncpus,
502	("invalid cpuid %d", cpuid));
503
504	KASSERT(count > 0 && count <= 32, ("invalid count %d\n", count));
505
506	mask = count - 1;
507	KASSERT((count & mask) == 0, ("count %d is not power of 2\n", count));
508
509	lwkt_gettoken(&icu_irqmap_tok);
510
511	for (i = 0; i < count; ++i) {
512	struct icu_irqmap *map;
513	int intr = intrs[i];
514
515	KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
516	("invalid intr %d\n", intr));
517
518	map = &icu_irqmaps[cpuid][intr];
519	KASSERT(map->im_type == ICU_IMT_MSI,
520	("try release non-MSI intr %d, type %d\n",
521	intr, map->im_type));
522	KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr,
523	("intr %d, invalid MSI-base %d\n", intr, map->im_msi_base));
524	KASSERT((map->im_msi_base & mask) == 0,
525	("intr %d, MSI-base %d is not proper aligned %d\n",
526	intr, map->im_msi_base, count));
527
528	if (msi_base < 0) {
529	msi_base = map->im_msi_base;
530	} else {
531	KASSERT(map->im_msi_base == msi_base,
532	("intr %d, inconsistent MSI-base, "
533	"was %d, now %d\n",
534	intr, msi_base, map->im_msi_base));
535	}
536
537	if (intr_next < intr)
538	intr_next = intr;
539
540	map->im_type = ICU_IMT_UNUSED;
541	map->im_msi_base = -1;
542
543	if (bootverbose)
544	kprintf("release MSI intr %d on cpu%d\n", intr, cpuid);
545	}
546
547	KKASSERT(intr_next > 0);
548	KKASSERT(msi_base >= 0);
549
550	++intr_next;
551	if (intr_next < IDT_HWI_VECTORS) {
552	const struct icu_irqmap *map = &icu_irqmaps[cpuid][intr_next];
553
554	if (map->im_type == ICU_IMT_MSI) {
555	KASSERT(map->im_msi_base != msi_base,
556	("more than %d MSI was allocated\n", count));
557	}
558	}
559
560	lwkt_reltoken(&icu_irqmap_tok);
561	}
562
563	static void
564	icu_abi_msi_map(int intr, uint64_t addr, uint32_t data, int cpuid)
565	{
566	const struct icu_irqmap *map;
567
568	KASSERT(cpuid >= 0 && cpuid < ncpus,
569	("invalid cpuid %d", cpuid));
570
571	KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS,
572	("invalid intr %d\n", intr));
573
574	lwkt_gettoken(&icu_irqmap_tok);
575
576	map = &icu_irqmaps[cpuid][intr];
577	KASSERT(map->im_type == ICU_IMT_MSI,
578	("try map non-MSI intr %d, type %d\n", intr, map->im_type));
579	KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr,
580	("intr %d, invalid MSI-base %d\n", intr, map->im_msi_base));
581
582	msi_map(map->im_msi_base, addr, data, cpuid);
583
584	if (bootverbose)
585	kprintf("map MSI intr %d on cpu%d\n", intr, cpuid);
586
587	lwkt_reltoken(&icu_irqmap_tok);
588	}