X-Git-Url: https://gitweb.dragonflybsd.org/dragonfly.git/blobdiff_plain/eb5c3aeb86a1259db800715c417403716a2675df..ea5f4f9e8c49126e9adc10a0e5cce3abcf78cec7:/sys/bus/pci/pci.c

diff --git a/sys/bus/pci/pci.c b/sys/bus/pci/pci.c
index d1ff5df12a..f752fa102b 100644
--- a/sys/bus/pci/pci.c
+++ b/sys/bus/pci/pci.c
@@ -28,7 +28,6 @@
  * $FreeBSD: src/sys/dev/pci/pci.c,v 1.355.2.9.2.1 2009/04/15 03:14:26 kensmith Exp $
  */
 
-#include "opt_bus.h"
 #include "opt_acpi.h"
 #include "opt_compat_oldpci.h"
 
@@ -108,10 +107,12 @@ static void		pci_read_vpd(device_t pcib, pcicfgregs *cfg);
 static void		pci_disable_msi(device_t dev);
 static void		pci_enable_msi(device_t dev, uint64_t address,
 			    uint16_t data);
-static void		pci_enable_msix(device_t dev, u_int index,
+static void		pci_setup_msix_vector(device_t dev, u_int index,
 			    uint64_t address, uint32_t data);
-static void		pci_mask_msix(device_t dev, u_int index);
-static void		pci_unmask_msix(device_t dev, u_int index);
+static void		pci_mask_msix_vector(device_t dev, u_int index);
+static void		pci_unmask_msix_vector(device_t dev, u_int index);
+static void		pci_mask_msix_allvectors(device_t dev);
+static struct msix_vector *pci_find_msix_vector(device_t dev, int rid);
 static int		pci_msi_blacklisted(void);
 static void		pci_resume_msi(device_t dev);
 static void		pci_resume_msix(device_t dev);
@@ -172,8 +173,9 @@ static device_method_t pci_methods[] = {
 	DEVMETHOD(pci_assign_interrupt,	pci_assign_interrupt_method),
 	DEVMETHOD(pci_find_extcap,	pci_find_extcap_method),
 	DEVMETHOD(pci_alloc_msi,	pci_alloc_msi_method),
-	DEVMETHOD(pci_alloc_msix,	pci_alloc_msix_method),
 	DEVMETHOD(pci_release_msi,	pci_release_msi_method),
+	DEVMETHOD(pci_alloc_msix_vector, pci_alloc_msix_vector_method),
+	DEVMETHOD(pci_release_msix_vector, pci_release_msix_vector_method),
 	DEVMETHOD(pci_msi_count,	pci_msi_count_method),
 	DEVMETHOD(pci_msix_count,	pci_msix_count_method),
 
@@ -254,6 +256,9 @@ struct pci_quirk pci_quirks[] = {
 #define	PCI_MAPMEMP	0x02	/* prefetchable memory map */
 #define	PCI_MAPPORT	0x04	/* port map */
 
+#define PCI_MSIX_RID2VEC(rid)	((rid) - 1)	/* rid -> MSI-X vector # */
+#define PCI_MSIX_VEC2RID(vec)	((vec) + 1)	/* MSI-X vector # -> rid */
+
 struct devlist pci_devq;
 uint32_t pci_generation;
 uint32_t pci_numdevs = 0;
@@ -290,12 +295,10 @@ TUNABLE_INT("hw.pci.enable_msi", &pci_do_msi);
 SYSCTL_INT(_hw_pci, OID_AUTO, enable_msi, CTLFLAG_RW, &pci_do_msi, 1,
     "Enable support for MSI interrupts");
 
-static int pci_do_msix = 0;
-#if 0
+static int pci_do_msix = 1;
 TUNABLE_INT("hw.pci.enable_msix", &pci_do_msix);
 SYSCTL_INT(_hw_pci, OID_AUTO, enable_msix, CTLFLAG_RW, &pci_do_msix, 1,
     "Enable support for MSI-X interrupts");
-#endif
 
 static int pci_honor_msi_blacklist = 1;
 TUNABLE_INT("hw.pci.honor_msi_blacklist", &pci_honor_msi_blacklist);
@@ -680,6 +683,8 @@ pci_read_cap_msix(device_t pcib, int ptr, int nextptr, pcicfgregs *cfg)
 	msix->msix_pba_bar = PCIR_BAR(val & PCIM_MSIX_BIR_MASK);
 	msix->msix_pba_offset = val & ~PCIM_MSIX_BIR_MASK;
 
+	TAILQ_INIT(&msix->msix_vectors);
+
 #undef REG
 }
 
@@ -1363,14 +1368,15 @@ pci_find_extcap_method(device_t dev, device_t child, int capability,
 /*
  * Support for MSI-X message interrupts.
  */
-void
-pci_enable_msix(device_t dev, u_int index, uint64_t address, uint32_t data)
+static void
+pci_setup_msix_vector(device_t dev, u_int index, uint64_t address,
+    uint32_t data)
 {
 	struct pci_devinfo *dinfo = device_get_ivars(dev);
 	struct pcicfg_msix *msix = &dinfo->cfg.msix;
 	uint32_t offset;
 
-	KASSERT(msix->msix_table_len > index, ("bogus index"));
+	KASSERT(msix->msix_msgnum > index, ("bogus index"));
 	offset = msix->msix_table_offset + index * 16;
 	bus_write_4(msix->msix_table_res, offset, address & 0xffffffff);
 	bus_write_4(msix->msix_table_res, offset + 4, address >> 32);
@@ -1380,8 +1386,8 @@ pci_enable_msix(device_t dev, u_int index, uint64_t address, uint32_t data)
 	pci_ht_map_msi(dev, address);
 }
 
-void
-pci_mask_msix(device_t dev, u_int index)
+static void
+pci_mask_msix_vector(device_t dev, u_int index)
 {
 	struct pci_devinfo *dinfo = device_get_ivars(dev);
 	struct pcicfg_msix *msix = &dinfo->cfg.msix;
@@ -1396,14 +1402,14 @@ pci_mask_msix(device_t dev, u_int index)
 	}
 }
 
-void
-pci_unmask_msix(device_t dev, u_int index)
+static void
+pci_unmask_msix_vector(device_t dev, u_int index)
 {
 	struct pci_devinfo *dinfo = device_get_ivars(dev);
 	struct pcicfg_msix *msix = &dinfo->cfg.msix;
 	uint32_t offset, val;
 
-	KASSERT(msix->msix_table_len > index, ("bogus index"));
+	KASSERT(msix->msix_msgnum > index, ("bogus index"));
 	offset = msix->msix_table_offset + index * 16 + 12;
 	val = bus_read_4(msix->msix_table_res, offset);
 	if (val & PCIM_MSIX_VCTRL_MASK) {
@@ -1413,13 +1419,16 @@ pci_unmask_msix(device_t dev, u_int index)
 }
 
 int
-pci_pending_msix(device_t dev, u_int index)
+pci_pending_msix_vector(device_t dev, u_int index)
 {
 	struct pci_devinfo *dinfo = device_get_ivars(dev);
 	struct pcicfg_msix *msix = &dinfo->cfg.msix;
 	uint32_t offset, bit;
 
-	KASSERT(msix->msix_table_len > index, ("bogus index"));
+	KASSERT(msix->msix_table_res != NULL && msix->msix_pba_res != NULL,
+	    ("MSI-X is not setup yet\n"));
+
+	KASSERT(msix->msix_msgnum > index, ("bogus index"));
 	offset = msix->msix_pba_offset + (index / 32) * 4;
 	bit = 1 << index % 32;
 	return (bus_read_4(msix->msix_pba_res, offset) & bit);
@@ -1435,23 +1444,22 @@ pci_resume_msix(device_t dev)
 {
 	struct pci_devinfo *dinfo = device_get_ivars(dev);
 	struct pcicfg_msix *msix = &dinfo->cfg.msix;
-	struct msix_table_entry *mte;
-	struct msix_vector *mv;
-	int i;
 
-	if (msix->msix_alloc > 0) {
-		/* First, mask all vectors. */
-		for (i = 0; i < msix->msix_msgnum; i++)
-			pci_mask_msix(dev, i);
+	if (msix->msix_table_res != NULL) {
+		const struct msix_vector *mv;
+
+		pci_mask_msix_allvectors(dev);
 
-		/* Second, program any messages with at least one handler. */
-		for (i = 0; i < msix->msix_table_len; i++) {
-			mte = &msix->msix_table[i];
-			if (mte->mte_vector == 0 || mte->mte_handlers == 0)
+		TAILQ_FOREACH(mv, &msix->msix_vectors, mv_link) {
+			u_int vector;
+
+			if (mv->mv_address == 0)
 				continue;
-			mv = &msix->msix_vectors[mte->mte_vector - 1];
-			pci_enable_msix(dev, i, mv->mv_address, mv->mv_data);
-			pci_unmask_msix(dev, i);
+
+			vector = PCI_MSIX_RID2VEC(mv->mv_rid);
+			pci_setup_msix_vector(dev, vector,
+			    mv->mv_address, mv->mv_data);
+			pci_unmask_msix_vector(dev, vector);
 		}
 	}
 	pci_write_config(dev, msix->msix_location + PCIR_MSIX_CTRL,
@@ -1459,29 +1467,139 @@ pci_resume_msix(device_t dev)
 }
 
 /*
- * Attempt to allocate *count MSI-X messages.  The actual number allocated is
- * returned in *count.  After this function returns, each message will be
- * available to the driver as SYS_RES_IRQ resources starting at rid 1.
+ * Attempt to allocate one MSI-X message at the specified vector on cpuid.
+ *
+ * After this function returns, the MSI-X's rid will be saved in rid0.
  */
 int
-pci_alloc_msix_method(device_t dev, device_t child, int *count)
+pci_alloc_msix_vector_method(device_t dev, device_t child, u_int vector,
+    int *rid0, int cpuid)
 {
 	struct pci_devinfo *dinfo = device_get_ivars(child);
+	struct pcicfg_msix *msix = &dinfo->cfg.msix;
+	struct msix_vector *mv;
+	struct resource_list_entry *rle;
+	int error, irq, rid;
+
+	KASSERT(msix->msix_table_res != NULL &&
+	    msix->msix_pba_res != NULL, ("MSI-X is not setup yet\n"));
+	KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d\n", cpuid));
+	KASSERT(vector < msix->msix_msgnum,
+	    ("invalid MSI-X vector %u, total %d\n", vector, msix->msix_msgnum));
+
+	if (bootverbose) {
+		device_printf(child,
+		    "attempting to allocate MSI-X #%u vector (%d supported)\n",
+		    vector, msix->msix_msgnum);
+	}
+
+	/* Set rid according to vector number */
+	rid = PCI_MSIX_VEC2RID(vector);
+
+	/* Vector has already been allocated */
+	mv = pci_find_msix_vector(child, rid);
+	if (mv != NULL)
+		return EBUSY;
+
+	/* Allocate a message. */
+	error = PCIB_ALLOC_MSIX(device_get_parent(dev), child, &irq, cpuid);
+	if (error)
+		return error;
+	resource_list_add(&dinfo->resources, SYS_RES_IRQ, rid,
+	    irq, irq, 1, cpuid);
+
+	if (bootverbose) {
+		rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, rid);
+		device_printf(child, "using IRQ %lu for MSI-X on cpu%d\n",
+		    rle->start, cpuid);
+	}
+
+	/* Update counts of alloc'd messages. */
+	msix->msix_alloc++;
+
+	mv = kmalloc(sizeof(*mv), M_DEVBUF, M_WAITOK | M_ZERO);
+	mv->mv_rid = rid;
+	TAILQ_INSERT_TAIL(&msix->msix_vectors, mv, mv_link);
+
+	*rid0 = rid;
+	return 0;
+}
+
+int
+pci_release_msix_vector_method(device_t dev, device_t child, int rid)
+{
+	struct pci_devinfo *dinfo = device_get_ivars(child);
+	struct pcicfg_msix *msix = &dinfo->cfg.msix;
+	struct resource_list_entry *rle;
+	struct msix_vector *mv;
+	int irq, cpuid;
+
+	KASSERT(msix->msix_table_res != NULL &&
+	    msix->msix_pba_res != NULL, ("MSI-X is not setup yet\n"));
+	KASSERT(msix->msix_alloc > 0, ("No MSI-X allocated\n"));
+	KASSERT(rid > 0, ("invalid rid %d\n", rid));
+
+	mv = pci_find_msix_vector(child, rid);
+	KASSERT(mv != NULL, ("MSI-X rid %d is not allocated\n", rid));
+	KASSERT(mv->mv_address == 0, ("MSI-X rid %d not teardown\n", rid));
+
+	/* Make sure resource is no longer allocated. */
+	rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, rid);
+	KASSERT(rle != NULL, ("missing MSI-X resource, rid %d\n", rid));
+	KASSERT(rle->res == NULL,
+	    ("MSI-X resource is still allocated, rid %d\n", rid));
+
+	irq = rle->start;
+	cpuid = rle->cpuid;
+
+	/* Free the resource list entries. */
+	resource_list_delete(&dinfo->resources, SYS_RES_IRQ, rid);
+
+	/* Release the IRQ. */
+	PCIB_RELEASE_MSIX(device_get_parent(dev), child, irq, cpuid);
+
+	TAILQ_REMOVE(&msix->msix_vectors, mv, mv_link);
+	kfree(mv, M_DEVBUF);
+
+	msix->msix_alloc--;
+	return (0);
+}
+
+/*
+ * Return the max supported MSI-X messages this device supports.
+ * Basically, assuming the MD code can alloc messages, this function
+ * should return the maximum value that pci_alloc_msix() can return.
+ * Thus, it is subject to the tunables, etc.
+ */
+int
+pci_msix_count_method(device_t dev, device_t child)
+{
+	struct pci_devinfo *dinfo = device_get_ivars(child);
+	struct pcicfg_msix *msix = &dinfo->cfg.msix;
+
+	if (pci_do_msix && msix->msix_location != 0)
+		return (msix->msix_msgnum);
+	return (0);
+}
+
+int
+pci_setup_msix(device_t dev)
+{
+	struct pci_devinfo *dinfo = device_get_ivars(dev);
 	pcicfgregs *cfg = &dinfo->cfg;
 	struct resource_list_entry *rle;
-	int actual, error, i, irq, max;
+	struct resource *table_res, *pba_res;
 
-	/* Don't let count == 0 get us into trouble. */
-	if (*count == 0)
-		return (EINVAL);
+	KASSERT(cfg->msix.msix_table_res == NULL &&
+	    cfg->msix.msix_pba_res == NULL, ("MSI-X has been setup yet\n"));
 
 	/* If rid 0 is allocated, then fail. */
 	rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
 	if (rle != NULL && rle->res != NULL)
 		return (ENXIO);
 
-	/* Already have allocated messages? */
-	if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0)
+	/* Already have allocated MSIs? */
+	if (cfg->msi.msi_alloc != 0)
 		return (ENXIO);
 
 	/* If MSI is blacklisted for this system, fail. */
@@ -1489,16 +1607,21 @@ pci_alloc_msix_method(device_t dev, device_t child, int *count)
 		return (ENXIO);
 
 	/* MSI-X capability present? */
-	if (cfg->msix.msix_location == 0 || !pci_do_msix)
+	if (cfg->msix.msix_location == 0 || cfg->msix.msix_msgnum == 0 ||
+	    !pci_do_msix)
 		return (ENODEV);
 
+	KASSERT(cfg->msix.msix_alloc == 0 &&
+	    TAILQ_EMPTY(&cfg->msix.msix_vectors),
+	    ("MSI-X vector has been allocated\n"));
+
 	/* Make sure the appropriate BARs are mapped. */
 	rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY,
 	    cfg->msix.msix_table_bar);
 	if (rle == NULL || rle->res == NULL ||
 	    !(rman_get_flags(rle->res) & RF_ACTIVE))
 		return (ENXIO);
-	cfg->msix.msix_table_res = rle->res;
+	table_res = rle->res;
 	if (cfg->msix.msix_pba_bar != cfg->msix.msix_table_bar) {
 		rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY,
 		    cfg->msix.msix_pba_bar);
@@ -1506,165 +1629,88 @@ pci_alloc_msix_method(device_t dev, device_t child, int *count)
 		    !(rman_get_flags(rle->res) & RF_ACTIVE))
 			return (ENXIO);
 	}
-	cfg->msix.msix_pba_res = rle->res;
+	pba_res = rle->res;
 
-	if (bootverbose)
-		device_printf(child,
-		    "attempting to allocate %d MSI-X vectors (%d supported)\n",
-		    *count, cfg->msix.msix_msgnum);
-	max = min(*count, cfg->msix.msix_msgnum);
-	for (i = 0; i < max; i++) {
-		/* Allocate a message. */
-		error = PCIB_ALLOC_MSIX(device_get_parent(dev), child, &irq);
-		if (error)
-			break;
-		resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq,
-		    irq, 1, -1);
-	}
-	actual = i;
+	cfg->msix.msix_table_res = table_res;
+	cfg->msix.msix_pba_res = pba_res;
 
-	if (actual == 0) {
-		if (bootverbose) {
-			device_printf(child,
-			    "could not allocate any MSI-X vectors\n");
-		}
-		return  (ENXIO);
-	}
-
-	if (bootverbose) {
-		rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 1);
-		if (actual == 1)
-			device_printf(child, "using IRQ %lu for MSI-X\n",
-			    rle->start);
-		else {
-			int run;
+	pci_mask_msix_allvectors(dev);
 
-			/*
-			 * Be fancy and try to print contiguous runs of
-			 * IRQ values as ranges.  'irq' is the previous IRQ.
-			 * 'run' is true if we are in a range.
-			 */
-			device_printf(child, "using IRQs %lu", rle->start);
-			irq = rle->start;
-			run = 0;
-			for (i = 1; i < actual; i++) {
-				rle = resource_list_find(&dinfo->resources,
-				    SYS_RES_IRQ, i + 1);
+	return 0;
+}
 
-				/* Still in a run? */
-				if (rle->start == irq + 1) {
-					run = 1;
-					irq++;
-					continue;
-				}
+void
+pci_teardown_msix(device_t dev)
+{
+	struct pci_devinfo *dinfo = device_get_ivars(dev);
+	struct pcicfg_msix *msix = &dinfo->cfg.msix;
 
-				/* Finish previous range. */
-				if (run) {
-					kprintf("-%d", irq);
-					run = 0;
-				}
+	KASSERT(msix->msix_table_res != NULL &&
+	    msix->msix_pba_res != NULL, ("MSI-X is not setup yet\n"));
+	KASSERT(msix->msix_alloc == 0 && TAILQ_EMPTY(&msix->msix_vectors),
+	    ("MSI-X vector is still allocated\n"));
 
-				/* Start new range. */
-				kprintf(",%lu", rle->start);
-				irq = rle->start;
-			}
+	pci_mask_msix_allvectors(dev);
 
-			/* Unfinished range? */
-			if (run)
-				kprintf("-%d", irq);
-			kprintf(" for MSI-X\n");
-		}
-	}
+	msix->msix_table_res = NULL;
+	msix->msix_pba_res = NULL;
+}
 
-	/* Mask all vectors. */
-	for (i = 0; i < cfg->msix.msix_msgnum; i++)
-		pci_mask_msix(child, i);
+void
+pci_enable_msix(device_t dev)
+{
+	struct pci_devinfo *dinfo = device_get_ivars(dev);
+	struct pcicfg_msix *msix = &dinfo->cfg.msix;
 
-	/* Allocate and initialize vector data and virtual table. */
-	cfg->msix.msix_vectors = kmalloc(sizeof(struct msix_vector) * actual,
-	    M_DEVBUF, M_WAITOK | M_ZERO);
-	cfg->msix.msix_table = kmalloc(sizeof(struct msix_table_entry) * actual,
-	    M_DEVBUF, M_WAITOK | M_ZERO);
-	for (i = 0; i < actual; i++) {
-		rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
-		cfg->msix.msix_vectors[i].mv_irq = rle->start;
-		cfg->msix.msix_table[i].mte_vector = i + 1;
-	}
+	KASSERT(msix->msix_table_res != NULL &&
+	    msix->msix_pba_res != NULL, ("MSI-X is not setup yet\n"));
 
 	/* Update control register to enable MSI-X. */
-	cfg->msix.msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
-	pci_write_config(child, cfg->msix.msix_location + PCIR_MSIX_CTRL,
-	    cfg->msix.msix_ctrl, 2);
-
-	/* Update counts of alloc'd messages. */
-	cfg->msix.msix_alloc = actual;
-	cfg->msix.msix_table_len = actual;
-	*count = actual;
-	return (0);
+	msix->msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
+	pci_write_config(dev, msix->msix_location + PCIR_MSIX_CTRL,
+	    msix->msix_ctrl, 2);
 }
 
-static int
-pci_release_msix(device_t dev, device_t child)
+void
+pci_disable_msix(device_t dev)
 {
-	struct pci_devinfo *dinfo = device_get_ivars(child);
+	struct pci_devinfo *dinfo = device_get_ivars(dev);
 	struct pcicfg_msix *msix = &dinfo->cfg.msix;
-	struct resource_list_entry *rle;
-	int i;
 
-	/* Do we have any messages to release? */
-	if (msix->msix_alloc == 0)
-		return (ENODEV);
+	KASSERT(msix->msix_table_res != NULL &&
+	    msix->msix_pba_res != NULL, ("MSI-X is not setup yet\n"));
 
-	/* Make sure none of the resources are allocated. */
-	for (i = 0; i < msix->msix_table_len; i++) {
-		if (msix->msix_table[i].mte_vector == 0)
-			continue;
-		if (msix->msix_table[i].mte_handlers > 0)
-			return (EBUSY);
-		rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
-		KASSERT(rle != NULL, ("missing resource"));
-		if (rle->res != NULL)
-			return (EBUSY);
-	}
+	/* Disable MSI -> HT mapping. */
+	pci_ht_map_msi(dev, 0);
 
 	/* Update control register to disable MSI-X. */
 	msix->msix_ctrl &= ~PCIM_MSIXCTRL_MSIX_ENABLE;
-	pci_write_config(child, msix->msix_location + PCIR_MSIX_CTRL,
+	pci_write_config(dev, msix->msix_location + PCIR_MSIX_CTRL,
 	    msix->msix_ctrl, 2);
+}
 
-	/* Free the resource list entries. */
-	for (i = 0; i < msix->msix_table_len; i++) {
-		if (msix->msix_table[i].mte_vector == 0)
-			continue;
-		resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
-	}
-	kfree(msix->msix_table, M_DEVBUF);
-	msix->msix_table_len = 0;
+static void
+pci_mask_msix_allvectors(device_t dev)
+{
+	struct pci_devinfo *dinfo = device_get_ivars(dev);
+	u_int i;
 
-	/* Release the IRQs. */
-	for (i = 0; i < msix->msix_alloc; i++)
-		PCIB_RELEASE_MSIX(device_get_parent(dev), child,
-		    msix->msix_vectors[i].mv_irq);
-	kfree(msix->msix_vectors, M_DEVBUF);
-	msix->msix_alloc = 0;
-	return (0);
+	for (i = 0; i < dinfo->cfg.msix.msix_msgnum; ++i)
+		pci_mask_msix_vector(dev, i);
 }
 
-/*
- * Return the max supported MSI-X messages this device supports.
- * Basically, assuming the MD code can alloc messages, this function
- * should return the maximum value that pci_alloc_msix() can return.
- * Thus, it is subject to the tunables, etc.
- */
-int
-pci_msix_count_method(device_t dev, device_t child)
+static struct msix_vector *
+pci_find_msix_vector(device_t dev, int rid)
 {
-	struct pci_devinfo *dinfo = device_get_ivars(child);
+	struct pci_devinfo *dinfo = device_get_ivars(dev);
 	struct pcicfg_msix *msix = &dinfo->cfg.msix;
+	struct msix_vector *mv;
 
-	if (pci_do_msix && msix->msix_location != 0)
-		return (msix->msix_msgnum);
-	return (0);
+	TAILQ_FOREACH(mv, &msix->msix_vectors, mv_link) {
+		if (mv->mv_rid == rid)
+			return mv;
+	}
+	return NULL;
 }
 
 /*
@@ -1687,7 +1733,7 @@ pci_ht_map_msi(device_t dev, uint64_t addr)
 		    ht->ht_msictrl, 2);
 	}
 
-	if (!addr && ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) {
+	if (!addr && (ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE)) {
 		/* Disable MSI -> HT mapping. */
 		ht->ht_msictrl &= ~PCIM_HTCMD_MSI_ENABLE;
 		pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND,
@@ -1856,7 +1902,7 @@ pci_alloc_msi_method(device_t dev, device_t child, int *rid, int count,
 		return (ENXIO);
 
 	/* Already have allocated messages? */
-	if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0)
+	if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_table_res != NULL)
 		return (ENXIO);
 
 	/* If MSI is blacklisted for this system, fail. */
@@ -1864,7 +1910,8 @@ pci_alloc_msi_method(device_t dev, device_t child, int *rid, int count,
 		return (ENXIO);
 
 	/* MSI capability present? */
-	if (cfg->msi.msi_location == 0 || !pci_do_msi)
+	if (cfg->msi.msi_location == 0 || cfg->msi.msi_msgnum == 0 ||
+	    !pci_do_msi)
 		return (ENODEV);
 
 	KASSERT(count <= cfg->msi.msi_msgnum, ("large MSI count %d, max %d\n",
@@ -1872,8 +1919,8 @@ pci_alloc_msi_method(device_t dev, device_t child, int *rid, int count,
 
 	if (bootverbose) {
 		device_printf(child,
-		    "attempting to allocate %d MSI vectors (%d supported)\n",
-		    count, cfg->msi.msi_msgnum);
+		    "attempting to allocate %d MSI vector%s (%d supported)\n",
+		    count, count > 1 ? "s" : "", cfg->msi.msi_msgnum);
 	}
 
 	if (start_cpuid < 0)
@@ -1962,12 +2009,7 @@ pci_release_msi_method(device_t dev, device_t child)
 	struct pci_devinfo *dinfo = device_get_ivars(child);
 	struct pcicfg_msi *msi = &dinfo->cfg.msi;
 	struct resource_list_entry *rle;
-	int error, i, irqs[32], cpuid = -1;
-
-	/* Try MSI-X first. */
-	error = pci_release_msix(dev, child);
-	if (error != ENODEV)
-		return (error);
+	int i, irqs[32], cpuid = -1;
 
 	/* Do we have any messages to release? */
 	if (msi->msi_alloc == 0)
@@ -2647,10 +2689,16 @@ pci_assign_interrupt(device_t bus, device_t dev, int force_route)
 	/* Let the user override the IRQ with a tunable. */
 	irq = PCI_INVALID_IRQ;
 	ksnprintf(tunable_name, sizeof(tunable_name),
-	    "hw.pci%d.%d.%d.INT%c.irq",
-	    cfg->domain, cfg->bus, cfg->slot, cfg->intpin + 'A' - 1);
-	if (TUNABLE_INT_FETCH(tunable_name, &irq) && (irq >= 255 || irq <= 0))
-		irq = PCI_INVALID_IRQ;
+	    "hw.pci%d.%d.%d.%d.INT%c.irq",
+	    cfg->domain, cfg->bus, cfg->slot, cfg->func, cfg->intpin + 'A' - 1);
+	if (TUNABLE_INT_FETCH(tunable_name, &irq)) {
+		if (irq >= 255 || irq <= 0) {
+			irq = PCI_INVALID_IRQ;
+		} else {
+			BUS_CONFIG_INTR(bus, dev, irq,
+			    INTR_TRIGGER_LEVEL, INTR_POLARITY_LOW);
+		}
+	}
 
 	/*
 	 * If we didn't get an IRQ via the tunable, then we either use the
@@ -2943,18 +2991,14 @@ pci_child_detached(device_t parent __unused, device_t child)
 
 int
 pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags,
-    driver_intr_t *intr, void *arg, void **cookiep, lwkt_serialize_t serializer)
+    driver_intr_t *intr, void *arg, void **cookiep,
+    lwkt_serialize_t serializer, const char *desc)
 {
-	struct pci_devinfo *dinfo;
-	struct msix_table_entry *mte;
-	struct msix_vector *mv;
-	uint64_t addr;
-	uint32_t data;
 	int rid, error;
 	void *cookie;
 
 	error = bus_generic_setup_intr(dev, child, irq, flags, intr,
-	    arg, &cookie, serializer);
+	    arg, &cookie, serializer, desc);
 	if (error)
 		return (error);
 
@@ -2969,6 +3013,10 @@ pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags,
 		/* Make sure that INTx is enabled */
 		pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS);
 	} else {
+		struct pci_devinfo *dinfo = device_get_ivars(child);
+		uint64_t addr;
+		uint32_t data;
+
 		/*
 		 * Check to see if the interrupt is MSI or MSI-X.
 		 * Ask our parent to map the MSI and give
@@ -2976,48 +3024,47 @@ pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags,
 		 * If we fail for some reason, teardown the
 		 * interrupt handler.
 		 */
-		dinfo = device_get_ivars(child);
 		if (dinfo->cfg.msi.msi_alloc > 0) {
-			if (dinfo->cfg.msi.msi_addr == 0) {
-				KASSERT(dinfo->cfg.msi.msi_handlers == 0,
+			struct pcicfg_msi *msi = &dinfo->cfg.msi;
+
+			if (msi->msi_addr == 0) {
+				KASSERT(msi->msi_handlers == 0,
 			    ("MSI has handlers, but vectors not mapped"));
 				error = PCIB_MAP_MSI(device_get_parent(dev),
 				    child, rman_get_start(irq), &addr, &data,
 				    rman_get_cpuid(irq));
 				if (error)
 					goto bad;
-				dinfo->cfg.msi.msi_addr = addr;
-				dinfo->cfg.msi.msi_data = data;
+				msi->msi_addr = addr;
+				msi->msi_data = data;
 				pci_enable_msi(child, addr, data);
 			}
-			dinfo->cfg.msi.msi_handlers++;
+			msi->msi_handlers++;
 		} else {
+			struct msix_vector *mv;
+			u_int vector;
+
 			KASSERT(dinfo->cfg.msix.msix_alloc > 0,
-			    ("No MSI or MSI-X interrupts allocated"));
-			KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
-			    ("MSI-X index too high"));
-			mte = &dinfo->cfg.msix.msix_table[rid - 1];
-			KASSERT(mte->mte_vector != 0, ("no message vector"));
-			mv = &dinfo->cfg.msix.msix_vectors[mte->mte_vector - 1];
-			KASSERT(mv->mv_irq == rman_get_start(irq),
-			    ("IRQ mismatch"));
-			if (mv->mv_address == 0) {
-				KASSERT(mte->mte_handlers == 0,
-		    ("MSI-X table entry has handlers, but vector not mapped"));
-				error = PCIB_MAP_MSI(device_get_parent(dev),
-				    child, rman_get_start(irq), &addr, &data,
-				    rman_get_cpuid(irq));
-				if (error)
-					goto bad;
-				mv->mv_address = addr;
-				mv->mv_data = data;
-			}
-			if (mte->mte_handlers == 0) {
-				pci_enable_msix(child, rid - 1, mv->mv_address,
-				    mv->mv_data);
-				pci_unmask_msix(child, rid - 1);
-			}
-			mte->mte_handlers++;
+			    ("No MSI-X or MSI rid %d allocated\n", rid));
+
+			mv = pci_find_msix_vector(child, rid);
+			KASSERT(mv != NULL,
+			    ("MSI-X rid %d is not allocated\n", rid));
+			KASSERT(mv->mv_address == 0,
+			    ("MSI-X rid %d has been setup\n", rid));
+
+			error = PCIB_MAP_MSI(device_get_parent(dev),
+			    child, rman_get_start(irq), &addr, &data,
+			    rman_get_cpuid(irq));
+			if (error)
+				goto bad;
+			mv->mv_address = addr;
+			mv->mv_data = data;
+
+			vector = PCI_MSIX_RID2VEC(rid);
+			pci_setup_msix_vector(child, vector,
+			    mv->mv_address, mv->mv_data);
+			pci_unmask_msix_vector(child, vector);
 		}
 
 		/* Make sure that INTx is disabled if we are using MSI/MSIX */
@@ -3037,9 +3084,6 @@ int
 pci_teardown_intr(device_t dev, device_t child, struct resource *irq,
     void *cookie)
 {
-	struct msix_table_entry *mte;
-	struct resource_list_entry *rle;
-	struct pci_devinfo *dinfo;
 	int rid, error;
 
 	if (irq == NULL || !(rman_get_flags(irq) & RF_ACTIVE))
@@ -3054,35 +3098,40 @@ pci_teardown_intr(device_t dev, device_t child, struct resource *irq,
 		/* Mask INTx */
 		pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS);
 	} else {
+		struct pci_devinfo *dinfo = device_get_ivars(child);
+
 		/*
 		 * Check to see if the interrupt is MSI or MSI-X.  If so,
 		 * decrement the appropriate handlers count and mask the
 		 * MSI-X message, or disable MSI messages if the count
 		 * drops to 0.
 		 */
-		dinfo = device_get_ivars(child);
-		rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, rid);
-		if (rle->res != irq)
-			return (EINVAL);
 		if (dinfo->cfg.msi.msi_alloc > 0) {
-			KASSERT(rid <= dinfo->cfg.msi.msi_alloc,
-			    ("MSI-X index too high"));
-			if (dinfo->cfg.msi.msi_handlers == 0)
-				return (EINVAL);
-			dinfo->cfg.msi.msi_handlers--;
-			if (dinfo->cfg.msi.msi_handlers == 0)
+			struct pcicfg_msi *msi = &dinfo->cfg.msi;
+
+			KASSERT(rid <= msi->msi_alloc,
+			    ("MSI-X index too high\n"));
+			KASSERT(msi->msi_handlers > 0,
+			    ("MSI rid %d is not setup\n", rid));
+
+			msi->msi_handlers--;
+			if (msi->msi_handlers == 0)
 				pci_disable_msi(child);
 		} else {
+			struct msix_vector *mv;
+
 			KASSERT(dinfo->cfg.msix.msix_alloc > 0,
-			    ("No MSI or MSI-X interrupts allocated"));
-			KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
-			    ("MSI-X index too high"));
-			mte = &dinfo->cfg.msix.msix_table[rid - 1];
-			if (mte->mte_handlers == 0)
-				return (EINVAL);
-			mte->mte_handlers--;
-			if (mte->mte_handlers == 0)
-				pci_mask_msix(child, rid - 1);
+			    ("No MSI or MSI-X rid %d allocated", rid));
+
+			mv = pci_find_msix_vector(child, rid);
+			KASSERT(mv != NULL,
+			    ("MSI-X rid %d is not allocated\n", rid));
+			KASSERT(mv->mv_address != 0,
+			    ("MSI-X rid %d has not been setup\n", rid));
+
+			pci_mask_msix_vector(child, PCI_MSIX_RID2VEC(rid));
+			mv->mv_address = 0;
+			mv->mv_data = 0;
 		}
 	}
 	error = bus_generic_teardown_intr(dev, child, irq, cookie);
@@ -4049,22 +4098,16 @@ pci_alloc_1intr(device_t dev, int msi_enable, int *rid0, u_int *flags0)
 {
 	int rid, type;
 	u_int flags;
-	char env[64];
 
 	rid = 0;
 	type = PCI_INTR_TYPE_LEGACY;
 	flags = RF_SHAREABLE | RF_ACTIVE;
 
-	ksnprintf(env, sizeof(env), "hw.%s.msi.enable",
-	    device_get_nameunit(dev));
-	kgetenv_int(env, &msi_enable);
-
+	device_getenv_int(dev, "msi.enable", &msi_enable);
 	if (msi_enable) {
 		int cpu = -1;
 
-		ksnprintf(env, sizeof(env), "hw.%s.msi.cpu",
-		    device_get_nameunit(dev));
-		kgetenv_int(env, &cpu);
+		device_getenv_int(dev, "msi.cpu", &cpu);
 		if (cpu >= ncpus)
 			cpu = ncpus - 1;