X-Git-Url: https://gitweb.dragonflybsd.org/dragonfly.git/blobdiff_plain/9b33b2248c9d362077a8df8e7afd64f8104a8fd1..ea5f4f9e8c49126e9adc10a0e5cce3abcf78cec7:/sys/bus/pci/pci.c diff --git a/sys/bus/pci/pci.c b/sys/bus/pci/pci.c index 610ed62653..f752fa102b 100644 --- a/sys/bus/pci/pci.c +++ b/sys/bus/pci/pci.c @@ -28,7 +28,6 @@ * $FreeBSD: src/sys/dev/pci/pci.c,v 1.355.2.9.2.1 2009/04/15 03:14:26 kensmith Exp $ */ -#include "opt_bus.h" #include "opt_acpi.h" #include "opt_compat_oldpci.h" @@ -45,6 +44,8 @@ #include #include +#include + #include #include #include @@ -106,10 +107,12 @@ static void pci_read_vpd(device_t pcib, pcicfgregs *cfg); static void pci_disable_msi(device_t dev); static void pci_enable_msi(device_t dev, uint64_t address, uint16_t data); -static void pci_enable_msix(device_t dev, u_int index, +static void pci_setup_msix_vector(device_t dev, u_int index, uint64_t address, uint32_t data); -static void pci_mask_msix(device_t dev, u_int index); -static void pci_unmask_msix(device_t dev, u_int index); +static void pci_mask_msix_vector(device_t dev, u_int index); +static void pci_unmask_msix_vector(device_t dev, u_int index); +static void pci_mask_msix_allvectors(device_t dev); +static struct msix_vector *pci_find_msix_vector(device_t dev, int rid); static int pci_msi_blacklisted(void); static void pci_resume_msi(device_t dev); static void pci_resume_msix(device_t dev); @@ -170,9 +173,9 @@ static device_method_t pci_methods[] = { DEVMETHOD(pci_assign_interrupt, pci_assign_interrupt_method), DEVMETHOD(pci_find_extcap, pci_find_extcap_method), DEVMETHOD(pci_alloc_msi, pci_alloc_msi_method), - DEVMETHOD(pci_alloc_msix, pci_alloc_msix_method), - DEVMETHOD(pci_remap_msix, pci_remap_msix_method), DEVMETHOD(pci_release_msi, pci_release_msi_method), + DEVMETHOD(pci_alloc_msix_vector, pci_alloc_msix_vector_method), + DEVMETHOD(pci_release_msix_vector, pci_release_msix_vector_method), DEVMETHOD(pci_msi_count, pci_msi_count_method), DEVMETHOD(pci_msix_count, pci_msix_count_method), @@ -253,6 +256,9 @@ struct pci_quirk pci_quirks[] = { #define PCI_MAPMEMP 0x02 /* prefetchable memory map */ #define PCI_MAPPORT 0x04 /* port map */ +#define PCI_MSIX_RID2VEC(rid) ((rid) - 1) /* rid -> MSI-X vector # */ +#define PCI_MSIX_VEC2RID(vec) ((vec) + 1) /* MSI-X vector # -> rid */ + struct devlist pci_devq; uint32_t pci_generation; uint32_t pci_numdevs = 0; @@ -299,6 +305,8 @@ TUNABLE_INT("hw.pci.honor_msi_blacklist", &pci_honor_msi_blacklist); SYSCTL_INT(_hw_pci, OID_AUTO, honor_msi_blacklist, CTLFLAG_RD, &pci_honor_msi_blacklist, 1, "Honor chipset blacklist for MSI"); +static int pci_msi_cpuid; + /* Find a device_t by bus/slot/function in domain 0 */ device_t @@ -597,7 +605,6 @@ pci_read_cap_pmgt(device_t pcib, int ptr, int nextptr, pcicfgregs *cfg) static void pci_read_cap_ht(device_t pcib, int ptr, int nextptr, pcicfgregs *cfg) { -#ifdef notyet #if defined(__i386__) || defined(__x86_64__) #define REG(n, w) \ @@ -610,6 +617,9 @@ pci_read_cap_ht(device_t pcib, int ptr, int nextptr, pcicfgregs *cfg) /* Determine HT-specific capability type. */ val = REG(ptr + PCIR_HT_COMMAND, 2); + if ((val & 0xe000) == PCIM_HTCAP_SLAVE) + cfg->ht.ht_slave = ptr; + if ((val & PCIM_HTCMD_CAP_MASK) != PCIM_HTCAP_MSI_MAPPING) return; @@ -618,14 +628,14 @@ pci_read_cap_ht(device_t pcib, int ptr, int nextptr, pcicfgregs *cfg) addr = REG(ptr + PCIR_HTMSI_ADDRESS_HI, 4); addr <<= 32; addr |= REG(ptr + PCIR_HTMSI_ADDRESS_LO, 4); - if (addr != MSI_INTEL_ADDR_BASE) { + if (addr != MSI_X86_ADDR_BASE) { device_printf(pcib, "HT Bridge at pci%d:%d:%d:%d " "has non-default MSI window 0x%llx\n", cfg->domain, cfg->bus, cfg->slot, cfg->func, (long long)addr); } } else { - addr = MSI_INTEL_ADDR_BASE; + addr = MSI_X86_ADDR_BASE; } ht->ht_msimap = ptr; @@ -635,7 +645,6 @@ pci_read_cap_ht(device_t pcib, int ptr, int nextptr, pcicfgregs *cfg) #undef REG #endif /* __i386__ || __x86_64__ */ -#endif /* notyet */ } static void @@ -674,6 +683,8 @@ pci_read_cap_msix(device_t pcib, int ptr, int nextptr, pcicfgregs *cfg) msix->msix_pba_bar = PCIR_BAR(val & PCIM_MSIX_BIR_MASK); msix->msix_pba_offset = val & ~PCIM_MSIX_BIR_MASK; + TAILQ_INIT(&msix->msix_vectors); + #undef REG } @@ -828,6 +839,24 @@ pci_read_capabilities(device_t pcib, pcicfgregs *cfg) } } } + +#if defined(__i386__) || defined(__x86_64__) + /* + * Enable the MSI mapping window for all HyperTransport + * slaves. PCI-PCI bridges have their windows enabled via + * PCIB_MAP_MSI(). + */ + if (cfg->ht.ht_slave != 0 && cfg->ht.ht_msimap != 0 && + !(cfg->ht.ht_msictrl & PCIM_HTCMD_MSI_ENABLE)) { + device_printf(pcib, + "Enabling MSI window for HyperTransport slave at pci%d:%d:%d:%d\n", + cfg->domain, cfg->bus, cfg->slot, cfg->func); + cfg->ht.ht_msictrl |= PCIM_HTCMD_MSI_ENABLE; + WREG(cfg->ht.ht_msimap + PCIR_HT_COMMAND, cfg->ht.ht_msictrl, + 2); + } +#endif + /* REG and WREG use carry through to next functions */ } @@ -1339,14 +1368,15 @@ pci_find_extcap_method(device_t dev, device_t child, int capability, /* * Support for MSI-X message interrupts. */ -void -pci_enable_msix(device_t dev, u_int index, uint64_t address, uint32_t data) +static void +pci_setup_msix_vector(device_t dev, u_int index, uint64_t address, + uint32_t data) { struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msix *msix = &dinfo->cfg.msix; uint32_t offset; - KASSERT(msix->msix_table_len > index, ("bogus index")); + KASSERT(msix->msix_msgnum > index, ("bogus index")); offset = msix->msix_table_offset + index * 16; bus_write_4(msix->msix_table_res, offset, address & 0xffffffff); bus_write_4(msix->msix_table_res, offset + 4, address >> 32); @@ -1356,8 +1386,8 @@ pci_enable_msix(device_t dev, u_int index, uint64_t address, uint32_t data) pci_ht_map_msi(dev, address); } -void -pci_mask_msix(device_t dev, u_int index) +static void +pci_mask_msix_vector(device_t dev, u_int index) { struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msix *msix = &dinfo->cfg.msix; @@ -1372,14 +1402,14 @@ pci_mask_msix(device_t dev, u_int index) } } -void -pci_unmask_msix(device_t dev, u_int index) +static void +pci_unmask_msix_vector(device_t dev, u_int index) { struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msix *msix = &dinfo->cfg.msix; uint32_t offset, val; - KASSERT(msix->msix_table_len > index, ("bogus index")); + KASSERT(msix->msix_msgnum > index, ("bogus index")); offset = msix->msix_table_offset + index * 16 + 12; val = bus_read_4(msix->msix_table_res, offset); if (val & PCIM_MSIX_VCTRL_MASK) { @@ -1389,13 +1419,16 @@ pci_unmask_msix(device_t dev, u_int index) } int -pci_pending_msix(device_t dev, u_int index) +pci_pending_msix_vector(device_t dev, u_int index) { struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msix *msix = &dinfo->cfg.msix; uint32_t offset, bit; - KASSERT(msix->msix_table_len > index, ("bogus index")); + KASSERT(msix->msix_table_res != NULL && msix->msix_pba_res != NULL, + ("MSI-X is not setup yet\n")); + + KASSERT(msix->msix_msgnum > index, ("bogus index")); offset = msix->msix_pba_offset + (index / 32) * 4; bit = 1 << index % 32; return (bus_read_4(msix->msix_pba_res, offset) & bit); @@ -1411,23 +1444,22 @@ pci_resume_msix(device_t dev) { struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msix *msix = &dinfo->cfg.msix; - struct msix_table_entry *mte; - struct msix_vector *mv; - int i; - if (msix->msix_alloc > 0) { - /* First, mask all vectors. */ - for (i = 0; i < msix->msix_msgnum; i++) - pci_mask_msix(dev, i); + if (msix->msix_table_res != NULL) { + const struct msix_vector *mv; - /* Second, program any messages with at least one handler. */ - for (i = 0; i < msix->msix_table_len; i++) { - mte = &msix->msix_table[i]; - if (mte->mte_vector == 0 || mte->mte_handlers == 0) + pci_mask_msix_allvectors(dev); + + TAILQ_FOREACH(mv, &msix->msix_vectors, mv_link) { + u_int vector; + + if (mv->mv_address == 0) continue; - mv = &msix->msix_vectors[mte->mte_vector - 1]; - pci_enable_msix(dev, i, mv->mv_address, mv->mv_data); - pci_unmask_msix(dev, i); + + vector = PCI_MSIX_RID2VEC(mv->mv_rid); + pci_setup_msix_vector(dev, vector, + mv->mv_address, mv->mv_data); + pci_unmask_msix_vector(dev, vector); } } pci_write_config(dev, msix->msix_location + PCIR_MSIX_CTRL, @@ -1435,29 +1467,139 @@ pci_resume_msix(device_t dev) } /* - * Attempt to allocate *count MSI-X messages. The actual number allocated is - * returned in *count. After this function returns, each message will be - * available to the driver as SYS_RES_IRQ resources starting at rid 1. + * Attempt to allocate one MSI-X message at the specified vector on cpuid. + * + * After this function returns, the MSI-X's rid will be saved in rid0. */ int -pci_alloc_msix_method(device_t dev, device_t child, int *count) +pci_alloc_msix_vector_method(device_t dev, device_t child, u_int vector, + int *rid0, int cpuid) { struct pci_devinfo *dinfo = device_get_ivars(child); + struct pcicfg_msix *msix = &dinfo->cfg.msix; + struct msix_vector *mv; + struct resource_list_entry *rle; + int error, irq, rid; + + KASSERT(msix->msix_table_res != NULL && + msix->msix_pba_res != NULL, ("MSI-X is not setup yet\n")); + KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d\n", cpuid)); + KASSERT(vector < msix->msix_msgnum, + ("invalid MSI-X vector %u, total %d\n", vector, msix->msix_msgnum)); + + if (bootverbose) { + device_printf(child, + "attempting to allocate MSI-X #%u vector (%d supported)\n", + vector, msix->msix_msgnum); + } + + /* Set rid according to vector number */ + rid = PCI_MSIX_VEC2RID(vector); + + /* Vector has already been allocated */ + mv = pci_find_msix_vector(child, rid); + if (mv != NULL) + return EBUSY; + + /* Allocate a message. */ + error = PCIB_ALLOC_MSIX(device_get_parent(dev), child, &irq, cpuid); + if (error) + return error; + resource_list_add(&dinfo->resources, SYS_RES_IRQ, rid, + irq, irq, 1, cpuid); + + if (bootverbose) { + rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, rid); + device_printf(child, "using IRQ %lu for MSI-X on cpu%d\n", + rle->start, cpuid); + } + + /* Update counts of alloc'd messages. */ + msix->msix_alloc++; + + mv = kmalloc(sizeof(*mv), M_DEVBUF, M_WAITOK | M_ZERO); + mv->mv_rid = rid; + TAILQ_INSERT_TAIL(&msix->msix_vectors, mv, mv_link); + + *rid0 = rid; + return 0; +} + +int +pci_release_msix_vector_method(device_t dev, device_t child, int rid) +{ + struct pci_devinfo *dinfo = device_get_ivars(child); + struct pcicfg_msix *msix = &dinfo->cfg.msix; + struct resource_list_entry *rle; + struct msix_vector *mv; + int irq, cpuid; + + KASSERT(msix->msix_table_res != NULL && + msix->msix_pba_res != NULL, ("MSI-X is not setup yet\n")); + KASSERT(msix->msix_alloc > 0, ("No MSI-X allocated\n")); + KASSERT(rid > 0, ("invalid rid %d\n", rid)); + + mv = pci_find_msix_vector(child, rid); + KASSERT(mv != NULL, ("MSI-X rid %d is not allocated\n", rid)); + KASSERT(mv->mv_address == 0, ("MSI-X rid %d not teardown\n", rid)); + + /* Make sure resource is no longer allocated. */ + rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, rid); + KASSERT(rle != NULL, ("missing MSI-X resource, rid %d\n", rid)); + KASSERT(rle->res == NULL, + ("MSI-X resource is still allocated, rid %d\n", rid)); + + irq = rle->start; + cpuid = rle->cpuid; + + /* Free the resource list entries. */ + resource_list_delete(&dinfo->resources, SYS_RES_IRQ, rid); + + /* Release the IRQ. */ + PCIB_RELEASE_MSIX(device_get_parent(dev), child, irq, cpuid); + + TAILQ_REMOVE(&msix->msix_vectors, mv, mv_link); + kfree(mv, M_DEVBUF); + + msix->msix_alloc--; + return (0); +} + +/* + * Return the max supported MSI-X messages this device supports. + * Basically, assuming the MD code can alloc messages, this function + * should return the maximum value that pci_alloc_msix() can return. + * Thus, it is subject to the tunables, etc. + */ +int +pci_msix_count_method(device_t dev, device_t child) +{ + struct pci_devinfo *dinfo = device_get_ivars(child); + struct pcicfg_msix *msix = &dinfo->cfg.msix; + + if (pci_do_msix && msix->msix_location != 0) + return (msix->msix_msgnum); + return (0); +} + +int +pci_setup_msix(device_t dev) +{ + struct pci_devinfo *dinfo = device_get_ivars(dev); pcicfgregs *cfg = &dinfo->cfg; struct resource_list_entry *rle; - int actual, error, i, irq, max; + struct resource *table_res, *pba_res; - /* Don't let count == 0 get us into trouble. */ - if (*count == 0) - return (EINVAL); + KASSERT(cfg->msix.msix_table_res == NULL && + cfg->msix.msix_pba_res == NULL, ("MSI-X has been setup yet\n")); /* If rid 0 is allocated, then fail. */ rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0); if (rle != NULL && rle->res != NULL) return (ENXIO); - /* Already have allocated messages? */ - if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0) + /* Already have allocated MSIs? */ + if (cfg->msi.msi_alloc != 0) return (ENXIO); /* If MSI is blacklisted for this system, fail. */ @@ -1465,16 +1607,21 @@ pci_alloc_msix_method(device_t dev, device_t child, int *count) return (ENXIO); /* MSI-X capability present? */ - if (cfg->msix.msix_location == 0 || !pci_do_msix) + if (cfg->msix.msix_location == 0 || cfg->msix.msix_msgnum == 0 || + !pci_do_msix) return (ENODEV); + KASSERT(cfg->msix.msix_alloc == 0 && + TAILQ_EMPTY(&cfg->msix.msix_vectors), + ("MSI-X vector has been allocated\n")); + /* Make sure the appropriate BARs are mapped. */ rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY, cfg->msix.msix_table_bar); if (rle == NULL || rle->res == NULL || !(rman_get_flags(rle->res) & RF_ACTIVE)) return (ENXIO); - cfg->msix.msix_table_res = rle->res; + table_res = rle->res; if (cfg->msix.msix_pba_bar != cfg->msix.msix_table_bar) { rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY, cfg->msix.msix_pba_bar); @@ -1482,322 +1629,88 @@ pci_alloc_msix_method(device_t dev, device_t child, int *count) !(rman_get_flags(rle->res) & RF_ACTIVE)) return (ENXIO); } - cfg->msix.msix_pba_res = rle->res; + pba_res = rle->res; - if (bootverbose) - device_printf(child, - "attempting to allocate %d MSI-X vectors (%d supported)\n", - *count, cfg->msix.msix_msgnum); - max = min(*count, cfg->msix.msix_msgnum); - for (i = 0; i < max; i++) { - /* Allocate a message. */ - error = PCIB_ALLOC_MSIX(device_get_parent(dev), child, &irq); - if (error) - break; - resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq, - irq, 1, -1); - } - actual = i; + cfg->msix.msix_table_res = table_res; + cfg->msix.msix_pba_res = pba_res; - if (actual == 0) { - if (bootverbose) { - device_printf(child, - "could not allocate any MSI-X vectors\n"); - } - return (ENXIO); - } + pci_mask_msix_allvectors(dev); - if (bootverbose) { - rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 1); - if (actual == 1) - device_printf(child, "using IRQ %lu for MSI-X\n", - rle->start); - else { - int run; - - /* - * Be fancy and try to print contiguous runs of - * IRQ values as ranges. 'irq' is the previous IRQ. - * 'run' is true if we are in a range. - */ - device_printf(child, "using IRQs %lu", rle->start); - irq = rle->start; - run = 0; - for (i = 1; i < actual; i++) { - rle = resource_list_find(&dinfo->resources, - SYS_RES_IRQ, i + 1); - - /* Still in a run? */ - if (rle->start == irq + 1) { - run = 1; - irq++; - continue; - } - - /* Finish previous range. */ - if (run) { - kprintf("-%d", irq); - run = 0; - } - - /* Start new range. */ - kprintf(",%lu", rle->start); - irq = rle->start; - } - - /* Unfinished range? */ - if (run) - kprintf("-%d", irq); - kprintf(" for MSI-X\n"); - } - } - - /* Mask all vectors. */ - for (i = 0; i < cfg->msix.msix_msgnum; i++) - pci_mask_msix(child, i); - - /* Allocate and initialize vector data and virtual table. */ - cfg->msix.msix_vectors = kmalloc(sizeof(struct msix_vector) * actual, - M_DEVBUF, M_WAITOK | M_ZERO); - cfg->msix.msix_table = kmalloc(sizeof(struct msix_table_entry) * actual, - M_DEVBUF, M_WAITOK | M_ZERO); - for (i = 0; i < actual; i++) { - rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); - cfg->msix.msix_vectors[i].mv_irq = rle->start; - cfg->msix.msix_table[i].mte_vector = i + 1; - } - - /* Update control register to enable MSI-X. */ - cfg->msix.msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; - pci_write_config(child, cfg->msix.msix_location + PCIR_MSIX_CTRL, - cfg->msix.msix_ctrl, 2); - - /* Update counts of alloc'd messages. */ - cfg->msix.msix_alloc = actual; - cfg->msix.msix_table_len = actual; - *count = actual; - return (0); + return 0; } -/* - * By default, pci_alloc_msix() will assign the allocated IRQ - * resources consecutively to the first N messages in the MSI-X table. - * However, device drivers may want to use different layouts if they - * either receive fewer messages than they asked for, or they wish to - * populate the MSI-X table sparsely. This method allows the driver - * to specify what layout it wants. It must be called after a - * successful pci_alloc_msix() but before any of the associated - * SYS_RES_IRQ resources are allocated via bus_alloc_resource(). - * - * The 'vectors' array contains 'count' message vectors. The array - * maps directly to the MSI-X table in that index 0 in the array - * specifies the vector for the first message in the MSI-X table, etc. - * The vector value in each array index can either be 0 to indicate - * that no vector should be assigned to a message slot, or it can be a - * number from 1 to N (where N is the count returned from a - * succcessful call to pci_alloc_msix()) to indicate which message - * vector (IRQ) to be used for the corresponding message. - * - * On successful return, each message with a non-zero vector will have - * an associated SYS_RES_IRQ whose rid is equal to the array index + - * 1. Additionally, if any of the IRQs allocated via the previous - * call to pci_alloc_msix() are not used in the mapping, those IRQs - * will be kfreed back to the system automatically. - * - * For example, suppose a driver has a MSI-X table with 6 messages and - * asks for 6 messages, but pci_alloc_msix() only returns a count of - * 3. Call the three vectors allocated by pci_alloc_msix() A, B, and - * C. After the call to pci_alloc_msix(), the device will be setup to - * have an MSI-X table of ABC--- (where - means no vector assigned). - * If the driver ten passes a vector array of { 1, 0, 1, 2, 0, 2 }, - * then the MSI-X table will look like A-AB-B, and the 'C' vector will - * be kfreed back to the system. This device will also have valid - * SYS_RES_IRQ rids of 1, 3, 4, and 6. - * - * In any case, the SYS_RES_IRQ rid X will always map to the message - * at MSI-X table index X - 1 and will only be valid if a vector is - * assigned to that table entry. - */ -int -pci_remap_msix_method(device_t dev, device_t child, int count, - const u_int *vectors) +void +pci_teardown_msix(device_t dev) { - struct pci_devinfo *dinfo = device_get_ivars(child); + struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msix *msix = &dinfo->cfg.msix; - struct resource_list_entry *rle; - int i, irq, j, *used; - - /* - * Have to have at least one message in the table but the - * table can't be bigger than the actual MSI-X table in the - * device. - */ - if (count == 0 || count > msix->msix_msgnum) - return (EINVAL); - - /* Sanity check the vectors. */ - for (i = 0; i < count; i++) - if (vectors[i] > msix->msix_alloc) - return (EINVAL); - /* - * Make sure there aren't any holes in the vectors to be used. - * It's a big pain to support it, and it doesn't really make - * sense anyway. Also, at least one vector must be used. - */ - used = kmalloc(sizeof(int) * msix->msix_alloc, M_DEVBUF, M_WAITOK | - M_ZERO); - for (i = 0; i < count; i++) - if (vectors[i] != 0) - used[vectors[i] - 1] = 1; - for (i = 0; i < msix->msix_alloc - 1; i++) - if (used[i] == 0 && used[i + 1] == 1) { - kfree(used, M_DEVBUF); - return (EINVAL); - } - if (used[0] != 1) { - kfree(used, M_DEVBUF); - return (EINVAL); - } - - /* Make sure none of the resources are allocated. */ - for (i = 0; i < msix->msix_table_len; i++) { - if (msix->msix_table[i].mte_vector == 0) - continue; - if (msix->msix_table[i].mte_handlers > 0) - return (EBUSY); - rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); - KASSERT(rle != NULL, ("missing resource")); - if (rle->res != NULL) - return (EBUSY); - } + KASSERT(msix->msix_table_res != NULL && + msix->msix_pba_res != NULL, ("MSI-X is not setup yet\n")); + KASSERT(msix->msix_alloc == 0 && TAILQ_EMPTY(&msix->msix_vectors), + ("MSI-X vector is still allocated\n")); - /* Free the existing resource list entries. */ - for (i = 0; i < msix->msix_table_len; i++) { - if (msix->msix_table[i].mte_vector == 0) - continue; - resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1); - } + pci_mask_msix_allvectors(dev); - /* - * Build the new virtual table keeping track of which vectors are - * used. - */ - kfree(msix->msix_table, M_DEVBUF); - msix->msix_table = kmalloc(sizeof(struct msix_table_entry) * count, - M_DEVBUF, M_WAITOK | M_ZERO); - for (i = 0; i < count; i++) - msix->msix_table[i].mte_vector = vectors[i]; - msix->msix_table_len = count; - - /* Free any unused IRQs and resize the vectors array if necessary. */ - j = msix->msix_alloc - 1; - if (used[j] == 0) { - struct msix_vector *vec; - - while (used[j] == 0) { - PCIB_RELEASE_MSIX(device_get_parent(dev), child, - msix->msix_vectors[j].mv_irq); - j--; - } - vec = kmalloc(sizeof(struct msix_vector) * (j + 1), M_DEVBUF, - M_WAITOK); - bcopy(msix->msix_vectors, vec, sizeof(struct msix_vector) * - (j + 1)); - kfree(msix->msix_vectors, M_DEVBUF); - msix->msix_vectors = vec; - msix->msix_alloc = j + 1; - } - kfree(used, M_DEVBUF); + msix->msix_table_res = NULL; + msix->msix_pba_res = NULL; +} - /* Map the IRQs onto the rids. */ - for (i = 0; i < count; i++) { - if (vectors[i] == 0) - continue; - irq = msix->msix_vectors[vectors[i]].mv_irq; - resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq, - irq, 1, -1); - } +void +pci_enable_msix(device_t dev) +{ + struct pci_devinfo *dinfo = device_get_ivars(dev); + struct pcicfg_msix *msix = &dinfo->cfg.msix; - if (bootverbose) { - device_printf(child, "Remapped MSI-X IRQs as: "); - for (i = 0; i < count; i++) { - if (i != 0) - kprintf(", "); - if (vectors[i] == 0) - kprintf("---"); - else - kprintf("%d", - msix->msix_vectors[vectors[i]].mv_irq); - } - kprintf("\n"); - } + KASSERT(msix->msix_table_res != NULL && + msix->msix_pba_res != NULL, ("MSI-X is not setup yet\n")); - return (0); + /* Update control register to enable MSI-X. */ + msix->msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; + pci_write_config(dev, msix->msix_location + PCIR_MSIX_CTRL, + msix->msix_ctrl, 2); } -static int -pci_release_msix(device_t dev, device_t child) +void +pci_disable_msix(device_t dev) { - struct pci_devinfo *dinfo = device_get_ivars(child); + struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msix *msix = &dinfo->cfg.msix; - struct resource_list_entry *rle; - int i; - /* Do we have any messages to release? */ - if (msix->msix_alloc == 0) - return (ENODEV); + KASSERT(msix->msix_table_res != NULL && + msix->msix_pba_res != NULL, ("MSI-X is not setup yet\n")); - /* Make sure none of the resources are allocated. */ - for (i = 0; i < msix->msix_table_len; i++) { - if (msix->msix_table[i].mte_vector == 0) - continue; - if (msix->msix_table[i].mte_handlers > 0) - return (EBUSY); - rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); - KASSERT(rle != NULL, ("missing resource")); - if (rle->res != NULL) - return (EBUSY); - } + /* Disable MSI -> HT mapping. */ + pci_ht_map_msi(dev, 0); /* Update control register to disable MSI-X. */ msix->msix_ctrl &= ~PCIM_MSIXCTRL_MSIX_ENABLE; - pci_write_config(child, msix->msix_location + PCIR_MSIX_CTRL, + pci_write_config(dev, msix->msix_location + PCIR_MSIX_CTRL, msix->msix_ctrl, 2); +} - /* Free the resource list entries. */ - for (i = 0; i < msix->msix_table_len; i++) { - if (msix->msix_table[i].mte_vector == 0) - continue; - resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1); - } - kfree(msix->msix_table, M_DEVBUF); - msix->msix_table_len = 0; - - /* Release the IRQs. */ - for (i = 0; i < msix->msix_alloc; i++) - PCIB_RELEASE_MSIX(device_get_parent(dev), child, - msix->msix_vectors[i].mv_irq); - kfree(msix->msix_vectors, M_DEVBUF); - msix->msix_alloc = 0; - return (0); +static void +pci_mask_msix_allvectors(device_t dev) +{ + struct pci_devinfo *dinfo = device_get_ivars(dev); + u_int i; + + for (i = 0; i < dinfo->cfg.msix.msix_msgnum; ++i) + pci_mask_msix_vector(dev, i); } -/* - * Return the max supported MSI-X messages this device supports. - * Basically, assuming the MD code can alloc messages, this function - * should return the maximum value that pci_alloc_msix() can return. - * Thus, it is subject to the tunables, etc. - */ -int -pci_msix_count_method(device_t dev, device_t child) +static struct msix_vector * +pci_find_msix_vector(device_t dev, int rid) { - struct pci_devinfo *dinfo = device_get_ivars(child); + struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msix *msix = &dinfo->cfg.msix; + struct msix_vector *mv; - if (pci_do_msix && msix->msix_location != 0) - return (msix->msix_msgnum); - return (0); + TAILQ_FOREACH(mv, &msix->msix_vectors, mv_link) { + if (mv->mv_rid == rid) + return mv; + } + return NULL; } /* @@ -1820,7 +1733,7 @@ pci_ht_map_msi(device_t dev, uint64_t addr) ht->ht_msictrl, 2); } - if (!addr && ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) { + if (!addr && (ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE)) { /* Disable MSI -> HT mapping. */ ht->ht_msictrl &= ~PCIM_HTCMD_MSI_ENABLE; pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND, @@ -1904,82 +1817,6 @@ pci_resume_msi(device_t dev) 2); } -int -pci_remap_msi_irq(device_t dev, u_int irq) -{ - struct pci_devinfo *dinfo = device_get_ivars(dev); - pcicfgregs *cfg = &dinfo->cfg; - struct resource_list_entry *rle; - struct msix_table_entry *mte; - struct msix_vector *mv; - device_t bus; - uint64_t addr; - uint32_t data; - int error, i, j; - - bus = device_get_parent(dev); - - /* - * Handle MSI first. We try to find this IRQ among our list - * of MSI IRQs. If we find it, we request updated address and - * data registers and apply the results. - */ - if (cfg->msi.msi_alloc > 0) { - - /* If we don't have any active handlers, nothing to do. */ - if (cfg->msi.msi_handlers == 0) - return (0); - for (i = 0; i < cfg->msi.msi_alloc; i++) { - rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, - i + 1); - if (rle->start == irq) { - error = PCIB_MAP_MSI(device_get_parent(bus), - dev, irq, &addr, &data); - if (error) - return (error); - pci_disable_msi(dev); - dinfo->cfg.msi.msi_addr = addr; - dinfo->cfg.msi.msi_data = data; - pci_enable_msi(dev, addr, data); - return (0); - } - } - return (ENOENT); - } - - /* - * For MSI-X, we check to see if we have this IRQ. If we do, - * we request the updated mapping info. If that works, we go - * through all the slots that use this IRQ and update them. - */ - if (cfg->msix.msix_alloc > 0) { - for (i = 0; i < cfg->msix.msix_alloc; i++) { - mv = &cfg->msix.msix_vectors[i]; - if (mv->mv_irq == irq) { - error = PCIB_MAP_MSI(device_get_parent(bus), - dev, irq, &addr, &data); - if (error) - return (error); - mv->mv_address = addr; - mv->mv_data = data; - for (j = 0; j < cfg->msix.msix_table_len; j++) { - mte = &cfg->msix.msix_table[j]; - if (mte->mte_vector != i + 1) - continue; - if (mte->mte_handlers == 0) - continue; - pci_mask_msix(dev, j); - pci_enable_msix(dev, j, addr, data); - pci_unmask_msix(dev, j); - } - } - } - return (ENOENT); - } - - return (ENOENT); -} - /* * Returns true if the specified device is blacklisted because MSI * doesn't work. @@ -2026,22 +1863,38 @@ pci_msi_blacklisted(void) } /* - * Attempt to allocate *count MSI messages. The actual number allocated is - * returned in *count. After this function returns, each message will be - * available to the driver as SYS_RES_IRQ resources starting at a rid 1. + * Attempt to allocate count MSI messages on start_cpuid. + * + * If start_cpuid < 0, then the MSI messages' target CPU will be + * selected automaticly. + * + * If the caller explicitly specified the MSI messages' target CPU, + * i.e. start_cpuid >= 0, then we will try to allocate the count MSI + * messages on the specified CPU, if the allocation fails due to MD + * does not have enough vectors (EMSGSIZE), then we will try next + * available CPU, until the allocation fails on all CPUs. + * + * EMSGSIZE will be returned, if all available CPUs does not have + * enough vectors for the requested amount of MSI messages. Caller + * should either reduce the amount of MSI messages to be requested, + * or simply giving up using MSI. + * + * The available SYS_RES_IRQ resources' rids, which are >= 1, are + * returned in 'rid' array, if the allocation succeeds. */ int -pci_alloc_msi_method(device_t dev, device_t child, int *count) +pci_alloc_msi_method(device_t dev, device_t child, int *rid, int count, + int start_cpuid) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; struct resource_list_entry *rle; - int actual, error, i, irqs[32]; + int error, i, irqs[32], cpuid = 0; uint16_t ctrl; - /* Don't let count == 0 get us into trouble. */ - if (*count == 0) - return (EINVAL); + KASSERT(count != 0 && count <= 32 && powerof2(count), + ("invalid MSI count %d\n", count)); + KASSERT(start_cpuid < ncpus, ("invalid cpuid %d\n", start_cpuid)); /* If rid 0 is allocated, then fail. */ rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0); @@ -2049,7 +1902,7 @@ pci_alloc_msi_method(device_t dev, device_t child, int *count) return (ENXIO); /* Already have allocated messages? */ - if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0) + if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_table_res != NULL) return (ENXIO); /* If MSI is blacklisted for this system, fail. */ @@ -2057,50 +1910,51 @@ pci_alloc_msi_method(device_t dev, device_t child, int *count) return (ENXIO); /* MSI capability present? */ - if (cfg->msi.msi_location == 0 || !pci_do_msi) + if (cfg->msi.msi_location == 0 || cfg->msi.msi_msgnum == 0 || + !pci_do_msi) return (ENODEV); - if (bootverbose) - device_printf(child, - "attempting to allocate %d MSI vectors (%d supported)\n", - *count, cfg->msi.msi_msgnum); + KASSERT(count <= cfg->msi.msi_msgnum, ("large MSI count %d, max %d\n", + count, cfg->msi.msi_msgnum)); - /* Don't ask for more than the device supports. */ - actual = min(*count, cfg->msi.msi_msgnum); + if (bootverbose) { + device_printf(child, + "attempting to allocate %d MSI vector%s (%d supported)\n", + count, count > 1 ? "s" : "", cfg->msi.msi_msgnum); + } - /* Don't ask for more than 32 messages. */ - actual = min(actual, 32); + if (start_cpuid < 0) + start_cpuid = atomic_fetchadd_int(&pci_msi_cpuid, 1) % ncpus; - /* MSI requires power of 2 number of messages. */ - if (!powerof2(actual)) - return (EINVAL); + error = EINVAL; + for (i = 0; i < ncpus; ++i) { + cpuid = (start_cpuid + i) % ncpus; - for (;;) { - /* Try to allocate N messages. */ - error = PCIB_ALLOC_MSI(device_get_parent(dev), child, actual, - cfg->msi.msi_msgnum, irqs); + error = PCIB_ALLOC_MSI(device_get_parent(dev), child, count, + cfg->msi.msi_msgnum, irqs, cpuid); if (error == 0) break; - if (actual == 1) - return (error); - - /* Try N / 2. */ - actual >>= 1; + else if (error != EMSGSIZE) + return error; } + if (error) + return error; /* - * We now have N actual messages mapped onto SYS_RES_IRQ - * resources in the irqs[] array, so add new resources - * starting at rid 1. + * We now have N messages mapped onto SYS_RES_IRQ resources in + * the irqs[] array, so add new resources starting at rid 1. */ - for (i = 0; i < actual; i++) + for (i = 0; i < count; i++) { + rid[i] = i + 1; resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, - irqs[i], irqs[i], 1, -1); + irqs[i], irqs[i], 1, cpuid); + } if (bootverbose) { - if (actual == 1) - device_printf(child, "using IRQ %d for MSI\n", irqs[0]); - else { + if (count == 1) { + device_printf(child, "using IRQ %d on cpu%d for MSI\n", + irqs[0], cpuid); + } else { int run; /* @@ -2110,7 +1964,7 @@ pci_alloc_msi_method(device_t dev, device_t child, int *count) */ device_printf(child, "using IRQs %d", irqs[0]); run = 0; - for (i = 1; i < actual; i++) { + for (i = 1; i < count; i++) { /* Still in a run? */ if (irqs[i] == irqs[i - 1] + 1) { @@ -2130,22 +1984,21 @@ pci_alloc_msi_method(device_t dev, device_t child, int *count) /* Unfinished range? */ if (run) - kprintf("-%d", irqs[actual - 1]); - kprintf(" for MSI\n"); + kprintf("-%d", irqs[count - 1]); + kprintf(" for MSI on cpu%d\n", cpuid); } } - /* Update control register with actual count. */ + /* Update control register with count. */ ctrl = cfg->msi.msi_ctrl; ctrl &= ~PCIM_MSICTRL_MME_MASK; - ctrl |= (ffs(actual) - 1) << 4; + ctrl |= (ffs(count) - 1) << 4; cfg->msi.msi_ctrl = ctrl; pci_write_config(child, cfg->msi.msi_location + PCIR_MSI_CTRL, ctrl, 2); /* Update counts of alloc'd messages. */ - cfg->msi.msi_alloc = actual; + cfg->msi.msi_alloc = count; cfg->msi.msi_handlers = 0; - *count = actual; return (0); } @@ -2156,12 +2009,7 @@ pci_release_msi_method(device_t dev, device_t child) struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msi *msi = &dinfo->cfg.msi; struct resource_list_entry *rle; - int error, i, irqs[32]; - - /* Try MSI-X first. */ - error = pci_release_msix(dev, child); - if (error != ENODEV) - return (error); + int i, irqs[32], cpuid = -1; /* Do we have any messages to release? */ if (msi->msi_alloc == 0) @@ -2176,6 +2024,15 @@ pci_release_msi_method(device_t dev, device_t child) KASSERT(rle != NULL, ("missing MSI resource")); if (rle->res != NULL) return (EBUSY); + if (i == 0) { + cpuid = rle->cpuid; + KASSERT(cpuid >= 0 && cpuid < ncpus, + ("invalid MSI target cpuid %d\n", cpuid)); + } else { + KASSERT(rle->cpuid == cpuid, + ("MSI targets different cpus, " + "was cpu%d, now cpu%d", cpuid, rle->cpuid)); + } irqs[i] = rle->start; } @@ -2187,7 +2044,8 @@ pci_release_msi_method(device_t dev, device_t child) msi->msi_ctrl, 2); /* Release the messages. */ - PCIB_RELEASE_MSI(device_get_parent(dev), child, msi->msi_alloc, irqs); + PCIB_RELEASE_MSI(device_get_parent(dev), child, msi->msi_alloc, irqs, + cpuid); for (i = 0; i < msi->msi_alloc; i++) resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1); @@ -2831,10 +2689,16 @@ pci_assign_interrupt(device_t bus, device_t dev, int force_route) /* Let the user override the IRQ with a tunable. */ irq = PCI_INVALID_IRQ; ksnprintf(tunable_name, sizeof(tunable_name), - "hw.pci%d.%d.%d.INT%c.irq", - cfg->domain, cfg->bus, cfg->slot, cfg->intpin + 'A' - 1); - if (TUNABLE_INT_FETCH(tunable_name, &irq) && (irq >= 255 || irq <= 0)) - irq = PCI_INVALID_IRQ; + "hw.pci%d.%d.%d.%d.INT%c.irq", + cfg->domain, cfg->bus, cfg->slot, cfg->func, cfg->intpin + 'A' - 1); + if (TUNABLE_INT_FETCH(tunable_name, &irq)) { + if (irq >= 255 || irq <= 0) { + irq = PCI_INVALID_IRQ; + } else { + BUS_CONFIG_INTR(bus, dev, irq, + INTR_TRIGGER_LEVEL, INTR_POLARITY_LOW); + } + } /* * If we didn't get an IRQ via the tunable, then we either use the @@ -2862,7 +2726,7 @@ pci_assign_interrupt(device_t bus, device_t dev, int force_route) /* Add this IRQ as rid 0 interrupt resource. */ resource_list_add(&dinfo->resources, SYS_RES_IRQ, 0, irq, irq, 1, - machintr_intr_cpuid(irq)); + machintr_legacy_intr_cpuid(irq)); } void @@ -3127,20 +2991,14 @@ pci_child_detached(device_t parent __unused, device_t child) int pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags, - driver_intr_t *intr, void *arg, void **cookiep, lwkt_serialize_t serializer) + driver_intr_t *intr, void *arg, void **cookiep, + lwkt_serialize_t serializer, const char *desc) { -#ifdef MSI - struct pci_devinfo *dinfo; - struct msix_table_entry *mte; - struct msix_vector *mv; - uint64_t addr; - uint32_t data; - int rid; -#endif - int error; + int rid, error; void *cookie; + error = bus_generic_setup_intr(dev, child, irq, flags, intr, - arg, &cookie, serializer); + arg, &cookie, serializer, desc); if (error) return (error); @@ -3150,13 +3008,15 @@ pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags, return(0); } - pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS); -#ifdef MSI rid = rman_get_rid(irq); if (rid == 0) { /* Make sure that INTx is enabled */ pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS); } else { + struct pci_devinfo *dinfo = device_get_ivars(child); + uint64_t addr; + uint32_t data; + /* * Check to see if the interrupt is MSI or MSI-X. * Ask our parent to map the MSI and give @@ -3164,46 +3024,47 @@ pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags, * If we fail for some reason, teardown the * interrupt handler. */ - dinfo = device_get_ivars(child); if (dinfo->cfg.msi.msi_alloc > 0) { - if (dinfo->cfg.msi.msi_addr == 0) { - KASSERT(dinfo->cfg.msi.msi_handlers == 0, + struct pcicfg_msi *msi = &dinfo->cfg.msi; + + if (msi->msi_addr == 0) { + KASSERT(msi->msi_handlers == 0, ("MSI has handlers, but vectors not mapped")); error = PCIB_MAP_MSI(device_get_parent(dev), - child, rman_get_start(irq), &addr, &data); + child, rman_get_start(irq), &addr, &data, + rman_get_cpuid(irq)); if (error) goto bad; - dinfo->cfg.msi.msi_addr = addr; - dinfo->cfg.msi.msi_data = data; + msi->msi_addr = addr; + msi->msi_data = data; pci_enable_msi(child, addr, data); } - dinfo->cfg.msi.msi_handlers++; + msi->msi_handlers++; } else { + struct msix_vector *mv; + u_int vector; + KASSERT(dinfo->cfg.msix.msix_alloc > 0, - ("No MSI or MSI-X interrupts allocated")); - KASSERT(rid <= dinfo->cfg.msix.msix_table_len, - ("MSI-X index too high")); - mte = &dinfo->cfg.msix.msix_table[rid - 1]; - KASSERT(mte->mte_vector != 0, ("no message vector")); - mv = &dinfo->cfg.msix.msix_vectors[mte->mte_vector - 1]; - KASSERT(mv->mv_irq == rman_get_start(irq), - ("IRQ mismatch")); - if (mv->mv_address == 0) { - KASSERT(mte->mte_handlers == 0, - ("MSI-X table entry has handlers, but vector not mapped")); - error = PCIB_MAP_MSI(device_get_parent(dev), - child, rman_get_start(irq), &addr, &data); - if (error) - goto bad; - mv->mv_address = addr; - mv->mv_data = data; - } - if (mte->mte_handlers == 0) { - pci_enable_msix(child, rid - 1, mv->mv_address, - mv->mv_data); - pci_unmask_msix(child, rid - 1); - } - mte->mte_handlers++; + ("No MSI-X or MSI rid %d allocated\n", rid)); + + mv = pci_find_msix_vector(child, rid); + KASSERT(mv != NULL, + ("MSI-X rid %d is not allocated\n", rid)); + KASSERT(mv->mv_address == 0, + ("MSI-X rid %d has been setup\n", rid)); + + error = PCIB_MAP_MSI(device_get_parent(dev), + child, rman_get_start(irq), &addr, &data, + rman_get_cpuid(irq)); + if (error) + goto bad; + mv->mv_address = addr; + mv->mv_data = data; + + vector = PCI_MSIX_RID2VEC(rid); + pci_setup_msix_vector(child, vector, + mv->mv_address, mv->mv_data); + pci_unmask_msix_vector(child, vector); } /* Make sure that INTx is disabled if we are using MSI/MSIX */ @@ -3215,7 +3076,6 @@ pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags, return (error); } } -#endif *cookiep = cookie; return (0); } @@ -3224,13 +3084,7 @@ int pci_teardown_intr(device_t dev, device_t child, struct resource *irq, void *cookie) { -#ifdef MSI - struct msix_table_entry *mte; - struct resource_list_entry *rle; - struct pci_devinfo *dinfo; - int rid; -#endif - int error; + int rid, error; if (irq == NULL || !(rman_get_flags(irq) & RF_ACTIVE)) return (EINVAL); @@ -3239,50 +3093,51 @@ pci_teardown_intr(device_t dev, device_t child, struct resource *irq, if (device_get_parent(child) != dev) return(bus_generic_teardown_intr(dev, child, irq, cookie)); - pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS); -#ifdef MSI rid = rman_get_rid(irq); if (rid == 0) { /* Mask INTx */ pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS); } else { + struct pci_devinfo *dinfo = device_get_ivars(child); + /* * Check to see if the interrupt is MSI or MSI-X. If so, * decrement the appropriate handlers count and mask the * MSI-X message, or disable MSI messages if the count * drops to 0. */ - dinfo = device_get_ivars(child); - rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, rid); - if (rle->res != irq) - return (EINVAL); if (dinfo->cfg.msi.msi_alloc > 0) { - KASSERT(rid <= dinfo->cfg.msi.msi_alloc, - ("MSI-X index too high")); - if (dinfo->cfg.msi.msi_handlers == 0) - return (EINVAL); - dinfo->cfg.msi.msi_handlers--; - if (dinfo->cfg.msi.msi_handlers == 0) + struct pcicfg_msi *msi = &dinfo->cfg.msi; + + KASSERT(rid <= msi->msi_alloc, + ("MSI-X index too high\n")); + KASSERT(msi->msi_handlers > 0, + ("MSI rid %d is not setup\n", rid)); + + msi->msi_handlers--; + if (msi->msi_handlers == 0) pci_disable_msi(child); } else { + struct msix_vector *mv; + KASSERT(dinfo->cfg.msix.msix_alloc > 0, - ("No MSI or MSI-X interrupts allocated")); - KASSERT(rid <= dinfo->cfg.msix.msix_table_len, - ("MSI-X index too high")); - mte = &dinfo->cfg.msix.msix_table[rid - 1]; - if (mte->mte_handlers == 0) - return (EINVAL); - mte->mte_handlers--; - if (mte->mte_handlers == 0) - pci_mask_msix(child, rid - 1); + ("No MSI or MSI-X rid %d allocated", rid)); + + mv = pci_find_msix_vector(child, rid); + KASSERT(mv != NULL, + ("MSI-X rid %d is not allocated\n", rid)); + KASSERT(mv->mv_address != 0, + ("MSI-X rid %d has not been setup\n", rid)); + + pci_mask_msix_vector(child, PCI_MSIX_RID2VEC(rid)); + mv->mv_address = 0; + mv->mv_data = 0; } } error = bus_generic_teardown_intr(dev, child, irq, cookie); if (rid > 0) KASSERT(error == 0, ("%s: generic teardown failed for MSI/MSI-X", __func__)); -#endif - error = bus_generic_teardown_intr(dev, child, irq, cookie); return (error); } @@ -3875,11 +3730,9 @@ pci_alloc_resource(device_t dev, device_t child, int type, int *rid, * Can't alloc legacy interrupt once MSI messages * have been allocated. */ -#ifdef MSI if (*rid == 0 && (cfg->msi.msi_alloc > 0 || cfg->msix.msix_alloc > 0)) return (NULL); -#endif /* * If the child device doesn't have an * interrupt routed and is deserving of an @@ -4239,3 +4092,33 @@ pci_devlist_get_parent(pcicfgregs *cfg) } #endif /* COMPAT_OLDPCI */ + +int +pci_alloc_1intr(device_t dev, int msi_enable, int *rid0, u_int *flags0) +{ + int rid, type; + u_int flags; + + rid = 0; + type = PCI_INTR_TYPE_LEGACY; + flags = RF_SHAREABLE | RF_ACTIVE; + + device_getenv_int(dev, "msi.enable", &msi_enable); + if (msi_enable) { + int cpu = -1; + + device_getenv_int(dev, "msi.cpu", &cpu); + if (cpu >= ncpus) + cpu = ncpus - 1; + + if (pci_alloc_msi(dev, &rid, 1, cpu) == 0) { + flags &= ~RF_SHAREABLE; + type = PCI_INTR_TYPE_MSI; + } + } + + *rid0 = rid; + *flags0 = flags; + + return type; +}