bnx: Split RX/TX serializers
[dragonfly.git] / sys / dev / netif / bnx / if_bnx.c
index eb6a5a5..170fcac 100644 (file)
@@ -160,6 +160,11 @@ static void        bnx_enable_intr(struct bnx_softc *);
 static void    bnx_disable_intr(struct bnx_softc *);
 static void    bnx_txeof(struct bnx_tx_ring *, uint16_t);
 static void    bnx_rxeof(struct bnx_rx_ret_ring *, uint16_t, int);
+static int     bnx_alloc_intr(struct bnx_softc *);
+static int     bnx_setup_intr(struct bnx_softc *);
+static void    bnx_free_intr(struct bnx_softc *);
+static void    bnx_teardown_intr(struct bnx_softc *, int);
+static void    bnx_check_intr(void *);
 
 static void    bnx_start(struct ifnet *, struct ifaltq_subque *);
 static int     bnx_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
@@ -169,6 +174,13 @@ static void        bnx_watchdog(struct ifnet *);
 static int     bnx_ifmedia_upd(struct ifnet *);
 static void    bnx_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 static void    bnx_tick(void *);
+static void    bnx_serialize(struct ifnet *, enum ifnet_serialize);
+static void    bnx_deserialize(struct ifnet *, enum ifnet_serialize);
+static int     bnx_tryserialize(struct ifnet *, enum ifnet_serialize);
+#ifdef INVARIANTS
+static void    bnx_serialize_assert(struct ifnet *, enum ifnet_serialize,
+                   boolean_t);
+#endif
 
 static int     bnx_alloc_jumbo_mem(struct bnx_softc *);
 static void    bnx_free_jumbo_mem(struct bnx_softc *);
@@ -201,6 +213,7 @@ static int  bnx_encap(struct bnx_tx_ring *, struct mbuf **,
                    uint32_t *, int *);
 static int     bnx_setup_tso(struct bnx_tx_ring *, struct mbuf **,
                    uint16_t *, uint16_t *);
+static void    bnx_setup_serialize(struct bnx_softc *);
 
 static void    bnx_reset(struct bnx_softc *);
 static int     bnx_chipinit(struct bnx_softc *);
@@ -909,7 +922,7 @@ bnx_free_tx_ring(struct bnx_tx_ring *txr)
 static int
 bnx_init_tx_ring(struct bnx_tx_ring *txr)
 {
-       txr->bnx_txcnt = 0;
+       txr->bnx_tx_cnt = 0;
        txr->bnx_tx_saved_considx = 0;
        txr->bnx_tx_prodidx = 0;
 
@@ -1670,22 +1683,19 @@ bnx_attach(device_t dev)
        struct ifnet *ifp;
        struct bnx_softc *sc;
        uint32_t hwcfg = 0;
-       int error = 0, rid, capmask;
+       int error = 0, rid, capmask, i;
        uint8_t ether_addr[ETHER_ADDR_LEN];
        uint16_t product;
-       driver_intr_t *intr_func;
        uintptr_t mii_priv = 0;
-       u_int intr_flags;
 #ifdef BNX_TSO_DEBUG
        char desc[32];
-       int i;
 #endif
 
        sc = device_get_softc(dev);
        sc->bnx_dev = dev;
        callout_init_mp(&sc->bnx_stat_timer);
-       callout_init_mp(&sc->bnx_intr_timer);
        lwkt_serialize_init(&sc->bnx_jslot_serializer);
+       lwkt_serialize_init(&sc->bnx_main_serialize);
 
        product = pci_get_device(dev);
 
@@ -1807,7 +1817,7 @@ bnx_attach(device_t dev)
                 * For the rest of the chips in these two families, we will
                 * have to poll the status block at high rate (10ms currently)
                 * to check whether the interrupt is hosed or not.
-                * See bnx_intr_check() for details.
+                * See bnx_check_intr() for details.
                 */
                sc->bnx_flags |= BNX_FLAG_STATUSTAG_BUG;
        }
@@ -1870,21 +1880,12 @@ bnx_attach(device_t dev)
        /*
         * Allocate interrupt
         */
-       sc->bnx_irq_type = pci_alloc_1intr(dev, bnx_msi_enable, &sc->bnx_irq_rid,
-           &intr_flags);
-
-       sc->bnx_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->bnx_irq_rid,
-           intr_flags);
-       if (sc->bnx_irq == NULL) {
-               device_printf(dev, "couldn't map interrupt\n");
-               error = ENXIO;
+       error = bnx_alloc_intr(sc);
+       if (error)
                goto fail;
-       }
 
-       if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI) {
-               sc->bnx_flags |= BNX_FLAG_ONESHOT_MSI;
-               bnx_enable_msi(sc);
-       }
+       /* Setup serializers */
+       bnx_setup_serialize(sc);
 
        /* Set default tuneable values. */
        sc->bnx_rx_coal_ticks = BNX_RX_COAL_TICKS_DEF;
@@ -1904,10 +1905,14 @@ bnx_attach(device_t dev)
 #endif
        ifp->if_watchdog = bnx_watchdog;
        ifp->if_init = bnx_init;
+       ifp->if_serialize = bnx_serialize;
+       ifp->if_deserialize = bnx_deserialize;
+       ifp->if_tryserialize = bnx_tryserialize;
+#ifdef INVARIANTS
+       ifp->if_serialize_assert = bnx_serialize_assert;
+#endif
        ifp->if_mtu = ETHERMTU;
        ifp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
-       ifq_set_maxlen(&ifp->if_snd, BGE_TX_RING_CNT - 1);
-       ifq_set_ready(&ifp->if_snd);
 
        ifp->if_capabilities |= IFCAP_HWCSUM;
        ifp->if_hwassist = BNX_CSUM_FEATURES;
@@ -1917,6 +1922,10 @@ bnx_attach(device_t dev)
        }
        ifp->if_capenable = ifp->if_capabilities;
 
+       ifq_set_maxlen(&ifp->if_snd, BGE_TX_RING_CNT - 1);
+       ifq_set_ready(&ifp->if_snd);
+       ifq_set_subq_cnt(&ifp->if_snd, sc->bnx_tx_ringcnt);
+
        /*
         * Figure out what sort of media we have by checking the
         * hardware config word in the first 32k of NIC internal memory,
@@ -2114,35 +2123,34 @@ bnx_attach(device_t dev)
         */
        ether_ifattach(ifp, ether_addr, NULL);
 
-       ifq_set_cpuid(&ifp->if_snd, sc->bnx_intr_cpuid);
+       /* Setup TX rings and subqueues */
+       for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
+               struct ifaltq_subque *ifsq = ifq_get_subq(&ifp->if_snd, i);
+               struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
+
+               ifsq_set_cpuid(ifsq, txr->bnx_tx_cpuid);
+               ifsq_set_hw_serialize(ifsq, &txr->bnx_tx_serialize);
+#ifdef notyet
+               ifsq_set_priv(ifsq, txr);
+               txr->ifsq = ifsq;
+
+               ifsq_watchdog_init(&txr->tx_watchdog, ifsq, bce_watchdog);
+#endif
+       }
 
 #ifdef IFPOLL_ENABLE
        ifpoll_compat_setup(&sc->bnx_npoll,
            &sc->bnx_sysctl_ctx, sc->bnx_sysctl_tree,
-           device_get_unit(dev), ifp->if_serializer);
+           device_get_unit(dev), &sc->bnx_main_serialize);
 #endif
 
-       if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI) {
-               if (sc->bnx_flags & BNX_FLAG_ONESHOT_MSI) {
-                       intr_func = bnx_msi_oneshot;
-                       if (bootverbose)
-                               device_printf(dev, "oneshot MSI\n");
-               } else {
-                       intr_func = bnx_msi;
-               }
-       } else {
-               intr_func = bnx_intr_legacy;
-       }
-       error = bus_setup_intr(dev, sc->bnx_irq, INTR_MPSAFE, intr_func, sc,
-           &sc->bnx_intrhand, ifp->if_serializer);
+       error = bnx_setup_intr(sc);
        if (error) {
                ether_ifdetach(ifp);
-               device_printf(dev, "couldn't set up irq\n");
                goto fail;
        }
 
-       sc->bnx_intr_cpuid = rman_get_cpuid(sc->bnx_irq);
-       sc->bnx_stat_cpuid = sc->bnx_intr_cpuid;
+       sc->bnx_stat_cpuid = sc->bnx_intr_data[0].bnx_intr_cpuid;
 
        return(0);
 fail:
@@ -2158,11 +2166,11 @@ bnx_detach(device_t dev)
        if (device_is_attached(dev)) {
                struct ifnet *ifp = &sc->arpcom.ac_if;
 
-               lwkt_serialize_enter(ifp->if_serializer);
+               ifnet_serialize_all(ifp);
                bnx_stop(sc);
                bnx_reset(sc);
-               bus_teardown_intr(dev, sc->bnx_irq, sc->bnx_intrhand);
-               lwkt_serialize_exit(ifp->if_serializer);
+               bnx_teardown_intr(sc, sc->bnx_intr_cnt);
+               ifnet_deserialize_all(ifp);
 
                ether_ifdetach(ifp);
        }
@@ -2173,12 +2181,7 @@ bnx_detach(device_t dev)
                device_delete_child(dev, sc->bnx_miibus);
        bus_generic_detach(dev);
 
-       if (sc->bnx_irq != NULL) {
-               bus_release_resource(dev, SYS_RES_IRQ, sc->bnx_irq_rid,
-                   sc->bnx_irq);
-       }
-       if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI)
-               pci_release_msi(dev);
+       bnx_free_intr(sc);
 
        if (sc->bnx_res != NULL) {
                bus_release_resource(dev, SYS_RES_MEMORY,
@@ -2190,6 +2193,9 @@ bnx_detach(device_t dev)
 
        bnx_dma_free(sc);
 
+       if (sc->bnx_serialize != NULL)
+               kfree(sc->bnx_serialize, M_DEVBUF);
+
        return 0;
 }
 
@@ -2518,15 +2524,15 @@ bnx_txeof(struct bnx_tx_ring *txr, uint16_t tx_cons)
                        m_freem(buf->bnx_tx_mbuf);
                        buf->bnx_tx_mbuf = NULL;
                }
-               txr->bnx_txcnt--;
+               txr->bnx_tx_cnt--;
                BNX_INC(txr->bnx_tx_saved_considx, BGE_TX_RING_CNT);
        }
 
-       if ((BGE_TX_RING_CNT - txr->bnx_txcnt) >=
+       if ((BGE_TX_RING_CNT - txr->bnx_tx_cnt) >=
            (BNX_NSEG_RSVD + BNX_NSEG_SPARE))
                ifq_clr_oactive(&ifp->if_snd);
 
-       if (txr->bnx_txcnt == 0)
+       if (txr->bnx_tx_cnt == 0)
                ifp->if_timer = 0;
 
        if (!ifq_is_empty(&ifp->if_snd))
@@ -2540,14 +2546,14 @@ bnx_npoll(struct ifnet *ifp, struct ifpoll_info *info)
 {
        struct bnx_softc *sc = ifp->if_softc;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        if (info != NULL) {
                int cpuid = sc->bnx_npoll.ifpc_cpuid;
 
                info->ifpi_rx[cpuid].poll_func = bnx_npoll_compat;
                info->ifpi_rx[cpuid].arg = NULL;
-               info->ifpi_rx[cpuid].serializer = ifp->if_serializer;
+               info->ifpi_rx[cpuid].serializer = &sc->bnx_main_serialize;
 
                if (ifp->if_flags & IFF_RUNNING)
                        bnx_disable_intr(sc);
@@ -2555,7 +2561,7 @@ bnx_npoll(struct ifnet *ifp, struct ifpoll_info *info)
        } else {
                if (ifp->if_flags & IFF_RUNNING)
                        bnx_enable_intr(sc);
-               ifq_set_cpuid(&ifp->if_snd, sc->bnx_intr_cpuid);
+               ifq_set_cpuid(&ifp->if_snd, sc->bnx_tx_ring[0].bnx_tx_cpuid);
        }
 }
 
@@ -2568,7 +2574,7 @@ bnx_npoll_compat(struct ifnet *ifp, void *arg __unused, int cycle)
        struct bge_status_block *sblk = sc->bnx_ldata.bnx_status_block;
        uint16_t rx_prod, tx_cons;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_SERIALIZED(&sc->bnx_main_serialize);
 
        if (sc->bnx_npoll.ifpc_stcount-- == 0) {
                sc->bnx_npoll.ifpc_stcount = sc->bnx_npoll.ifpc_stfrac;
@@ -2586,14 +2592,17 @@ bnx_npoll_compat(struct ifnet *ifp, void *arg __unused, int cycle)
         */
        cpu_lfence();
 
-       rx_prod = sblk->bge_idx[0].bge_rx_prod_idx;
-       tx_cons = sblk->bge_idx[0].bge_tx_cons_idx;
-
+       lwkt_serialize_enter(&ret->bnx_rx_ret_serialize);
+       rx_prod = *ret->bnx_rx_considx;
        if (ret->bnx_rx_saved_considx != rx_prod)
                bnx_rxeof(ret, rx_prod, cycle);
+       lwkt_serialize_exit(&ret->bnx_rx_ret_serialize);
 
+       lwkt_serialize_enter(&txr->bnx_tx_serialize);
+       tx_cons = *txr->bnx_tx_considx;
        if (txr->bnx_tx_saved_considx != tx_cons)
                bnx_txeof(txr, tx_cons);
+       lwkt_serialize_exit(&txr->bnx_tx_serialize);
 }
 
 #endif /* IFPOLL_ENABLE */
@@ -2644,9 +2653,10 @@ bnx_intr(struct bnx_softc *sc)
 {
        struct ifnet *ifp = &sc->arpcom.ac_if;
        struct bge_status_block *sblk = sc->bnx_ldata.bnx_status_block;
-       uint16_t rx_prod, tx_cons;
        uint32_t status;
 
+       ASSERT_SERIALIZED(&sc->bnx_main_serialize);
+
        sc->bnx_status_tag = sblk->bge_status_tag;
        /*
         * Use a load fence to ensure that status_tag is saved 
@@ -2654,8 +2664,6 @@ bnx_intr(struct bnx_softc *sc)
         */
        cpu_lfence();
 
-       rx_prod = sblk->bge_idx[0].bge_rx_prod_idx;
-       tx_cons = sblk->bge_idx[0].bge_tx_cons_idx;
        status = sblk->bge_status;
 
        if ((status & BGE_STATFLAG_LINKSTATE_CHANGED) || sc->bnx_link_evt)
@@ -2664,12 +2672,19 @@ bnx_intr(struct bnx_softc *sc)
        if (ifp->if_flags & IFF_RUNNING) {
                struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0]; /* XXX */
                struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0]; /* XXX */
+               uint16_t rx_prod, tx_cons;
 
+               lwkt_serialize_enter(&ret->bnx_rx_ret_serialize);
+               rx_prod = *ret->bnx_rx_considx;
                if (ret->bnx_rx_saved_considx != rx_prod)
                        bnx_rxeof(ret, rx_prod, -1);
+               lwkt_serialize_exit(&ret->bnx_rx_ret_serialize);
 
+               lwkt_serialize_enter(&txr->bnx_tx_serialize);
+               tx_cons = *txr->bnx_tx_considx;
                if (txr->bnx_tx_saved_considx != tx_cons)
                        bnx_txeof(txr, tx_cons);
+               lwkt_serialize_exit(&txr->bnx_tx_serialize);
        }
 
        bnx_writembx(sc, BGE_MBX_IRQ0_LO, sc->bnx_status_tag << 24);
@@ -2679,9 +2694,8 @@ static void
 bnx_tick(void *xsc)
 {
        struct bnx_softc *sc = xsc;
-       struct ifnet *ifp = &sc->arpcom.ac_if;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       lwkt_serialize_enter(&sc->bnx_main_serialize);
 
        KKASSERT(mycpuid == sc->bnx_stat_cpuid);
 
@@ -2701,7 +2715,7 @@ bnx_tick(void *xsc)
 
        callout_reset(&sc->bnx_stat_timer, hz, bnx_tick, sc);
 
-       lwkt_serialize_exit(ifp->if_serializer);
+       lwkt_serialize_exit(&sc->bnx_main_serialize);
 }
 
 static void
@@ -2777,7 +2791,7 @@ bnx_encap(struct bnx_tx_ring *txr, struct mbuf **m_head0, uint32_t *txidx,
        idx = *txidx;
        map = txr->bnx_tx_buf[idx].bnx_tx_dmamap;
 
-       maxsegs = (BGE_TX_RING_CNT - txr->bnx_txcnt) - BNX_NSEG_RSVD;
+       maxsegs = (BGE_TX_RING_CNT - txr->bnx_tx_cnt) - BNX_NSEG_RSVD;
        KASSERT(maxsegs >= BNX_NSEG_SPARE,
                ("not enough segments %d", maxsegs));
 
@@ -2856,7 +2870,7 @@ bnx_encap(struct bnx_tx_ring *txr, struct mbuf **m_head0, uint32_t *txidx,
        txr->bnx_tx_buf[*txidx].bnx_tx_dmamap = txr->bnx_tx_buf[idx].bnx_tx_dmamap;
        txr->bnx_tx_buf[idx].bnx_tx_dmamap = map;
        txr->bnx_tx_buf[idx].bnx_tx_mbuf = m_head;
-       txr->bnx_txcnt += nsegs;
+       txr->bnx_tx_cnt += nsegs;
 
        BNX_INC(idx, BGE_TX_RING_CNT);
        *txidx = idx;
@@ -2882,6 +2896,7 @@ bnx_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
        int nsegs = 0;
 
        ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq);
+       ASSERT_SERIALIZED(&txr->bnx_tx_serialize);
 
        if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd))
                return;
@@ -2895,7 +2910,7 @@ bnx_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
                 * sure there are BGE_NSEG_SPARE descriptors for
                 * jumbo buffers' or TSO segments' defragmentation.
                 */
-               if ((BGE_TX_RING_CNT - txr->bnx_txcnt) <
+               if ((BGE_TX_RING_CNT - txr->bnx_tx_cnt) <
                    (BNX_NSEG_RSVD + BNX_NSEG_SPARE)) {
                        ifq_set_oactive(&ifp->if_snd);
                        break;
@@ -2946,7 +2961,7 @@ bnx_init(void *xsc)
        uint32_t mode;
        int i;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        /* Cancel pending I/O and flush buffers. */
        bnx_stop(sc);
@@ -3028,7 +3043,7 @@ bnx_init(void *xsc)
        else
                CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 2);
 
-       if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI) {
+       if (sc->bnx_intr_type == PCI_INTR_TYPE_MSI) {
                if (bootverbose) {
                        if_printf(ifp, "MSI_MODE: %#x\n",
                            CSR_READ_4(sc, BGE_MSI_MODE));
@@ -3158,7 +3173,7 @@ bnx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
        struct ifreq *ifr = (struct ifreq *)data;
        int mask, error = 0;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        switch (command) {
        case SIOCSIFMTU:
@@ -3252,7 +3267,7 @@ bnx_watchdog(struct ifnet *ifp)
        IFNET_STAT_INC(ifp, oerrors, 1);
 
        if (!ifq_is_empty(&ifp->if_snd))
-               if_devstart(ifp);
+               if_devstart_sched(ifp);
 }
 
 /*
@@ -3265,7 +3280,7 @@ bnx_stop(struct bnx_softc *sc)
        struct ifnet *ifp = &sc->arpcom.ac_if;
        int i;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        callout_stop(&sc->bnx_stat_timer);
 
@@ -3336,10 +3351,10 @@ bnx_shutdown(device_t dev)
        struct bnx_softc *sc = device_get_softc(dev);
        struct ifnet *ifp = &sc->arpcom.ac_if;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
        bnx_stop(sc);
        bnx_reset(sc);
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 }
 
 static int
@@ -3348,9 +3363,9 @@ bnx_suspend(device_t dev)
        struct bnx_softc *sc = device_get_softc(dev);
        struct ifnet *ifp = &sc->arpcom.ac_if;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
        bnx_stop(sc);
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 
        return 0;
 }
@@ -3361,16 +3376,16 @@ bnx_resume(device_t dev)
        struct bnx_softc *sc = device_get_softc(dev);
        struct ifnet *ifp = &sc->arpcom.ac_if;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
 
        if (ifp->if_flags & IFF_UP) {
                bnx_init(sc);
 
                if (!ifq_is_empty(&ifp->if_snd))
-                       if_devstart(ifp);
+                       if_devstart_sched(ifp);
        }
 
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 
        return 0;
 }
@@ -3476,6 +3491,7 @@ bnx_dma_alloc(device_t dev)
         * Create DMA tag and maps for RX mbufs.
         */
        std->bnx_sc = sc;
+       lwkt_serialize_init(&std->bnx_rx_std_serialize);
        error = bus_dma_tag_create(sc->bnx_cdata.bnx_parent_tag, 1, 0,
            BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
            NULL, NULL, MCLBYTES, 1, MCLBYTES,
@@ -3528,6 +3544,11 @@ bnx_dma_alloc(device_t dev)
 
                ret->bnx_sc = sc;
                ret->bnx_std = std;
+
+               /* XXX */
+               ret->bnx_rx_considx =
+               &sc->bnx_ldata.bnx_status_block->bge_idx[0].bge_rx_prod_idx;
+
                error = bnx_create_rx_ret_ring(ret);
                if (error) {
                        device_printf(dev,
@@ -3554,6 +3575,10 @@ bnx_dma_alloc(device_t dev)
                else
                        mbx += 0xc;
 
+               /* XXX */
+               txr->bnx_tx_considx =
+               &sc->bnx_ldata.bnx_status_block->bge_idx[0].bge_tx_cons_idx;
+
                error = bnx_create_tx_ring(txr);
                if (error) {
                        device_printf(dev,
@@ -3785,7 +3810,7 @@ bnx_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *coal,
        struct ifnet *ifp = &sc->arpcom.ac_if;
        int error = 0, v;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
 
        v = *coal;
        error = sysctl_handle_int(oidp, &v, 0, req);
@@ -3801,7 +3826,7 @@ bnx_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *coal,
                }
        }
 
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
        return error;
 }
 
@@ -3810,7 +3835,7 @@ bnx_coal_change(struct bnx_softc *sc)
 {
        struct ifnet *ifp = &sc->arpcom.ac_if;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        if (sc->bnx_coal_chg & BNX_RX_COAL_TICKS_CHG) {
                CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS,
@@ -3888,52 +3913,59 @@ bnx_coal_change(struct bnx_softc *sc)
 }
 
 static void
-bnx_intr_check(void *xsc)
+bnx_check_intr(void *xintr)
 {
-       struct bnx_softc *sc = xsc;
-       struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0]; /* XXX */
-       struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0]; /* XXX */
-       struct ifnet *ifp = &sc->arpcom.ac_if;
-       struct bge_status_block *sblk = sc->bnx_ldata.bnx_status_block;
+       struct bnx_intr_data *intr = xintr;
+       struct bnx_rx_ret_ring *ret;
+       struct bnx_tx_ring *txr;
+       struct ifnet *ifp;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       lwkt_serialize_enter(intr->bnx_intr_serialize);
 
-       KKASSERT(mycpuid == sc->bnx_intr_cpuid);
+       KKASSERT(mycpuid == intr->bnx_intr_cpuid);
 
+       ifp = &intr->bnx_sc->arpcom.ac_if;
        if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) != IFF_RUNNING) {
-               lwkt_serialize_exit(ifp->if_serializer);
+               lwkt_serialize_exit(intr->bnx_intr_serialize);
                return;
        }
 
-       if (sblk->bge_idx[0].bge_rx_prod_idx != ret->bnx_rx_saved_considx ||
-           sblk->bge_idx[0].bge_tx_cons_idx != txr->bnx_tx_saved_considx) {
-               if (sc->bnx_rx_check_considx == ret->bnx_rx_saved_considx &&
-                   sc->bnx_tx_check_considx == txr->bnx_tx_saved_considx) {
-                       if (!sc->bnx_intr_maylose) {
-                               sc->bnx_intr_maylose = TRUE;
+       txr = intr->bnx_txr;
+       ret = intr->bnx_ret;
+
+       if (*ret->bnx_rx_considx != ret->bnx_rx_saved_considx ||
+           *txr->bnx_tx_considx != txr->bnx_tx_saved_considx) {
+               if (intr->bnx_rx_check_considx == ret->bnx_rx_saved_considx &&
+                   intr->bnx_tx_check_considx == txr->bnx_tx_saved_considx) {
+                       if (!intr->bnx_intr_maylose) {
+                               intr->bnx_intr_maylose = TRUE;
                                goto done;
                        }
                        if (bootverbose)
                                if_printf(ifp, "lost interrupt\n");
-                       bnx_msi(sc);
+                       intr->bnx_intr_func(intr->bnx_intr_arg);
                }
        }
-       sc->bnx_intr_maylose = FALSE;
-       sc->bnx_rx_check_considx = ret->bnx_rx_saved_considx;
-       sc->bnx_tx_check_considx = txr->bnx_tx_saved_considx;
+       intr->bnx_intr_maylose = FALSE;
+       intr->bnx_rx_check_considx = ret->bnx_rx_saved_considx;
+       intr->bnx_tx_check_considx = txr->bnx_tx_saved_considx;
 
 done:
-       callout_reset(&sc->bnx_intr_timer, BNX_INTR_CKINTVL,
-           bnx_intr_check, sc);
-       lwkt_serialize_exit(ifp->if_serializer);
+       callout_reset(&intr->bnx_intr_timer, BNX_INTR_CKINTVL,
+           intr->bnx_intr_check, intr);
+       lwkt_serialize_exit(intr->bnx_intr_serialize);
 }
 
 static void
 bnx_enable_intr(struct bnx_softc *sc)
 {
        struct ifnet *ifp = &sc->arpcom.ac_if;
+       int i;
 
-       lwkt_serialize_handler_enable(ifp->if_serializer);
+       for (i = 0; i < sc->bnx_intr_cnt; ++i) {
+               lwkt_serialize_handler_enable(
+                   sc->bnx_intr_data[i].bnx_intr_serialize);
+       }
 
        /*
         * Enable interrupt.
@@ -3958,23 +3990,35 @@ bnx_enable_intr(struct bnx_softc *sc)
        BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_SET);
 
        if (sc->bnx_flags & BNX_FLAG_STATUSTAG_BUG) {
-               sc->bnx_intr_maylose = FALSE;
-               sc->bnx_rx_check_considx = 0;
-               sc->bnx_tx_check_considx = 0;
-
                if (bootverbose)
                        if_printf(ifp, "status tag bug workaround\n");
 
-               /* 10ms check interval */
-               callout_reset_bycpu(&sc->bnx_intr_timer, BNX_INTR_CKINTVL,
-                   bnx_intr_check, sc, sc->bnx_intr_cpuid);
+               for (i = 0; i < sc->bnx_intr_cnt; ++i) {
+                       struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
+
+                       intr->bnx_intr_maylose = FALSE;
+                       intr->bnx_rx_check_considx = 0;
+                       intr->bnx_tx_check_considx = 0;
+                       callout_reset_bycpu(&intr->bnx_intr_timer,
+                           BNX_INTR_CKINTVL, intr->bnx_intr_check, intr,
+                           intr->bnx_intr_cpuid);
+               }
        }
 }
 
 static void
 bnx_disable_intr(struct bnx_softc *sc)
 {
-       struct ifnet *ifp = &sc->arpcom.ac_if;
+       int i;
+
+       for (i = 0; i < sc->bnx_intr_cnt; ++i) {
+               struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
+
+               callout_stop(&intr->bnx_intr_timer);
+               intr->bnx_intr_maylose = FALSE;
+               intr->bnx_rx_check_considx = 0;
+               intr->bnx_tx_check_considx = 0;
+       }
 
        /*
         * Mask the interrupt when we start polling.
@@ -3987,14 +4031,11 @@ bnx_disable_intr(struct bnx_softc *sc)
         */
        bnx_writembx(sc, BGE_MBX_IRQ0_LO, 1);
 
-       callout_stop(&sc->bnx_intr_timer);
-       sc->bnx_intr_maylose = FALSE;
-       sc->bnx_rx_check_considx = 0;
-       sc->bnx_tx_check_considx = 0;
-
        sc->bnx_npoll.ifpc_stcount = 0;
-
-       lwkt_serialize_handler_disable(ifp->if_serializer);
+       for (i = 0; i < sc->bnx_intr_cnt; ++i) {
+               lwkt_serialize_handler_disable(
+                   sc->bnx_intr_data[i].bnx_intr_serialize);
+       }
 }
 
 static int
@@ -4215,6 +4256,8 @@ bnx_create_tx_ring(struct bnx_tx_ring *txr)
        bus_size_t txmaxsz, txmaxsegsz;
        int i, error;
 
+       lwkt_serialize_init(&txr->bnx_tx_serialize);
+
        /*
         * Create DMA tag and maps for TX mbufs.
         */
@@ -4314,7 +4357,7 @@ bnx_sysctl_force_defrag(SYSCTL_HANDLER_ARGS)
        if (error || req->newptr == NULL)
                return error;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
        for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
                txr = &sc->bnx_tx_ring[i];
                if (defrag)
@@ -4322,7 +4365,7 @@ bnx_sysctl_force_defrag(SYSCTL_HANDLER_ARGS)
                else
                        txr->bnx_tx_flags &= ~BNX_TX_FLAG_FORCE_DEFRAG;
        }
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 
        return 0;
 }
@@ -4340,10 +4383,10 @@ bnx_sysctl_tx_wreg(SYSCTL_HANDLER_ARGS)
        if (error || req->newptr == NULL)
                return error;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
        for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
                sc->bnx_tx_ring[i].bnx_tx_wreg = tx_wreg;
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 
        return 0;
 }
@@ -4353,6 +4396,8 @@ bnx_create_rx_ret_ring(struct bnx_rx_ret_ring *ret)
 {
        int error;
 
+       lwkt_serialize_init(&ret->bnx_rx_ret_serialize);
+
        /*
         * Create DMA stuffs for RX return ring.
         */
@@ -4396,3 +4441,188 @@ bnx_destroy_rx_ret_ring(struct bnx_rx_ret_ring *ret)
        bnx_dma_block_free(ret->bnx_rx_ret_ring_tag,
            ret->bnx_rx_ret_ring_map, ret->bnx_rx_ret_ring);
 }
+
+static int
+bnx_alloc_intr(struct bnx_softc *sc)
+{
+       struct bnx_intr_data *intr;
+       u_int intr_flags;
+
+       sc->bnx_intr_cnt = 1;
+
+       intr = &sc->bnx_intr_data[0];
+       intr->bnx_sc = sc;
+       intr->bnx_ret = &sc->bnx_rx_ret_ring[0];
+       intr->bnx_txr = &sc->bnx_tx_ring[0];
+       intr->bnx_intr_serialize = &sc->bnx_main_serialize;
+       callout_init_mp(&intr->bnx_intr_timer);
+       intr->bnx_intr_check = bnx_check_intr;
+
+       sc->bnx_intr_type = pci_alloc_1intr(sc->bnx_dev, bnx_msi_enable,
+           &intr->bnx_intr_rid, &intr_flags);
+
+       intr->bnx_intr_res = bus_alloc_resource_any(sc->bnx_dev, SYS_RES_IRQ,
+           &intr->bnx_intr_rid, intr_flags);
+       if (intr->bnx_intr_res == NULL) {
+               device_printf(sc->bnx_dev, "could not alloc interrupt\n");
+               return ENXIO;
+       }
+
+       if (sc->bnx_intr_type == PCI_INTR_TYPE_MSI) {
+               sc->bnx_flags |= BNX_FLAG_ONESHOT_MSI;
+               bnx_enable_msi(sc);
+
+               if (sc->bnx_flags & BNX_FLAG_ONESHOT_MSI) {
+                       intr->bnx_intr_func = bnx_msi_oneshot;
+                       if (bootverbose)
+                               device_printf(sc->bnx_dev, "oneshot MSI\n");
+               } else {
+                       intr->bnx_intr_func = bnx_msi;
+               }
+       } else {
+               intr->bnx_intr_func = bnx_intr_legacy;
+       }
+       intr->bnx_intr_arg = sc;
+       intr->bnx_intr_cpuid = rman_get_cpuid(intr->bnx_intr_res);
+
+       intr->bnx_txr->bnx_tx_cpuid = intr->bnx_intr_cpuid;
+
+       return 0;
+}
+
+static int
+bnx_setup_intr(struct bnx_softc *sc)
+{
+       int error, i;
+
+       for (i = 0; i < sc->bnx_intr_cnt; ++i) {
+               struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
+
+               error = bus_setup_intr_descr(sc->bnx_dev, intr->bnx_intr_res,
+                   INTR_MPSAFE, intr->bnx_intr_func, intr->bnx_intr_arg,
+                   &intr->bnx_intr_hand, intr->bnx_intr_serialize,
+                   intr->bnx_intr_desc);
+               if (error) {
+                       device_printf(sc->bnx_dev,
+                           "could not set up %dth intr\n", i);
+                       bnx_teardown_intr(sc, i);
+                       return error;
+               }
+       }
+       return 0;
+}
+
+static void
+bnx_teardown_intr(struct bnx_softc *sc, int cnt)
+{
+       int i;
+
+       for (i = 0; i < cnt; ++i) {
+               struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
+
+               bus_teardown_intr(sc->bnx_dev, intr->bnx_intr_res,
+                   intr->bnx_intr_hand);
+       }
+}
+
+static void
+bnx_free_intr(struct bnx_softc *sc)
+{
+       struct bnx_intr_data *intr;
+
+       KKASSERT(sc->bnx_intr_cnt <= 1);
+       intr = &sc->bnx_intr_data[0];
+
+       if (intr->bnx_intr_res != NULL) {
+               bus_release_resource(sc->bnx_dev, SYS_RES_IRQ,
+                   intr->bnx_intr_rid, intr->bnx_intr_res);
+       }
+       if (sc->bnx_intr_type == PCI_INTR_TYPE_MSI)
+               pci_release_msi(sc->bnx_dev);
+}
+
+static void
+bnx_setup_serialize(struct bnx_softc *sc)
+{
+       int i, j;
+
+       /*
+        * Allocate serializer array
+        */
+
+       /* Main + RX STD + TX + RX RET */
+       sc->bnx_serialize_cnt = 1 + 1 + sc->bnx_tx_ringcnt + sc->bnx_rx_retcnt;
+
+       sc->bnx_serialize =
+           kmalloc(sc->bnx_serialize_cnt * sizeof(struct lwkt_serialize *),
+               M_DEVBUF, M_WAITOK | M_ZERO);
+
+       /*
+        * Setup serializers
+        *
+        * NOTE: Order is critical
+        */
+
+       i = 0;
+
+       KKASSERT(i < sc->bnx_serialize_cnt);
+       sc->bnx_serialize[i++] = &sc->bnx_main_serialize;
+
+       KKASSERT(i < sc->bnx_serialize_cnt);
+       sc->bnx_serialize[i++] = &sc->bnx_rx_std_ring.bnx_rx_std_serialize;
+
+       for (j = 0; j < sc->bnx_rx_retcnt; ++j) {
+               KKASSERT(i < sc->bnx_serialize_cnt);
+               sc->bnx_serialize[i++] =
+                   &sc->bnx_rx_ret_ring[j].bnx_rx_ret_serialize;
+       }
+
+       for (j = 0; j < sc->bnx_tx_ringcnt; ++j) {
+               KKASSERT(i < sc->bnx_serialize_cnt);
+               sc->bnx_serialize[i++] =
+                   &sc->bnx_tx_ring[j].bnx_tx_serialize;
+       }
+
+       KKASSERT(i == sc->bnx_serialize_cnt);
+}
+
+static void
+bnx_serialize(struct ifnet *ifp, enum ifnet_serialize slz)
+{
+       struct bnx_softc *sc = ifp->if_softc;
+
+       ifnet_serialize_array_enter(sc->bnx_serialize,
+           sc->bnx_serialize_cnt, slz);
+}
+
+static void
+bnx_deserialize(struct ifnet *ifp, enum ifnet_serialize slz)
+{
+       struct bnx_softc *sc = ifp->if_softc;
+
+       ifnet_serialize_array_exit(sc->bnx_serialize,
+           sc->bnx_serialize_cnt, slz);
+}
+
+static int
+bnx_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz)
+{
+       struct bnx_softc *sc = ifp->if_softc;
+
+       return ifnet_serialize_array_try(sc->bnx_serialize,
+           sc->bnx_serialize_cnt, slz);
+}
+
+#ifdef INVARIANTS
+
+static void
+bnx_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz,
+    boolean_t serialized)
+{
+       struct bnx_softc *sc = ifp->if_softc;
+
+       ifnet_serialize_array_assert(sc->bnx_serialize, sc->bnx_serialize_cnt,
+           slz, serialized);
+}
+
+#endif /* INVARIANTS */