bnx: Split RX/TX serializers
[dragonfly.git] / sys / dev / netif / bnx / if_bnx.c
index 4d0cb79..170fcac 100644 (file)
@@ -159,7 +159,12 @@ static void        bnx_intr(struct bnx_softc *);
 static void    bnx_enable_intr(struct bnx_softc *);
 static void    bnx_disable_intr(struct bnx_softc *);
 static void    bnx_txeof(struct bnx_tx_ring *, uint16_t);
-static void    bnx_rxeof(struct bnx_softc *, uint16_t, int);
+static void    bnx_rxeof(struct bnx_rx_ret_ring *, uint16_t, int);
+static int     bnx_alloc_intr(struct bnx_softc *);
+static int     bnx_setup_intr(struct bnx_softc *);
+static void    bnx_free_intr(struct bnx_softc *);
+static void    bnx_teardown_intr(struct bnx_softc *, int);
+static void    bnx_check_intr(void *);
 
 static void    bnx_start(struct ifnet *, struct ifaltq_subque *);
 static int     bnx_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
@@ -169,6 +174,13 @@ static void        bnx_watchdog(struct ifnet *);
 static int     bnx_ifmedia_upd(struct ifnet *);
 static void    bnx_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 static void    bnx_tick(void *);
+static void    bnx_serialize(struct ifnet *, enum ifnet_serialize);
+static void    bnx_deserialize(struct ifnet *, enum ifnet_serialize);
+static int     bnx_tryserialize(struct ifnet *, enum ifnet_serialize);
+#ifdef INVARIANTS
+static void    bnx_serialize_assert(struct ifnet *, enum ifnet_serialize,
+                   boolean_t);
+#endif
 
 static int     bnx_alloc_jumbo_mem(struct bnx_softc *);
 static void    bnx_free_jumbo_mem(struct bnx_softc *);
@@ -176,19 +188,21 @@ static struct bnx_jslot
                *bnx_jalloc(struct bnx_softc *);
 static void    bnx_jfree(void *);
 static void    bnx_jref(void *);
-static int     bnx_newbuf_std(struct bnx_softc *, int, int);
+static int     bnx_newbuf_std(struct bnx_rx_ret_ring *, int, int);
 static int     bnx_newbuf_jumbo(struct bnx_softc *, int, int);
-static void    bnx_setup_rxdesc_std(struct bnx_softc *, int);
+static void    bnx_setup_rxdesc_std(struct bnx_rx_std_ring *, int);
 static void    bnx_setup_rxdesc_jumbo(struct bnx_softc *, int);
-static int     bnx_init_rx_ring_std(struct bnx_softc *);
-static void    bnx_free_rx_ring_std(struct bnx_softc *);
+static int     bnx_init_rx_ring_std(struct bnx_rx_std_ring *);
+static void    bnx_free_rx_ring_std(struct bnx_rx_std_ring *);
 static int     bnx_init_rx_ring_jumbo(struct bnx_softc *);
 static void    bnx_free_rx_ring_jumbo(struct bnx_softc *);
 static void    bnx_free_tx_ring(struct bnx_tx_ring *);
 static int     bnx_init_tx_ring(struct bnx_tx_ring *);
 static int     bnx_create_tx_ring(struct bnx_tx_ring *);
 static void    bnx_destroy_tx_ring(struct bnx_tx_ring *);
-static int     bnx_dma_alloc(struct bnx_softc *);
+static int     bnx_create_rx_ret_ring(struct bnx_rx_ret_ring *);
+static void    bnx_destroy_rx_ret_ring(struct bnx_rx_ret_ring *);
+static int     bnx_dma_alloc(device_t);
 static void    bnx_dma_free(struct bnx_softc *);
 static int     bnx_dma_block_alloc(struct bnx_softc *, bus_size_t,
                    bus_dma_tag_t *, bus_dmamap_t *, void **, bus_addr_t *);
@@ -199,6 +213,7 @@ static int  bnx_encap(struct bnx_tx_ring *, struct mbuf **,
                    uint32_t *, int *);
 static int     bnx_setup_tso(struct bnx_tx_ring *, struct mbuf **,
                    uint16_t *, uint16_t *);
+static void    bnx_setup_serialize(struct bnx_softc *);
 
 static void    bnx_reset(struct bnx_softc *);
 static int     bnx_chipinit(struct bnx_softc *);
@@ -233,6 +248,7 @@ static int  bnx_get_eaddr(struct bnx_softc *, uint8_t[]);
 
 static void    bnx_coal_change(struct bnx_softc *);
 static int     bnx_sysctl_force_defrag(SYSCTL_HANDLER_ARGS);
+static int     bnx_sysctl_tx_wreg(SYSCTL_HANDLER_ARGS);
 static int     bnx_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS);
 static int     bnx_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS);
 static int     bnx_sysctl_rx_coal_bds(SYSCTL_HANDLER_ARGS);
@@ -689,12 +705,13 @@ bnx_jfree(void *arg)
  * Intialize a standard receive ring descriptor.
  */
 static int
-bnx_newbuf_std(struct bnx_softc *sc, int i, int init)
+bnx_newbuf_std(struct bnx_rx_ret_ring *ret, int i, int init)
 {
        struct mbuf *m_new = NULL;
        bus_dma_segment_t seg;
        bus_dmamap_t map;
        int error, nsegs;
+       struct bnx_rx_buf *rb;
 
        m_new = m_getcl(init ? MB_WAIT : MB_DONTWAIT, MT_DATA, M_PKTHDR);
        if (m_new == NULL)
@@ -702,45 +719,44 @@ bnx_newbuf_std(struct bnx_softc *sc, int i, int init)
        m_new->m_len = m_new->m_pkthdr.len = MCLBYTES;
        m_adj(m_new, ETHER_ALIGN);
 
-       error = bus_dmamap_load_mbuf_segment(sc->bnx_cdata.bnx_rx_mtag,
-                       sc->bnx_cdata.bnx_rx_tmpmap, m_new,
-                       &seg, 1, &nsegs, BUS_DMA_NOWAIT);
+       error = bus_dmamap_load_mbuf_segment(ret->bnx_rx_mtag,
+           ret->bnx_rx_tmpmap, m_new, &seg, 1, &nsegs, BUS_DMA_NOWAIT);
        if (error) {
                m_freem(m_new);
                return error;
        }
 
+       rb = &ret->bnx_std->bnx_rx_std_buf[i];
+
        if (!init) {
-               bus_dmamap_sync(sc->bnx_cdata.bnx_rx_mtag,
-                               sc->bnx_cdata.bnx_rx_std_dmamap[i],
-                               BUS_DMASYNC_POSTREAD);
-               bus_dmamap_unload(sc->bnx_cdata.bnx_rx_mtag,
-                       sc->bnx_cdata.bnx_rx_std_dmamap[i]);
+               bus_dmamap_sync(ret->bnx_rx_mtag, rb->bnx_rx_dmamap,
+                   BUS_DMASYNC_POSTREAD);
+               bus_dmamap_unload(ret->bnx_rx_mtag, rb->bnx_rx_dmamap);
        }
 
-       map = sc->bnx_cdata.bnx_rx_tmpmap;
-       sc->bnx_cdata.bnx_rx_tmpmap = sc->bnx_cdata.bnx_rx_std_dmamap[i];
-       sc->bnx_cdata.bnx_rx_std_dmamap[i] = map;
+       map = ret->bnx_rx_tmpmap;
+       ret->bnx_rx_tmpmap = rb->bnx_rx_dmamap;
+       rb->bnx_rx_dmamap = map;
 
-       sc->bnx_cdata.bnx_rx_std_chain[i].bnx_mbuf = m_new;
-       sc->bnx_cdata.bnx_rx_std_chain[i].bnx_paddr = seg.ds_addr;
+       rb->bnx_rx_mbuf = m_new;
+       rb->bnx_rx_paddr = seg.ds_addr;
 
-       bnx_setup_rxdesc_std(sc, i);
+       bnx_setup_rxdesc_std(ret->bnx_std, i);
        return 0;
 }
 
 static void
-bnx_setup_rxdesc_std(struct bnx_softc *sc, int i)
+bnx_setup_rxdesc_std(struct bnx_rx_std_ring *std, int i)
 {
-       struct bnx_rxchain *rc;
+       const struct bnx_rx_buf *rb;
        struct bge_rx_bd *r;
 
-       rc = &sc->bnx_cdata.bnx_rx_std_chain[i];
-       r = &sc->bnx_ldata.bnx_rx_std_ring[i];
+       rb = &std->bnx_rx_std_buf[i];
+       r = &std->bnx_rx_std_ring[i];
 
-       r->bge_addr.bge_addr_lo = BGE_ADDR_LO(rc->bnx_paddr);
-       r->bge_addr.bge_addr_hi = BGE_ADDR_HI(rc->bnx_paddr);
-       r->bge_len = rc->bnx_mbuf->m_len;
+       r->bge_addr.bge_addr_lo = BGE_ADDR_LO(rb->bnx_rx_paddr);
+       r->bge_addr.bge_addr_hi = BGE_ADDR_HI(rb->bnx_rx_paddr);
+       r->bge_len = rb->bnx_rx_mbuf->m_len;
        r->bge_idx = i;
        r->bge_flags = BGE_RXBDFLAG_END;
 }
@@ -785,8 +801,8 @@ bnx_newbuf_jumbo(struct bnx_softc *sc, int i, int init)
        paddr += ETHER_ALIGN;
 
        /* Save necessary information */
-       sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_mbuf = m_new;
-       sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_paddr = paddr;
+       sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_rx_mbuf = m_new;
+       sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_rx_paddr = paddr;
 
        /* Set up the descriptor. */
        bnx_setup_rxdesc_jumbo(sc, i);
@@ -797,51 +813,50 @@ static void
 bnx_setup_rxdesc_jumbo(struct bnx_softc *sc, int i)
 {
        struct bge_rx_bd *r;
-       struct bnx_rxchain *rc;
+       struct bnx_rx_buf *rc;
 
        r = &sc->bnx_ldata.bnx_rx_jumbo_ring[i];
        rc = &sc->bnx_cdata.bnx_rx_jumbo_chain[i];
 
-       r->bge_addr.bge_addr_lo = BGE_ADDR_LO(rc->bnx_paddr);
-       r->bge_addr.bge_addr_hi = BGE_ADDR_HI(rc->bnx_paddr);
-       r->bge_len = rc->bnx_mbuf->m_len;
+       r->bge_addr.bge_addr_lo = BGE_ADDR_LO(rc->bnx_rx_paddr);
+       r->bge_addr.bge_addr_hi = BGE_ADDR_HI(rc->bnx_rx_paddr);
+       r->bge_len = rc->bnx_rx_mbuf->m_len;
        r->bge_idx = i;
        r->bge_flags = BGE_RXBDFLAG_END|BGE_RXBDFLAG_JUMBO_RING;
 }
 
 static int
-bnx_init_rx_ring_std(struct bnx_softc *sc)
+bnx_init_rx_ring_std(struct bnx_rx_std_ring *std)
 {
        int i, error;
 
        for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
-               error = bnx_newbuf_std(sc, i, 1);
+               /* Use the first RX return ring's tmp RX mbuf DMA map */
+               error = bnx_newbuf_std(&std->bnx_sc->bnx_rx_ret_ring[0], i, 1);
                if (error)
                        return error;
        }
 
-       sc->bnx_std = BGE_STD_RX_RING_CNT - 1;
-       bnx_writembx(sc, BGE_MBX_RX_STD_PROD_LO, sc->bnx_std);
+       std->bnx_rx_std = BGE_STD_RX_RING_CNT - 1;
+       bnx_writembx(std->bnx_sc, BGE_MBX_RX_STD_PROD_LO, std->bnx_rx_std);
 
        return(0);
 }
 
 static void
-bnx_free_rx_ring_std(struct bnx_softc *sc)
+bnx_free_rx_ring_std(struct bnx_rx_std_ring *std)
 {
        int i;
 
        for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
-               struct bnx_rxchain *rc = &sc->bnx_cdata.bnx_rx_std_chain[i];
+               struct bnx_rx_buf *rb = &std->bnx_rx_std_buf[i];
 
-               if (rc->bnx_mbuf != NULL) {
-                       bus_dmamap_unload(sc->bnx_cdata.bnx_rx_mtag,
-                                         sc->bnx_cdata.bnx_rx_std_dmamap[i]);
-                       m_freem(rc->bnx_mbuf);
-                       rc->bnx_mbuf = NULL;
+               if (rb->bnx_rx_mbuf != NULL) {
+                       bus_dmamap_unload(std->bnx_rx_mtag, rb->bnx_rx_dmamap);
+                       m_freem(rb->bnx_rx_mbuf);
+                       rb->bnx_rx_mbuf = NULL;
                }
-               bzero(&sc->bnx_ldata.bnx_rx_std_ring[i],
-                   sizeof(struct bge_rx_bd));
+               bzero(&std->bnx_rx_std_ring[i], sizeof(struct bge_rx_bd));
        }
 }
 
@@ -874,11 +889,11 @@ bnx_free_rx_ring_jumbo(struct bnx_softc *sc)
        int i;
 
        for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
-               struct bnx_rxchain *rc = &sc->bnx_cdata.bnx_rx_jumbo_chain[i];
+               struct bnx_rx_buf *rc = &sc->bnx_cdata.bnx_rx_jumbo_chain[i];
 
-               if (rc->bnx_mbuf != NULL) {
-                       m_freem(rc->bnx_mbuf);
-                       rc->bnx_mbuf = NULL;
+               if (rc->bnx_rx_mbuf != NULL) {
+                       m_freem(rc->bnx_rx_mbuf);
+                       rc->bnx_rx_mbuf = NULL;
                }
                bzero(&sc->bnx_ldata.bnx_rx_jumbo_ring[i],
                    sizeof(struct bge_rx_bd));
@@ -891,11 +906,13 @@ bnx_free_tx_ring(struct bnx_tx_ring *txr)
        int i;
 
        for (i = 0; i < BGE_TX_RING_CNT; i++) {
-               if (txr->bnx_tx_chain[i] != NULL) {
+               struct bnx_tx_buf *buf = &txr->bnx_tx_buf[i];
+
+               if (buf->bnx_tx_mbuf != NULL) {
                        bus_dmamap_unload(txr->bnx_tx_mtag,
-                           txr->bnx_tx_dmamap[i]);
-                       m_freem(txr->bnx_tx_chain[i]);
-                       txr->bnx_tx_chain[i] = NULL;
+                           buf->bnx_tx_dmamap);
+                       m_freem(buf->bnx_tx_mbuf);
+                       buf->bnx_tx_mbuf = NULL;
                }
                bzero(&txr->bnx_tx_ring[i], sizeof(struct bge_tx_bd));
        }
@@ -905,13 +922,12 @@ bnx_free_tx_ring(struct bnx_tx_ring *txr)
 static int
 bnx_init_tx_ring(struct bnx_tx_ring *txr)
 {
-       txr->bnx_txcnt = 0;
+       txr->bnx_tx_cnt = 0;
        txr->bnx_tx_saved_considx = 0;
        txr->bnx_tx_prodidx = 0;
 
        /* Initialize transmit producer index for host-memory send ring. */
-       bnx_writembx(txr->bnx_sc, BGE_MBX_TX_HOST_PROD0_LO,
-           txr->bnx_tx_prodidx);
+       bnx_writembx(txr->bnx_sc, txr->bnx_tx_mbx, txr->bnx_tx_prodidx);
 
        return(0);
 }
@@ -1077,6 +1093,7 @@ static int
 bnx_blockinit(struct bnx_softc *sc)
 {
        struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0];
+       struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0];
        struct bge_rcb *rcb;
        bus_size_t vrcb;
        bge_hostaddr taddr;
@@ -1191,9 +1208,9 @@ bnx_blockinit(struct bnx_softc *sc)
        /* Initialize the standard receive producer ring control block. */
        rcb = &sc->bnx_ldata.bnx_info.bnx_std_rx_rcb;
        rcb->bge_hostaddr.bge_addr_lo =
-           BGE_ADDR_LO(sc->bnx_ldata.bnx_rx_std_ring_paddr);
+           BGE_ADDR_LO(sc->bnx_rx_std_ring.bnx_rx_std_ring_paddr);
        rcb->bge_hostaddr.bge_addr_hi =
-           BGE_ADDR_HI(sc->bnx_ldata.bnx_rx_std_ring_paddr);
+           BGE_ADDR_HI(sc->bnx_rx_std_ring.bnx_rx_std_ring_paddr);
        if (BNX_IS_57765_PLUS(sc)) {
                /*
                 * Bits 31-16: Programmable ring size (2048, 1024, 512, .., 32)
@@ -1345,7 +1362,7 @@ bnx_blockinit(struct bnx_softc *sc)
         * within the host, so the nicaddr field in the RCB isn't used.
         */
        vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB;
-       BGE_HOSTADDR(taddr, sc->bnx_ldata.bnx_rx_return_ring_paddr);
+       BGE_HOSTADDR(taddr, ret->bnx_rx_ret_ring_paddr);
        RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, taddr.bge_addr_hi);
        RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, taddr.bge_addr_lo);
        RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
@@ -1666,22 +1683,19 @@ bnx_attach(device_t dev)
        struct ifnet *ifp;
        struct bnx_softc *sc;
        uint32_t hwcfg = 0;
-       int error = 0, rid, capmask;
+       int error = 0, rid, capmask, i;
        uint8_t ether_addr[ETHER_ADDR_LEN];
        uint16_t product;
-       driver_intr_t *intr_func;
        uintptr_t mii_priv = 0;
-       u_int intr_flags;
 #ifdef BNX_TSO_DEBUG
        char desc[32];
-       int i;
 #endif
 
        sc = device_get_softc(dev);
        sc->bnx_dev = dev;
        callout_init_mp(&sc->bnx_stat_timer);
-       callout_init_mp(&sc->bnx_intr_timer);
        lwkt_serialize_init(&sc->bnx_jslot_serializer);
+       lwkt_serialize_init(&sc->bnx_main_serialize);
 
        product = pci_get_device(dev);
 
@@ -1803,7 +1817,7 @@ bnx_attach(device_t dev)
                 * For the rest of the chips in these two families, we will
                 * have to poll the status block at high rate (10ms currently)
                 * to check whether the interrupt is hosed or not.
-                * See bnx_intr_check() for details.
+                * See bnx_check_intr() for details.
                 */
                sc->bnx_flags |= BNX_FLAG_STATUSTAG_BUG;
        }
@@ -1857,29 +1871,21 @@ bnx_attach(device_t dev)
 
        /* XXX */
        sc->bnx_tx_ringcnt = 1;
+       sc->bnx_rx_retcnt = 1;
 
-       error = bnx_dma_alloc(sc);
+       error = bnx_dma_alloc(dev);
        if (error)
                goto fail;
 
        /*
         * Allocate interrupt
         */
-       sc->bnx_irq_type = pci_alloc_1intr(dev, bnx_msi_enable, &sc->bnx_irq_rid,
-           &intr_flags);
-
-       sc->bnx_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->bnx_irq_rid,
-           intr_flags);
-       if (sc->bnx_irq == NULL) {
-               device_printf(dev, "couldn't map interrupt\n");
-               error = ENXIO;
+       error = bnx_alloc_intr(sc);
+       if (error)
                goto fail;
-       }
 
-       if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI) {
-               sc->bnx_flags |= BNX_FLAG_ONESHOT_MSI;
-               bnx_enable_msi(sc);
-       }
+       /* Setup serializers */
+       bnx_setup_serialize(sc);
 
        /* Set default tuneable values. */
        sc->bnx_rx_coal_ticks = BNX_RX_COAL_TICKS_DEF;
@@ -1899,10 +1905,14 @@ bnx_attach(device_t dev)
 #endif
        ifp->if_watchdog = bnx_watchdog;
        ifp->if_init = bnx_init;
+       ifp->if_serialize = bnx_serialize;
+       ifp->if_deserialize = bnx_deserialize;
+       ifp->if_tryserialize = bnx_tryserialize;
+#ifdef INVARIANTS
+       ifp->if_serialize_assert = bnx_serialize_assert;
+#endif
        ifp->if_mtu = ETHERMTU;
        ifp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
-       ifq_set_maxlen(&ifp->if_snd, BGE_TX_RING_CNT - 1);
-       ifq_set_ready(&ifp->if_snd);
 
        ifp->if_capabilities |= IFCAP_HWCSUM;
        ifp->if_hwassist = BNX_CSUM_FEATURES;
@@ -1912,6 +1922,10 @@ bnx_attach(device_t dev)
        }
        ifp->if_capenable = ifp->if_capabilities;
 
+       ifq_set_maxlen(&ifp->if_snd, BGE_TX_RING_CNT - 1);
+       ifq_set_ready(&ifp->if_snd);
+       ifq_set_subq_cnt(&ifp->if_snd, sc->bnx_tx_ringcnt);
+
        /*
         * Figure out what sort of media we have by checking the
         * hardware config word in the first 32k of NIC internal memory,
@@ -2078,9 +2092,10 @@ bnx_attach(device_t dev)
            sc, 0, bnx_sysctl_force_defrag, "I",
            "Force defragment on TX path");
 
-       SYSCTL_ADD_INT(&sc->bnx_sysctl_ctx,
+       SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
            SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
-           "tx_wreg", CTLFLAG_RW, &sc->bnx_tx_ring[0].bnx_tx_wreg, 0,
+           "tx_wreg", CTLTYPE_INT | CTLFLAG_RW,
+           sc, 0, bnx_sysctl_tx_wreg, "I",
            "# of segments before writing to hardware register");
 
        SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
@@ -2108,35 +2123,34 @@ bnx_attach(device_t dev)
         */
        ether_ifattach(ifp, ether_addr, NULL);
 
-       ifq_set_cpuid(&ifp->if_snd, sc->bnx_intr_cpuid);
+       /* Setup TX rings and subqueues */
+       for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
+               struct ifaltq_subque *ifsq = ifq_get_subq(&ifp->if_snd, i);
+               struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
+
+               ifsq_set_cpuid(ifsq, txr->bnx_tx_cpuid);
+               ifsq_set_hw_serialize(ifsq, &txr->bnx_tx_serialize);
+#ifdef notyet
+               ifsq_set_priv(ifsq, txr);
+               txr->ifsq = ifsq;
+
+               ifsq_watchdog_init(&txr->tx_watchdog, ifsq, bce_watchdog);
+#endif
+       }
 
 #ifdef IFPOLL_ENABLE
        ifpoll_compat_setup(&sc->bnx_npoll,
            &sc->bnx_sysctl_ctx, sc->bnx_sysctl_tree,
-           device_get_unit(dev), ifp->if_serializer);
+           device_get_unit(dev), &sc->bnx_main_serialize);
 #endif
 
-       if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI) {
-               if (sc->bnx_flags & BNX_FLAG_ONESHOT_MSI) {
-                       intr_func = bnx_msi_oneshot;
-                       if (bootverbose)
-                               device_printf(dev, "oneshot MSI\n");
-               } else {
-                       intr_func = bnx_msi;
-               }
-       } else {
-               intr_func = bnx_intr_legacy;
-       }
-       error = bus_setup_intr(dev, sc->bnx_irq, INTR_MPSAFE, intr_func, sc,
-           &sc->bnx_intrhand, ifp->if_serializer);
+       error = bnx_setup_intr(sc);
        if (error) {
                ether_ifdetach(ifp);
-               device_printf(dev, "couldn't set up irq\n");
                goto fail;
        }
 
-       sc->bnx_intr_cpuid = rman_get_cpuid(sc->bnx_irq);
-       sc->bnx_stat_cpuid = sc->bnx_intr_cpuid;
+       sc->bnx_stat_cpuid = sc->bnx_intr_data[0].bnx_intr_cpuid;
 
        return(0);
 fail:
@@ -2152,11 +2166,11 @@ bnx_detach(device_t dev)
        if (device_is_attached(dev)) {
                struct ifnet *ifp = &sc->arpcom.ac_if;
 
-               lwkt_serialize_enter(ifp->if_serializer);
+               ifnet_serialize_all(ifp);
                bnx_stop(sc);
                bnx_reset(sc);
-               bus_teardown_intr(dev, sc->bnx_irq, sc->bnx_intrhand);
-               lwkt_serialize_exit(ifp->if_serializer);
+               bnx_teardown_intr(sc, sc->bnx_intr_cnt);
+               ifnet_deserialize_all(ifp);
 
                ether_ifdetach(ifp);
        }
@@ -2167,12 +2181,7 @@ bnx_detach(device_t dev)
                device_delete_child(dev, sc->bnx_miibus);
        bus_generic_detach(dev);
 
-       if (sc->bnx_irq != NULL) {
-               bus_release_resource(dev, SYS_RES_IRQ, sc->bnx_irq_rid,
-                   sc->bnx_irq);
-       }
-       if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI)
-               pci_release_msi(dev);
+       bnx_free_intr(sc);
 
        if (sc->bnx_res != NULL) {
                bus_release_resource(dev, SYS_RES_MEMORY,
@@ -2184,6 +2193,9 @@ bnx_detach(device_t dev)
 
        bnx_dma_free(sc);
 
+       if (sc->bnx_serialize != NULL)
+               kfree(sc->bnx_serialize, M_DEVBUF);
+
        return 0;
 }
 
@@ -2376,14 +2388,14 @@ bnx_reset(struct bnx_softc *sc)
  */
 
 static void
-bnx_rxeof(struct bnx_softc *sc, uint16_t rx_prod, int count)
+bnx_rxeof(struct bnx_rx_ret_ring *ret, uint16_t rx_prod, int count)
 {
-       struct ifnet *ifp;
+       struct bnx_softc *sc = ret->bnx_sc;
+       struct bnx_rx_std_ring *std = ret->bnx_std;
+       struct ifnet *ifp = &sc->arpcom.ac_if;
        int stdcnt = 0, jumbocnt = 0;
 
-       ifp = &sc->arpcom.ac_if;
-
-       while (sc->bnx_rx_saved_considx != rx_prod && count != 0) {
+       while (ret->bnx_rx_saved_considx != rx_prod && count != 0) {
                struct bge_rx_bd        *cur_rx;
                uint32_t                rxidx;
                struct mbuf             *m = NULL;
@@ -2392,11 +2404,10 @@ bnx_rxeof(struct bnx_softc *sc, uint16_t rx_prod, int count)
 
                --count;
 
-               cur_rx =
-           &sc->bnx_ldata.bnx_rx_return_ring[sc->bnx_rx_saved_considx];
+               cur_rx = &ret->bnx_rx_ret_ring[ret->bnx_rx_saved_considx];
 
                rxidx = cur_rx->bge_idx;
-               BNX_INC(sc->bnx_rx_saved_considx, BNX_RETURN_RING_CNT);
+               BNX_INC(ret->bnx_rx_saved_considx, BNX_RETURN_RING_CNT);
 
                if (cur_rx->bge_flags & BGE_RXBDFLAG_VLAN_TAG) {
                        have_tag = 1;
@@ -2416,7 +2427,7 @@ bnx_rxeof(struct bnx_softc *sc, uint16_t rx_prod, int count)
                                continue;
                        }
 
-                       m = sc->bnx_cdata.bnx_rx_jumbo_chain[rxidx].bnx_mbuf;
+                       m = sc->bnx_cdata.bnx_rx_jumbo_chain[rxidx].bnx_rx_mbuf;
                        if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) {
                                IFNET_STAT_INC(ifp, ierrors, 1);
                                bnx_setup_rxdesc_jumbo(sc, sc->bnx_jumbo);
@@ -2428,27 +2439,27 @@ bnx_rxeof(struct bnx_softc *sc, uint16_t rx_prod, int count)
                                continue;
                        }
                } else {
-                       BNX_INC(sc->bnx_std, BGE_STD_RX_RING_CNT);
+                       BNX_INC(std->bnx_rx_std, BGE_STD_RX_RING_CNT);
                        stdcnt++;
 
-                       if (rxidx != sc->bnx_std) {
+                       if (rxidx != std->bnx_rx_std) {
                                IFNET_STAT_INC(ifp, ierrors, 1);
                                if_printf(ifp, "sw std index(%d) "
                                    "and hw std index(%d) mismatch, drop!\n",
-                                   sc->bnx_std, rxidx);
-                               bnx_setup_rxdesc_std(sc, rxidx);
+                                   std->bnx_rx_std, rxidx);
+                               bnx_setup_rxdesc_std(std, rxidx);
                                continue;
                        }
 
-                       m = sc->bnx_cdata.bnx_rx_std_chain[rxidx].bnx_mbuf;
+                       m = std->bnx_rx_std_buf[rxidx].bnx_rx_mbuf;
                        if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) {
                                IFNET_STAT_INC(ifp, ierrors, 1);
-                               bnx_setup_rxdesc_std(sc, sc->bnx_std);
+                               bnx_setup_rxdesc_std(std, std->bnx_rx_std);
                                continue;
                        }
-                       if (bnx_newbuf_std(sc, sc->bnx_std, 0)) {
+                       if (bnx_newbuf_std(ret, std->bnx_rx_std, 0)) {
                                IFNET_STAT_INC(ifp, ierrors, 1);
-                               bnx_setup_rxdesc_std(sc, sc->bnx_std);
+                               bnx_setup_rxdesc_std(std, std->bnx_rx_std);
                                continue;
                        }
                }
@@ -2484,9 +2495,9 @@ bnx_rxeof(struct bnx_softc *sc, uint16_t rx_prod, int count)
                ifp->if_input(ifp, m);
        }
 
-       bnx_writembx(sc, BGE_MBX_RX_CONS0_LO, sc->bnx_rx_saved_considx);
+       bnx_writembx(sc, BGE_MBX_RX_CONS0_LO, ret->bnx_rx_saved_considx);
        if (stdcnt)
-               bnx_writembx(sc, BGE_MBX_RX_STD_PROD_LO, sc->bnx_std);
+               bnx_writembx(sc, BGE_MBX_RX_STD_PROD_LO, std->bnx_rx_std);
        if (jumbocnt)
                bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, sc->bnx_jumbo);
 }
@@ -2501,25 +2512,27 @@ bnx_txeof(struct bnx_tx_ring *txr, uint16_t tx_cons)
         * frames that have been sent.
         */
        while (txr->bnx_tx_saved_considx != tx_cons) {
+               struct bnx_tx_buf *buf;
                uint32_t idx = 0;
 
                idx = txr->bnx_tx_saved_considx;
-               if (txr->bnx_tx_chain[idx] != NULL) {
+               buf = &txr->bnx_tx_buf[idx];
+               if (buf->bnx_tx_mbuf != NULL) {
                        IFNET_STAT_INC(ifp, opackets, 1);
                        bus_dmamap_unload(txr->bnx_tx_mtag,
-                           txr->bnx_tx_dmamap[idx]);
-                       m_freem(txr->bnx_tx_chain[idx]);
-                       txr->bnx_tx_chain[idx] = NULL;
+                           buf->bnx_tx_dmamap);
+                       m_freem(buf->bnx_tx_mbuf);
+                       buf->bnx_tx_mbuf = NULL;
                }
-               txr->bnx_txcnt--;
+               txr->bnx_tx_cnt--;
                BNX_INC(txr->bnx_tx_saved_considx, BGE_TX_RING_CNT);
        }
 
-       if ((BGE_TX_RING_CNT - txr->bnx_txcnt) >=
+       if ((BGE_TX_RING_CNT - txr->bnx_tx_cnt) >=
            (BNX_NSEG_RSVD + BNX_NSEG_SPARE))
                ifq_clr_oactive(&ifp->if_snd);
 
-       if (txr->bnx_txcnt == 0)
+       if (txr->bnx_tx_cnt == 0)
                ifp->if_timer = 0;
 
        if (!ifq_is_empty(&ifp->if_snd))
@@ -2533,14 +2546,14 @@ bnx_npoll(struct ifnet *ifp, struct ifpoll_info *info)
 {
        struct bnx_softc *sc = ifp->if_softc;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        if (info != NULL) {
                int cpuid = sc->bnx_npoll.ifpc_cpuid;
 
                info->ifpi_rx[cpuid].poll_func = bnx_npoll_compat;
                info->ifpi_rx[cpuid].arg = NULL;
-               info->ifpi_rx[cpuid].serializer = ifp->if_serializer;
+               info->ifpi_rx[cpuid].serializer = &sc->bnx_main_serialize;
 
                if (ifp->if_flags & IFF_RUNNING)
                        bnx_disable_intr(sc);
@@ -2548,7 +2561,7 @@ bnx_npoll(struct ifnet *ifp, struct ifpoll_info *info)
        } else {
                if (ifp->if_flags & IFF_RUNNING)
                        bnx_enable_intr(sc);
-               ifq_set_cpuid(&ifp->if_snd, sc->bnx_intr_cpuid);
+               ifq_set_cpuid(&ifp->if_snd, sc->bnx_tx_ring[0].bnx_tx_cpuid);
        }
 }
 
@@ -2557,10 +2570,11 @@ bnx_npoll_compat(struct ifnet *ifp, void *arg __unused, int cycle)
 {
        struct bnx_softc *sc = ifp->if_softc;
        struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0]; /* XXX */
+       struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0]; /* XXX */
        struct bge_status_block *sblk = sc->bnx_ldata.bnx_status_block;
        uint16_t rx_prod, tx_cons;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_SERIALIZED(&sc->bnx_main_serialize);
 
        if (sc->bnx_npoll.ifpc_stcount-- == 0) {
                sc->bnx_npoll.ifpc_stcount = sc->bnx_npoll.ifpc_stfrac;
@@ -2578,17 +2592,17 @@ bnx_npoll_compat(struct ifnet *ifp, void *arg __unused, int cycle)
         */
        cpu_lfence();
 
-       rx_prod = sblk->bge_idx[0].bge_rx_prod_idx;
-       tx_cons = sblk->bge_idx[0].bge_tx_cons_idx;
-
-       if (sc->bnx_rx_saved_considx != rx_prod)
-               bnx_rxeof(sc, rx_prod, cycle);
+       lwkt_serialize_enter(&ret->bnx_rx_ret_serialize);
+       rx_prod = *ret->bnx_rx_considx;
+       if (ret->bnx_rx_saved_considx != rx_prod)
+               bnx_rxeof(ret, rx_prod, cycle);
+       lwkt_serialize_exit(&ret->bnx_rx_ret_serialize);
 
+       lwkt_serialize_enter(&txr->bnx_tx_serialize);
+       tx_cons = *txr->bnx_tx_considx;
        if (txr->bnx_tx_saved_considx != tx_cons)
                bnx_txeof(txr, tx_cons);
-
-       if (sc->bnx_coal_chg)
-               bnx_coal_change(sc);
+       lwkt_serialize_exit(&txr->bnx_tx_serialize);
 }
 
 #endif /* IFPOLL_ENABLE */
@@ -2639,9 +2653,10 @@ bnx_intr(struct bnx_softc *sc)
 {
        struct ifnet *ifp = &sc->arpcom.ac_if;
        struct bge_status_block *sblk = sc->bnx_ldata.bnx_status_block;
-       uint16_t rx_prod, tx_cons;
        uint32_t status;
 
+       ASSERT_SERIALIZED(&sc->bnx_main_serialize);
+
        sc->bnx_status_tag = sblk->bge_status_tag;
        /*
         * Use a load fence to ensure that status_tag is saved 
@@ -2649,8 +2664,6 @@ bnx_intr(struct bnx_softc *sc)
         */
        cpu_lfence();
 
-       rx_prod = sblk->bge_idx[0].bge_rx_prod_idx;
-       tx_cons = sblk->bge_idx[0].bge_tx_cons_idx;
        status = sblk->bge_status;
 
        if ((status & BGE_STATFLAG_LINKSTATE_CHANGED) || sc->bnx_link_evt)
@@ -2658,27 +2671,31 @@ bnx_intr(struct bnx_softc *sc)
 
        if (ifp->if_flags & IFF_RUNNING) {
                struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0]; /* XXX */
+               struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0]; /* XXX */
+               uint16_t rx_prod, tx_cons;
 
-               if (sc->bnx_rx_saved_considx != rx_prod)
-                       bnx_rxeof(sc, rx_prod, -1);
+               lwkt_serialize_enter(&ret->bnx_rx_ret_serialize);
+               rx_prod = *ret->bnx_rx_considx;
+               if (ret->bnx_rx_saved_considx != rx_prod)
+                       bnx_rxeof(ret, rx_prod, -1);
+               lwkt_serialize_exit(&ret->bnx_rx_ret_serialize);
 
+               lwkt_serialize_enter(&txr->bnx_tx_serialize);
+               tx_cons = *txr->bnx_tx_considx;
                if (txr->bnx_tx_saved_considx != tx_cons)
                        bnx_txeof(txr, tx_cons);
+               lwkt_serialize_exit(&txr->bnx_tx_serialize);
        }
 
        bnx_writembx(sc, BGE_MBX_IRQ0_LO, sc->bnx_status_tag << 24);
-
-       if (sc->bnx_coal_chg)
-               bnx_coal_change(sc);
 }
 
 static void
 bnx_tick(void *xsc)
 {
        struct bnx_softc *sc = xsc;
-       struct ifnet *ifp = &sc->arpcom.ac_if;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       lwkt_serialize_enter(&sc->bnx_main_serialize);
 
        KKASSERT(mycpuid == sc->bnx_stat_cpuid);
 
@@ -2698,7 +2715,7 @@ bnx_tick(void *xsc)
 
        callout_reset(&sc->bnx_stat_timer, hz, bnx_tick, sc);
 
-       lwkt_serialize_exit(ifp->if_serializer);
+       lwkt_serialize_exit(&sc->bnx_main_serialize);
 }
 
 static void
@@ -2754,7 +2771,7 @@ bnx_encap(struct bnx_tx_ring *txr, struct mbuf **m_head0, uint32_t *txidx,
                        tso_nsegs = BNX_TSO_NSTATS - 1;
                else if (tso_nsegs < 0)
                        tso_nsegs = 0;
-               txr->sc->bnx_tsosegs[tso_nsegs]++;
+               txr->bnx_sc->bnx_tsosegs[tso_nsegs]++;
 #endif
        } else if (m_head->m_pkthdr.csum_flags & BNX_CSUM_FEATURES) {
                if (m_head->m_pkthdr.csum_flags & CSUM_IP)
@@ -2772,9 +2789,9 @@ bnx_encap(struct bnx_tx_ring *txr, struct mbuf **m_head0, uint32_t *txidx,
        }
 
        idx = *txidx;
-       map = txr->bnx_tx_dmamap[idx];
+       map = txr->bnx_tx_buf[idx].bnx_tx_dmamap;
 
-       maxsegs = (BGE_TX_RING_CNT - txr->bnx_txcnt) - BNX_NSEG_RSVD;
+       maxsegs = (BGE_TX_RING_CNT - txr->bnx_tx_cnt) - BNX_NSEG_RSVD;
        KASSERT(maxsegs >= BNX_NSEG_SPARE,
                ("not enough segments %d", maxsegs));
 
@@ -2850,10 +2867,10 @@ bnx_encap(struct bnx_tx_ring *txr, struct mbuf **m_head0, uint32_t *txidx,
         * Insure that the map for this transmission is placed at
         * the array index of the last descriptor in this chain.
         */
-       txr->bnx_tx_dmamap[*txidx] = txr->bnx_tx_dmamap[idx];
-       txr->bnx_tx_dmamap[idx] = map;
-       txr->bnx_tx_chain[idx] = m_head;
-       txr->bnx_txcnt += nsegs;
+       txr->bnx_tx_buf[*txidx].bnx_tx_dmamap = txr->bnx_tx_buf[idx].bnx_tx_dmamap;
+       txr->bnx_tx_buf[idx].bnx_tx_dmamap = map;
+       txr->bnx_tx_buf[idx].bnx_tx_mbuf = m_head;
+       txr->bnx_tx_cnt += nsegs;
 
        BNX_INC(idx, BGE_TX_RING_CNT);
        *txidx = idx;
@@ -2879,20 +2896,21 @@ bnx_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
        int nsegs = 0;
 
        ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq);
+       ASSERT_SERIALIZED(&txr->bnx_tx_serialize);
 
        if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd))
                return;
 
        prodidx = txr->bnx_tx_prodidx;
 
-       while (txr->bnx_tx_chain[prodidx] == NULL) {
+       while (txr->bnx_tx_buf[prodidx].bnx_tx_mbuf == NULL) {
                /*
                 * Sanity check: avoid coming within BGE_NSEG_RSVD
                 * descriptors of the end of the ring.  Also make
                 * sure there are BGE_NSEG_SPARE descriptors for
                 * jumbo buffers' or TSO segments' defragmentation.
                 */
-               if ((BGE_TX_RING_CNT - txr->bnx_txcnt) <
+               if ((BGE_TX_RING_CNT - txr->bnx_tx_cnt) <
                    (BNX_NSEG_RSVD + BNX_NSEG_SPARE)) {
                        ifq_set_oactive(&ifp->if_snd);
                        break;
@@ -2915,8 +2933,7 @@ bnx_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
 
                if (nsegs >= txr->bnx_tx_wreg) {
                        /* Transmit */
-                       bnx_writembx(txr->bnx_sc, BGE_MBX_TX_HOST_PROD0_LO,
-                           prodidx);
+                       bnx_writembx(txr->bnx_sc, txr->bnx_tx_mbx, prodidx);
                        nsegs = 0;
                }
 
@@ -2930,7 +2947,7 @@ bnx_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
 
        if (nsegs > 0) {
                /* Transmit */
-               bnx_writembx(txr->bnx_sc, BGE_MBX_TX_HOST_PROD0_LO, prodidx);
+               bnx_writembx(txr->bnx_sc, txr->bnx_tx_mbx, prodidx);
        }
        txr->bnx_tx_prodidx = prodidx;
 }
@@ -2944,7 +2961,7 @@ bnx_init(void *xsc)
        uint32_t mode;
        int i;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        /* Cancel pending I/O and flush buffers. */
        bnx_stop(sc);
@@ -2977,7 +2994,7 @@ bnx_init(void *xsc)
        bnx_setmulti(sc);
 
        /* Init RX ring. */
-       if (bnx_init_rx_ring_std(sc)) {
+       if (bnx_init_rx_ring_std(&sc->bnx_rx_std_ring)) {
                if_printf(ifp, "RX ring initialization failed\n");
                bnx_stop(sc);
                return;
@@ -2993,7 +3010,8 @@ bnx_init(void *xsc)
        }
 
        /* Init our RX return ring index */
-       sc->bnx_rx_saved_considx = 0;
+       for (i = 0; i < sc->bnx_rx_retcnt; ++i)
+               sc->bnx_rx_ret_ring[i].bnx_rx_saved_considx = 0;
 
        /* Init TX ring. */
        for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
@@ -3025,7 +3043,7 @@ bnx_init(void *xsc)
        else
                CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 2);
 
-       if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI) {
+       if (sc->bnx_intr_type == PCI_INTR_TYPE_MSI) {
                if (bootverbose) {
                        if_printf(ifp, "MSI_MODE: %#x\n",
                            CSR_READ_4(sc, BGE_MSI_MODE));
@@ -3155,7 +3173,7 @@ bnx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
        struct ifreq *ifr = (struct ifreq *)data;
        int mask, error = 0;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        switch (command) {
        case SIOCSIFMTU:
@@ -3249,7 +3267,7 @@ bnx_watchdog(struct ifnet *ifp)
        IFNET_STAT_INC(ifp, oerrors, 1);
 
        if (!ifq_is_empty(&ifp->if_snd))
-               if_devstart(ifp);
+               if_devstart_sched(ifp);
 }
 
 /*
@@ -3262,7 +3280,7 @@ bnx_stop(struct bnx_softc *sc)
        struct ifnet *ifp = &sc->arpcom.ac_if;
        int i;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        callout_stop(&sc->bnx_stat_timer);
 
@@ -3304,7 +3322,7 @@ bnx_stop(struct bnx_softc *sc)
        BNX_CLRBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
 
        /* Free the RX lists. */
-       bnx_free_rx_ring_std(sc);
+       bnx_free_rx_ring_std(&sc->bnx_rx_std_ring);
 
        /* Free jumbo RX list. */
        if (BNX_IS_JUMBO_CAPABLE(sc))
@@ -3333,10 +3351,10 @@ bnx_shutdown(device_t dev)
        struct bnx_softc *sc = device_get_softc(dev);
        struct ifnet *ifp = &sc->arpcom.ac_if;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
        bnx_stop(sc);
        bnx_reset(sc);
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 }
 
 static int
@@ -3345,9 +3363,9 @@ bnx_suspend(device_t dev)
        struct bnx_softc *sc = device_get_softc(dev);
        struct ifnet *ifp = &sc->arpcom.ac_if;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
        bnx_stop(sc);
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 
        return 0;
 }
@@ -3358,16 +3376,16 @@ bnx_resume(device_t dev)
        struct bnx_softc *sc = device_get_softc(dev);
        struct ifnet *ifp = &sc->arpcom.ac_if;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
 
        if (ifp->if_flags & IFF_UP) {
                bnx_init(sc);
 
                if (!ifq_is_empty(&ifp->if_snd))
-                       if_devstart(ifp);
+                       if_devstart_sched(ifp);
        }
 
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 
        return 0;
 }
@@ -3386,19 +3404,30 @@ bnx_setpromisc(struct bnx_softc *sc)
 static void
 bnx_dma_free(struct bnx_softc *sc)
 {
+       struct bnx_rx_std_ring *std = &sc->bnx_rx_std_ring;
        int i;
 
+       /* Destroy RX return rings */
+       if (sc->bnx_rx_ret_ring != NULL) {
+               for (i = 0; i < sc->bnx_rx_retcnt; ++i)
+                       bnx_destroy_rx_ret_ring(&sc->bnx_rx_ret_ring[i]);
+               kfree(sc->bnx_rx_ret_ring, M_DEVBUF);
+       }
+
        /* Destroy RX mbuf DMA stuffs. */
-       if (sc->bnx_cdata.bnx_rx_mtag != NULL) {
+       if (std->bnx_rx_mtag != NULL) {
                for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
-                       bus_dmamap_destroy(sc->bnx_cdata.bnx_rx_mtag,
-                           sc->bnx_cdata.bnx_rx_std_dmamap[i]);
+                       KKASSERT(std->bnx_rx_std_buf[i].bnx_rx_mbuf == NULL);
+                       bus_dmamap_destroy(std->bnx_rx_mtag,
+                           std->bnx_rx_std_buf[i].bnx_rx_dmamap);
                }
-               bus_dmamap_destroy(sc->bnx_cdata.bnx_rx_mtag,
-                                  sc->bnx_cdata.bnx_rx_tmpmap);
-               bus_dma_tag_destroy(sc->bnx_cdata.bnx_rx_mtag);
+               bus_dma_tag_destroy(std->bnx_rx_mtag);
        }
 
+       /* Destroy standard RX ring */
+       bnx_dma_block_free(std->bnx_rx_std_ring_tag,
+           std->bnx_rx_std_ring_map, std->bnx_rx_std_ring);
+
        /* Destroy TX rings */
        if (sc->bnx_tx_ring != NULL) {
                for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
@@ -3406,23 +3435,13 @@ bnx_dma_free(struct bnx_softc *sc)
                kfree(sc->bnx_tx_ring, M_DEVBUF);
        }
 
-       /* Destroy standard RX ring */
-       bnx_dma_block_free(sc->bnx_cdata.bnx_rx_std_ring_tag,
-                          sc->bnx_cdata.bnx_rx_std_ring_map,
-                          sc->bnx_ldata.bnx_rx_std_ring);
-
        if (BNX_IS_JUMBO_CAPABLE(sc))
                bnx_free_jumbo_mem(sc);
 
-       /* Destroy RX return ring */
-       bnx_dma_block_free(sc->bnx_cdata.bnx_rx_return_ring_tag,
-                          sc->bnx_cdata.bnx_rx_return_ring_map,
-                          sc->bnx_ldata.bnx_rx_return_ring);
-
        /* Destroy status block */
        bnx_dma_block_free(sc->bnx_cdata.bnx_status_tag,
-                          sc->bnx_cdata.bnx_status_map,
-                          sc->bnx_ldata.bnx_status_block);
+           sc->bnx_cdata.bnx_status_map,
+           sc->bnx_ldata.bnx_status_block);
 
        /* Destroy the parent tag */
        if (sc->bnx_cdata.bnx_parent_tag != NULL)
@@ -3430,10 +3449,11 @@ bnx_dma_free(struct bnx_softc *sc)
 }
 
 static int
-bnx_dma_alloc(struct bnx_softc *sc)
+bnx_dma_alloc(device_t dev)
 {
-       struct ifnet *ifp = &sc->arpcom.ac_if;
-       int i, error;
+       struct bnx_softc *sc = device_get_softc(dev);
+       struct bnx_rx_std_ring *std = &sc->bnx_rx_std_ring;
+       int i, error, mbx;
 
        /*
         * Allocate the parent bus DMA tag appropriate for PCI.
@@ -3446,52 +3466,56 @@ bnx_dma_alloc(struct bnx_softc *sc)
         * state machine will lockup and cause the device to hang.
         */
        error = bus_dma_tag_create(NULL, 1, BGE_DMA_BOUNDARY_4G,
-                                  BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
-                                  NULL, NULL,
-                                  BUS_SPACE_MAXSIZE_32BIT, 0,
-                                  BUS_SPACE_MAXSIZE_32BIT,
-                                  0, &sc->bnx_cdata.bnx_parent_tag);
+           BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
+           BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT,
+           0, &sc->bnx_cdata.bnx_parent_tag);
        if (error) {
-               if_printf(ifp, "could not allocate parent dma tag\n");
+               device_printf(dev, "could not create parent DMA tag\n");
                return error;
        }
 
        /*
-        * Create DMA tag and maps for RX mbufs.
+        * Create DMA stuffs for status block.
         */
-       error = bus_dma_tag_create(sc->bnx_cdata.bnx_parent_tag, 1, 0,
-                                  BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
-                                  NULL, NULL, MCLBYTES, 1, MCLBYTES,
-                                  BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK,
-                                  &sc->bnx_cdata.bnx_rx_mtag);
+       error = bnx_dma_block_alloc(sc, BGE_STATUS_BLK_SZ,
+           &sc->bnx_cdata.bnx_status_tag,
+           &sc->bnx_cdata.bnx_status_map,
+           (void *)&sc->bnx_ldata.bnx_status_block,
+           &sc->bnx_ldata.bnx_status_block_paddr);
        if (error) {
-               if_printf(ifp, "could not allocate RX mbuf dma tag\n");
+               device_printf(dev, "could not create status block\n");
                return error;
        }
 
-       error = bus_dmamap_create(sc->bnx_cdata.bnx_rx_mtag,
-                                 BUS_DMA_WAITOK, &sc->bnx_cdata.bnx_rx_tmpmap);
+       /*
+        * Create DMA tag and maps for RX mbufs.
+        */
+       std->bnx_sc = sc;
+       lwkt_serialize_init(&std->bnx_rx_std_serialize);
+       error = bus_dma_tag_create(sc->bnx_cdata.bnx_parent_tag, 1, 0,
+           BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
+           NULL, NULL, MCLBYTES, 1, MCLBYTES,
+           BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK, &std->bnx_rx_mtag);
        if (error) {
-               bus_dma_tag_destroy(sc->bnx_cdata.bnx_rx_mtag);
-               sc->bnx_cdata.bnx_rx_mtag = NULL;
+               device_printf(dev, "could not create RX mbuf DMA tag\n");
                return error;
        }
 
-       for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
-               error = bus_dmamap_create(sc->bnx_cdata.bnx_rx_mtag,
-                                         BUS_DMA_WAITOK,
-                                         &sc->bnx_cdata.bnx_rx_std_dmamap[i]);
+       for (i = 0; i < BGE_STD_RX_RING_CNT; ++i) {
+               error = bus_dmamap_create(std->bnx_rx_mtag, BUS_DMA_WAITOK,
+                   &std->bnx_rx_std_buf[i].bnx_rx_dmamap);
                if (error) {
                        int j;
 
                        for (j = 0; j < i; ++j) {
-                               bus_dmamap_destroy(sc->bnx_cdata.bnx_rx_mtag,
-                                       sc->bnx_cdata.bnx_rx_std_dmamap[j]);
+                               bus_dmamap_destroy(std->bnx_rx_mtag,
+                                   std->bnx_rx_std_buf[j].bnx_rx_dmamap);
                        }
-                       bus_dma_tag_destroy(sc->bnx_cdata.bnx_rx_mtag);
-                       sc->bnx_cdata.bnx_rx_mtag = NULL;
+                       bus_dma_tag_destroy(std->bnx_rx_mtag);
+                       std->bnx_rx_mtag = NULL;
 
-                       if_printf(ifp, "could not create DMA map for RX\n");
+                       device_printf(dev,
+                           "could not create %dth RX mbuf DMA map\n", i);
                        return error;
                }
        }
@@ -3500,53 +3524,43 @@ bnx_dma_alloc(struct bnx_softc *sc)
         * Create DMA stuffs for standard RX ring.
         */
        error = bnx_dma_block_alloc(sc, BGE_STD_RX_RING_SZ,
-                                   &sc->bnx_cdata.bnx_rx_std_ring_tag,
-                                   &sc->bnx_cdata.bnx_rx_std_ring_map,
-                                   (void *)&sc->bnx_ldata.bnx_rx_std_ring,
-                                   &sc->bnx_ldata.bnx_rx_std_ring_paddr);
+           &std->bnx_rx_std_ring_tag,
+           &std->bnx_rx_std_ring_map,
+           (void *)&std->bnx_rx_std_ring,
+           &std->bnx_rx_std_ring_paddr);
        if (error) {
-               if_printf(ifp, "could not create std RX ring\n");
+               device_printf(dev, "could not create std RX ring\n");
                return error;
        }
 
        /*
-        * Create jumbo buffer pool.
+        * Create RX return rings
         */
-       if (BNX_IS_JUMBO_CAPABLE(sc)) {
-               error = bnx_alloc_jumbo_mem(sc);
+       sc->bnx_rx_ret_ring = kmalloc_cachealign(
+           sizeof(struct bnx_rx_ret_ring) * sc->bnx_rx_retcnt, M_DEVBUF,
+           M_WAITOK | M_ZERO);
+       for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
+               struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i];
+
+               ret->bnx_sc = sc;
+               ret->bnx_std = std;
+
+               /* XXX */
+               ret->bnx_rx_considx =
+               &sc->bnx_ldata.bnx_status_block->bge_idx[0].bge_rx_prod_idx;
+
+               error = bnx_create_rx_ret_ring(ret);
                if (error) {
-                       if_printf(ifp, "could not create jumbo buffer pool\n");
+                       device_printf(dev,
+                           "could not create %dth RX ret ring\n", i);
                        return error;
                }
        }
 
        /*
-        * Create DMA stuffs for RX return ring.
+        * Create TX rings
         */
-       error = bnx_dma_block_alloc(sc,
-           BGE_RX_RTN_RING_SZ(BNX_RETURN_RING_CNT),
-           &sc->bnx_cdata.bnx_rx_return_ring_tag,
-           &sc->bnx_cdata.bnx_rx_return_ring_map,
-           (void *)&sc->bnx_ldata.bnx_rx_return_ring,
-           &sc->bnx_ldata.bnx_rx_return_ring_paddr);
-       if (error) {
-               if_printf(ifp, "could not create RX ret ring\n");
-               return error;
-       }
-
-       /*
-        * Create DMA stuffs for status block.
-        */
-       error = bnx_dma_block_alloc(sc, BGE_STATUS_BLK_SZ,
-                                   &sc->bnx_cdata.bnx_status_tag,
-                                   &sc->bnx_cdata.bnx_status_map,
-                                   (void *)&sc->bnx_ldata.bnx_status_block,
-                                   &sc->bnx_ldata.bnx_status_block_paddr);
-       if (error) {
-               if_printf(ifp, "could not create status block\n");
-               return error;
-       }
-
+       mbx = BGE_MBX_TX_HOST_PROD0_LO;
        sc->bnx_tx_ring = kmalloc_cachealign(
            sizeof(struct bnx_tx_ring) * sc->bnx_tx_ringcnt, M_DEVBUF,
            M_WAITOK | M_ZERO);
@@ -3554,10 +3568,33 @@ bnx_dma_alloc(struct bnx_softc *sc)
                struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
 
                txr->bnx_sc = sc;
+               txr->bnx_tx_mbx = mbx;
+
+               if (mbx & 0x4)
+                       mbx -= 0x4;
+               else
+                       mbx += 0xc;
+
+               /* XXX */
+               txr->bnx_tx_considx =
+               &sc->bnx_ldata.bnx_status_block->bge_idx[0].bge_tx_cons_idx;
+
                error = bnx_create_tx_ring(txr);
                if (error) {
-                       device_printf(sc->bnx_dev,
-                           "can't create %dth tx ring\n", i);
+                       device_printf(dev,
+                           "could not create %dth TX ring\n", i);
+                       return error;
+               }
+       }
+
+       /*
+        * Create jumbo buffer pool.
+        */
+       if (BNX_IS_JUMBO_CAPABLE(sc)) {
+               error = bnx_alloc_jumbo_mem(sc);
+               if (error) {
+                       device_printf(dev,
+                           "could not create jumbo buffer pool\n");
                        return error;
                }
        }
@@ -3773,7 +3810,7 @@ bnx_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *coal,
        struct ifnet *ifp = &sc->arpcom.ac_if;
        int error = 0, v;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
 
        v = *coal;
        error = sysctl_handle_int(oidp, &v, 0, req);
@@ -3783,10 +3820,13 @@ bnx_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *coal,
                } else {
                        *coal = v;
                        sc->bnx_coal_chg |= coal_chg_mask;
+
+                       /* Commit changes */
+                       bnx_coal_change(sc);
                }
        }
 
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
        return error;
 }
 
@@ -3795,7 +3835,7 @@ bnx_coal_change(struct bnx_softc *sc)
 {
        struct ifnet *ifp = &sc->arpcom.ac_if;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        if (sc->bnx_coal_chg & BNX_RX_COAL_TICKS_CHG) {
                CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS,
@@ -3873,51 +3913,59 @@ bnx_coal_change(struct bnx_softc *sc)
 }
 
 static void
-bnx_intr_check(void *xsc)
+bnx_check_intr(void *xintr)
 {
-       struct bnx_softc *sc = xsc;
-       struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0]; /* XXX */
-       struct ifnet *ifp = &sc->arpcom.ac_if;
-       struct bge_status_block *sblk = sc->bnx_ldata.bnx_status_block;
+       struct bnx_intr_data *intr = xintr;
+       struct bnx_rx_ret_ring *ret;
+       struct bnx_tx_ring *txr;
+       struct ifnet *ifp;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       lwkt_serialize_enter(intr->bnx_intr_serialize);
 
-       KKASSERT(mycpuid == sc->bnx_intr_cpuid);
+       KKASSERT(mycpuid == intr->bnx_intr_cpuid);
 
+       ifp = &intr->bnx_sc->arpcom.ac_if;
        if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) != IFF_RUNNING) {
-               lwkt_serialize_exit(ifp->if_serializer);
+               lwkt_serialize_exit(intr->bnx_intr_serialize);
                return;
        }
 
-       if (sblk->bge_idx[0].bge_rx_prod_idx != sc->bnx_rx_saved_considx ||
-           sblk->bge_idx[0].bge_tx_cons_idx != txr->bnx_tx_saved_considx) {
-               if (sc->bnx_rx_check_considx == sc->bnx_rx_saved_considx &&
-                   sc->bnx_tx_check_considx == txr->bnx_tx_saved_considx) {
-                       if (!sc->bnx_intr_maylose) {
-                               sc->bnx_intr_maylose = TRUE;
+       txr = intr->bnx_txr;
+       ret = intr->bnx_ret;
+
+       if (*ret->bnx_rx_considx != ret->bnx_rx_saved_considx ||
+           *txr->bnx_tx_considx != txr->bnx_tx_saved_considx) {
+               if (intr->bnx_rx_check_considx == ret->bnx_rx_saved_considx &&
+                   intr->bnx_tx_check_considx == txr->bnx_tx_saved_considx) {
+                       if (!intr->bnx_intr_maylose) {
+                               intr->bnx_intr_maylose = TRUE;
                                goto done;
                        }
                        if (bootverbose)
                                if_printf(ifp, "lost interrupt\n");
-                       bnx_msi(sc);
+                       intr->bnx_intr_func(intr->bnx_intr_arg);
                }
        }
-       sc->bnx_intr_maylose = FALSE;
-       sc->bnx_rx_check_considx = sc->bnx_rx_saved_considx;
-       sc->bnx_tx_check_considx = txr->bnx_tx_saved_considx;
+       intr->bnx_intr_maylose = FALSE;
+       intr->bnx_rx_check_considx = ret->bnx_rx_saved_considx;
+       intr->bnx_tx_check_considx = txr->bnx_tx_saved_considx;
 
 done:
-       callout_reset(&sc->bnx_intr_timer, BNX_INTR_CKINTVL,
-           bnx_intr_check, sc);
-       lwkt_serialize_exit(ifp->if_serializer);
+       callout_reset(&intr->bnx_intr_timer, BNX_INTR_CKINTVL,
+           intr->bnx_intr_check, intr);
+       lwkt_serialize_exit(intr->bnx_intr_serialize);
 }
 
 static void
 bnx_enable_intr(struct bnx_softc *sc)
 {
        struct ifnet *ifp = &sc->arpcom.ac_if;
+       int i;
 
-       lwkt_serialize_handler_enable(ifp->if_serializer);
+       for (i = 0; i < sc->bnx_intr_cnt; ++i) {
+               lwkt_serialize_handler_enable(
+                   sc->bnx_intr_data[i].bnx_intr_serialize);
+       }
 
        /*
         * Enable interrupt.
@@ -3942,23 +3990,35 @@ bnx_enable_intr(struct bnx_softc *sc)
        BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_SET);
 
        if (sc->bnx_flags & BNX_FLAG_STATUSTAG_BUG) {
-               sc->bnx_intr_maylose = FALSE;
-               sc->bnx_rx_check_considx = 0;
-               sc->bnx_tx_check_considx = 0;
-
                if (bootverbose)
                        if_printf(ifp, "status tag bug workaround\n");
 
-               /* 10ms check interval */
-               callout_reset_bycpu(&sc->bnx_intr_timer, BNX_INTR_CKINTVL,
-                   bnx_intr_check, sc, sc->bnx_intr_cpuid);
+               for (i = 0; i < sc->bnx_intr_cnt; ++i) {
+                       struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
+
+                       intr->bnx_intr_maylose = FALSE;
+                       intr->bnx_rx_check_considx = 0;
+                       intr->bnx_tx_check_considx = 0;
+                       callout_reset_bycpu(&intr->bnx_intr_timer,
+                           BNX_INTR_CKINTVL, intr->bnx_intr_check, intr,
+                           intr->bnx_intr_cpuid);
+               }
        }
 }
 
 static void
 bnx_disable_intr(struct bnx_softc *sc)
 {
-       struct ifnet *ifp = &sc->arpcom.ac_if;
+       int i;
+
+       for (i = 0; i < sc->bnx_intr_cnt; ++i) {
+               struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
+
+               callout_stop(&intr->bnx_intr_timer);
+               intr->bnx_intr_maylose = FALSE;
+               intr->bnx_rx_check_considx = 0;
+               intr->bnx_tx_check_considx = 0;
+       }
 
        /*
         * Mask the interrupt when we start polling.
@@ -3971,14 +4031,11 @@ bnx_disable_intr(struct bnx_softc *sc)
         */
        bnx_writembx(sc, BGE_MBX_IRQ0_LO, 1);
 
-       callout_stop(&sc->bnx_intr_timer);
-       sc->bnx_intr_maylose = FALSE;
-       sc->bnx_rx_check_considx = 0;
-       sc->bnx_tx_check_considx = 0;
-
        sc->bnx_npoll.ifpc_stcount = 0;
-
-       lwkt_serialize_handler_disable(ifp->if_serializer);
+       for (i = 0; i < sc->bnx_intr_cnt; ++i) {
+               lwkt_serialize_handler_disable(
+                   sc->bnx_intr_data[i].bnx_intr_serialize);
+       }
 }
 
 static int
@@ -4199,6 +4256,8 @@ bnx_create_tx_ring(struct bnx_tx_ring *txr)
        bus_size_t txmaxsz, txmaxsegsz;
        int i, error;
 
+       lwkt_serialize_init(&txr->bnx_tx_serialize);
+
        /*
         * Create DMA tag and maps for TX mbufs.
         */
@@ -4217,26 +4276,26 @@ bnx_create_tx_ring(struct bnx_tx_ring *txr)
            &txr->bnx_tx_mtag);
        if (error) {
                device_printf(txr->bnx_sc->bnx_dev,
-                   "could not allocate TX mbuf dma tag\n");
+                   "could not create TX mbuf DMA tag\n");
                return error;
        }
 
        for (i = 0; i < BGE_TX_RING_CNT; i++) {
                error = bus_dmamap_create(txr->bnx_tx_mtag,
                    BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
-                   &txr->bnx_tx_dmamap[i]);
+                   &txr->bnx_tx_buf[i].bnx_tx_dmamap);
                if (error) {
                        int j;
 
                        for (j = 0; j < i; ++j) {
                                bus_dmamap_destroy(txr->bnx_tx_mtag,
-                                       txr->bnx_tx_dmamap[j]);
+                                   txr->bnx_tx_buf[j].bnx_tx_dmamap);
                        }
                        bus_dma_tag_destroy(txr->bnx_tx_mtag);
                        txr->bnx_tx_mtag = NULL;
 
                        device_printf(txr->bnx_sc->bnx_dev,
-                           "could not create DMA map for TX\n");
+                           "could not create TX mbuf DMA map\n");
                        return error;
                }
        }
@@ -4245,8 +4304,10 @@ bnx_create_tx_ring(struct bnx_tx_ring *txr)
         * Create DMA stuffs for TX ring.
         */
        error = bnx_dma_block_alloc(txr->bnx_sc, BGE_TX_RING_SZ,
-           &txr->bnx_tx_ring_tag, &txr->bnx_tx_ring_map,
-           (void *)&txr->bnx_tx_ring, &txr->bnx_tx_ring_paddr);
+           &txr->bnx_tx_ring_tag,
+           &txr->bnx_tx_ring_map,
+           (void *)&txr->bnx_tx_ring,
+           &txr->bnx_tx_ring_paddr);
        if (error) {
                device_printf(txr->bnx_sc->bnx_dev,
                    "could not create TX ring\n");
@@ -4267,8 +4328,9 @@ bnx_destroy_tx_ring(struct bnx_tx_ring *txr)
                int i;
 
                for (i = 0; i < BGE_TX_RING_CNT; i++) {
+                       KKASSERT(txr->bnx_tx_buf[i].bnx_tx_mbuf == NULL);
                        bus_dmamap_destroy(txr->bnx_tx_mtag,
-                           txr->bnx_tx_dmamap[i]);
+                           txr->bnx_tx_buf[i].bnx_tx_dmamap);
                }
                bus_dma_tag_destroy(txr->bnx_tx_mtag);
        }
@@ -4295,7 +4357,7 @@ bnx_sysctl_force_defrag(SYSCTL_HANDLER_ARGS)
        if (error || req->newptr == NULL)
                return error;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
        for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
                txr = &sc->bnx_tx_ring[i];
                if (defrag)
@@ -4303,7 +4365,264 @@ bnx_sysctl_force_defrag(SYSCTL_HANDLER_ARGS)
                else
                        txr->bnx_tx_flags &= ~BNX_TX_FLAG_FORCE_DEFRAG;
        }
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
+
+       return 0;
+}
+
+static int
+bnx_sysctl_tx_wreg(SYSCTL_HANDLER_ARGS)
+{
+       struct bnx_softc *sc = (void *)arg1;
+       struct ifnet *ifp = &sc->arpcom.ac_if;
+       struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0];
+       int error, tx_wreg, i;
+
+       tx_wreg = txr->bnx_tx_wreg;
+       error = sysctl_handle_int(oidp, &tx_wreg, 0, req);
+       if (error || req->newptr == NULL)
+               return error;
+
+       ifnet_serialize_all(ifp);
+       for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
+               sc->bnx_tx_ring[i].bnx_tx_wreg = tx_wreg;
+       ifnet_deserialize_all(ifp);
+
+       return 0;
+}
+
+static int
+bnx_create_rx_ret_ring(struct bnx_rx_ret_ring *ret)
+{
+       int error;
+
+       lwkt_serialize_init(&ret->bnx_rx_ret_serialize);
+
+       /*
+        * Create DMA stuffs for RX return ring.
+        */
+       error = bnx_dma_block_alloc(ret->bnx_sc,
+           BGE_RX_RTN_RING_SZ(BNX_RETURN_RING_CNT),
+           &ret->bnx_rx_ret_ring_tag,
+           &ret->bnx_rx_ret_ring_map,
+           (void *)&ret->bnx_rx_ret_ring,
+           &ret->bnx_rx_ret_ring_paddr);
+       if (error) {
+               device_printf(ret->bnx_sc->bnx_dev,
+                   "could not create RX ret ring\n");
+               return error;
+       }
+
+       /* Shadow standard ring's RX mbuf DMA tag */
+       ret->bnx_rx_mtag = ret->bnx_std->bnx_rx_mtag;
+
+       /*
+        * Create tmp DMA map for RX mbufs.
+        */
+       error = bus_dmamap_create(ret->bnx_rx_mtag, BUS_DMA_WAITOK,
+           &ret->bnx_rx_tmpmap);
+       if (error) {
+               device_printf(ret->bnx_sc->bnx_dev,
+                   "could not create tmp RX mbuf DMA map\n");
+               ret->bnx_rx_mtag = NULL;
+               return error;
+       }
+       return 0;
+}
+
+static void
+bnx_destroy_rx_ret_ring(struct bnx_rx_ret_ring *ret)
+{
+       /* Destroy tmp RX mbuf DMA map */
+       if (ret->bnx_rx_mtag != NULL)
+               bus_dmamap_destroy(ret->bnx_rx_mtag, ret->bnx_rx_tmpmap);
+
+       /* Destroy RX return ring */
+       bnx_dma_block_free(ret->bnx_rx_ret_ring_tag,
+           ret->bnx_rx_ret_ring_map, ret->bnx_rx_ret_ring);
+}
+
+static int
+bnx_alloc_intr(struct bnx_softc *sc)
+{
+       struct bnx_intr_data *intr;
+       u_int intr_flags;
+
+       sc->bnx_intr_cnt = 1;
+
+       intr = &sc->bnx_intr_data[0];
+       intr->bnx_sc = sc;
+       intr->bnx_ret = &sc->bnx_rx_ret_ring[0];
+       intr->bnx_txr = &sc->bnx_tx_ring[0];
+       intr->bnx_intr_serialize = &sc->bnx_main_serialize;
+       callout_init_mp(&intr->bnx_intr_timer);
+       intr->bnx_intr_check = bnx_check_intr;
+
+       sc->bnx_intr_type = pci_alloc_1intr(sc->bnx_dev, bnx_msi_enable,
+           &intr->bnx_intr_rid, &intr_flags);
+
+       intr->bnx_intr_res = bus_alloc_resource_any(sc->bnx_dev, SYS_RES_IRQ,
+           &intr->bnx_intr_rid, intr_flags);
+       if (intr->bnx_intr_res == NULL) {
+               device_printf(sc->bnx_dev, "could not alloc interrupt\n");
+               return ENXIO;
+       }
+
+       if (sc->bnx_intr_type == PCI_INTR_TYPE_MSI) {
+               sc->bnx_flags |= BNX_FLAG_ONESHOT_MSI;
+               bnx_enable_msi(sc);
+
+               if (sc->bnx_flags & BNX_FLAG_ONESHOT_MSI) {
+                       intr->bnx_intr_func = bnx_msi_oneshot;
+                       if (bootverbose)
+                               device_printf(sc->bnx_dev, "oneshot MSI\n");
+               } else {
+                       intr->bnx_intr_func = bnx_msi;
+               }
+       } else {
+               intr->bnx_intr_func = bnx_intr_legacy;
+       }
+       intr->bnx_intr_arg = sc;
+       intr->bnx_intr_cpuid = rman_get_cpuid(intr->bnx_intr_res);
+
+       intr->bnx_txr->bnx_tx_cpuid = intr->bnx_intr_cpuid;
 
        return 0;
 }
+
+static int
+bnx_setup_intr(struct bnx_softc *sc)
+{
+       int error, i;
+
+       for (i = 0; i < sc->bnx_intr_cnt; ++i) {
+               struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
+
+               error = bus_setup_intr_descr(sc->bnx_dev, intr->bnx_intr_res,
+                   INTR_MPSAFE, intr->bnx_intr_func, intr->bnx_intr_arg,
+                   &intr->bnx_intr_hand, intr->bnx_intr_serialize,
+                   intr->bnx_intr_desc);
+               if (error) {
+                       device_printf(sc->bnx_dev,
+                           "could not set up %dth intr\n", i);
+                       bnx_teardown_intr(sc, i);
+                       return error;
+               }
+       }
+       return 0;
+}
+
+static void
+bnx_teardown_intr(struct bnx_softc *sc, int cnt)
+{
+       int i;
+
+       for (i = 0; i < cnt; ++i) {
+               struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
+
+               bus_teardown_intr(sc->bnx_dev, intr->bnx_intr_res,
+                   intr->bnx_intr_hand);
+       }
+}
+
+static void
+bnx_free_intr(struct bnx_softc *sc)
+{
+       struct bnx_intr_data *intr;
+
+       KKASSERT(sc->bnx_intr_cnt <= 1);
+       intr = &sc->bnx_intr_data[0];
+
+       if (intr->bnx_intr_res != NULL) {
+               bus_release_resource(sc->bnx_dev, SYS_RES_IRQ,
+                   intr->bnx_intr_rid, intr->bnx_intr_res);
+       }
+       if (sc->bnx_intr_type == PCI_INTR_TYPE_MSI)
+               pci_release_msi(sc->bnx_dev);
+}
+
+static void
+bnx_setup_serialize(struct bnx_softc *sc)
+{
+       int i, j;
+
+       /*
+        * Allocate serializer array
+        */
+
+       /* Main + RX STD + TX + RX RET */
+       sc->bnx_serialize_cnt = 1 + 1 + sc->bnx_tx_ringcnt + sc->bnx_rx_retcnt;
+
+       sc->bnx_serialize =
+           kmalloc(sc->bnx_serialize_cnt * sizeof(struct lwkt_serialize *),
+               M_DEVBUF, M_WAITOK | M_ZERO);
+
+       /*
+        * Setup serializers
+        *
+        * NOTE: Order is critical
+        */
+
+       i = 0;
+
+       KKASSERT(i < sc->bnx_serialize_cnt);
+       sc->bnx_serialize[i++] = &sc->bnx_main_serialize;
+
+       KKASSERT(i < sc->bnx_serialize_cnt);
+       sc->bnx_serialize[i++] = &sc->bnx_rx_std_ring.bnx_rx_std_serialize;
+
+       for (j = 0; j < sc->bnx_rx_retcnt; ++j) {
+               KKASSERT(i < sc->bnx_serialize_cnt);
+               sc->bnx_serialize[i++] =
+                   &sc->bnx_rx_ret_ring[j].bnx_rx_ret_serialize;
+       }
+
+       for (j = 0; j < sc->bnx_tx_ringcnt; ++j) {
+               KKASSERT(i < sc->bnx_serialize_cnt);
+               sc->bnx_serialize[i++] =
+                   &sc->bnx_tx_ring[j].bnx_tx_serialize;
+       }
+
+       KKASSERT(i == sc->bnx_serialize_cnt);
+}
+
+static void
+bnx_serialize(struct ifnet *ifp, enum ifnet_serialize slz)
+{
+       struct bnx_softc *sc = ifp->if_softc;
+
+       ifnet_serialize_array_enter(sc->bnx_serialize,
+           sc->bnx_serialize_cnt, slz);
+}
+
+static void
+bnx_deserialize(struct ifnet *ifp, enum ifnet_serialize slz)
+{
+       struct bnx_softc *sc = ifp->if_softc;
+
+       ifnet_serialize_array_exit(sc->bnx_serialize,
+           sc->bnx_serialize_cnt, slz);
+}
+
+static int
+bnx_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz)
+{
+       struct bnx_softc *sc = ifp->if_softc;
+
+       return ifnet_serialize_array_try(sc->bnx_serialize,
+           sc->bnx_serialize_cnt, slz);
+}
+
+#ifdef INVARIANTS
+
+static void
+bnx_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz,
+    boolean_t serialized)
+{
+       struct bnx_softc *sc = ifp->if_softc;
+
+       ifnet_serialize_array_assert(sc->bnx_serialize, sc->bnx_serialize_cnt,
+           slz, serialized);
+}
+
+#endif /* INVARIANTS */