Add ifpoll, which support hardware TX/RX queues based polling.
[dragonfly.git] / sys / dev / netif / emx / if_emx.c
index 9fe7841..662a023 100644 (file)
@@ -64,7 +64,7 @@
  * SUCH DAMAGE.
  */
 
-#include "opt_polling.h"
+#include "opt_ifpoll.h"
 #include "opt_serializer.h"
 #include "opt_rss.h"
 #include "opt_emx.h"
@@ -80,6 +80,7 @@
 #include <sys/proc.h>
 #include <sys/rman.h>
 #include <sys/serialize.h>
+#include <sys/serialize2.h>
 #include <sys/socket.h>
 #include <sys/sockio.h>
 #include <sys/sysctl.h>
 #include <net/if_media.h>
 #include <net/ifq_var.h>
 #include <net/toeplitz.h>
+#include <net/toeplitz2.h>
 #include <net/vlan/if_vlan_var.h>
 #include <net/vlan/if_vlan_ether.h>
+#include <net/if_poll.h>
 
 #include <netinet/in_systm.h>
 #include <netinet/in.h>
@@ -136,6 +139,7 @@ static const struct emx_device {
        EMX_DEVICE(82571EB_SERDES_DUAL),
        EMX_DEVICE(82571EB_SERDES_QUAD),
        EMX_DEVICE(82571EB_QUAD_COPPER),
+       EMX_DEVICE(82571EB_QUAD_COPPER_BP),
        EMX_DEVICE(82571EB_QUAD_COPPER_LP),
        EMX_DEVICE(82571EB_QUAD_FIBER),
        EMX_DEVICE(82571PT_QUAD_COPPER),
@@ -171,13 +175,20 @@ static void       emx_init(void *);
 static void    emx_stop(struct emx_softc *);
 static int     emx_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 static void    emx_start(struct ifnet *);
-#ifdef DEVICE_POLLING
-static void    emx_poll(struct ifnet *, enum poll_cmd, int);
+#ifdef IFPOLL_ENABLE
+static void    emx_qpoll(struct ifnet *, struct ifpoll_info *);
 #endif
 static void    emx_watchdog(struct ifnet *);
 static void    emx_media_status(struct ifnet *, struct ifmediareq *);
 static int     emx_media_change(struct ifnet *);
 static void    emx_timer(void *);
+static void    emx_serialize(struct ifnet *, enum ifnet_serialize);
+static void    emx_deserialize(struct ifnet *, enum ifnet_serialize);
+static int     emx_tryserialize(struct ifnet *, enum ifnet_serialize);
+#ifdef INVARIANTS
+static void    emx_serialize_assert(struct ifnet *, enum ifnet_serialize,
+                   boolean_t);
+#endif
 
 static void    emx_intr(void *);
 static void    emx_rxeof(struct emx_softc *, int, int);
@@ -225,6 +236,12 @@ static int emx_sysctl_int_throttle(SYSCTL_HANDLER_ARGS);
 static int     emx_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS);
 static void    emx_add_sysctl(struct emx_softc *);
 
+static void    emx_serialize_skipmain(struct emx_softc *);
+static void    emx_deserialize_skipmain(struct emx_softc *);
+#ifdef IFPOLL_ENABLE
+static int     emx_tryserialize_skipmain(struct emx_softc *);
+#endif
+
 /* Management and WOL Support */
 static void    emx_get_mgmt(struct emx_softc *);
 static void    emx_rel_mgmt(struct emx_softc *);
@@ -320,6 +337,45 @@ emx_rxcsum(uint32_t staterr, struct mbuf *mp)
        }
 }
 
+static __inline struct pktinfo *
+emx_rssinfo(struct mbuf *m, struct pktinfo *pi,
+           uint32_t mrq, uint32_t hash, uint32_t staterr)
+{
+       switch (mrq & EMX_RXDMRQ_RSSTYPE_MASK) {
+       case EMX_RXDMRQ_IPV4_TCP:
+               pi->pi_netisr = NETISR_IP;
+               pi->pi_flags = 0;
+               pi->pi_l3proto = IPPROTO_TCP;
+               break;
+
+       case EMX_RXDMRQ_IPV6_TCP:
+               pi->pi_netisr = NETISR_IPV6;
+               pi->pi_flags = 0;
+               pi->pi_l3proto = IPPROTO_TCP;
+               break;
+
+       case EMX_RXDMRQ_IPV4:
+               if (staterr & E1000_RXD_STAT_IXSM)
+                       return NULL;
+
+               if ((staterr &
+                    (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) ==
+                   E1000_RXD_STAT_TCPCS) {
+                       pi->pi_netisr = NETISR_IP;
+                       pi->pi_flags = 0;
+                       pi->pi_l3proto = IPPROTO_UDP;
+                       break;
+               }
+               /* FALL THROUGH */
+       default:
+               return NULL;
+       }
+
+       m->m_flags |= M_HASH;
+       m->m_pkthdr.hash = toeplitz_hash(hash);
+       return pi;
+}
+
 static int
 emx_probe(device_t dev)
 {
@@ -344,9 +400,21 @@ emx_attach(device_t dev)
 {
        struct emx_softc *sc = device_get_softc(dev);
        struct ifnet *ifp = &sc->arpcom.ac_if;
-       int error = 0;
+       int error = 0, i;
        uint16_t eeprom_data, device_id;
 
+       lwkt_serialize_init(&sc->main_serialize);
+       lwkt_serialize_init(&sc->tx_serialize);
+       for (i = 0; i < EMX_NRX_RING; ++i)
+               lwkt_serialize_init(&sc->rx_data[i].rx_serialize);
+
+       i = 0;
+       sc->serializes[i++] = &sc->main_serialize;
+       sc->serializes[i++] = &sc->tx_serialize;
+       sc->serializes[i++] = &sc->rx_data[0].rx_serialize;
+       sc->serializes[i++] = &sc->rx_data[1].rx_serialize;
+       KKASSERT(i == EMX_NSERIALIZE);
+
        callout_init(&sc->timer);
 
        sc->dev = sc->osdep.dev = dev;
@@ -598,7 +666,7 @@ emx_attach(device_t dev)
                sc->tx_int_nsegs = sc->oact_tx_desc;
 
        error = bus_setup_intr(dev, sc->intr_res, INTR_MPSAFE, emx_intr, sc,
-                              &sc->intr_tag, ifp->if_serializer);
+                              &sc->intr_tag, &sc->main_serialize);
        if (error) {
                device_printf(dev, "Failed to register interrupt handler");
                ether_ifdetach(&sc->arpcom.ac_if);
@@ -621,7 +689,7 @@ emx_detach(device_t dev)
        if (device_is_attached(dev)) {
                struct ifnet *ifp = &sc->arpcom.ac_if;
 
-               lwkt_serialize_enter(ifp->if_serializer);
+               ifnet_serialize_all(ifp);
 
                emx_stop(sc);
 
@@ -641,7 +709,7 @@ emx_detach(device_t dev)
 
                bus_teardown_intr(dev, sc->intr_res, sc->intr_tag);
 
-               lwkt_serialize_exit(ifp->if_serializer);
+               ifnet_deserialize_all(ifp);
 
                ether_ifdetach(ifp);
        }
@@ -678,7 +746,7 @@ emx_suspend(device_t dev)
        struct emx_softc *sc = device_get_softc(dev);
        struct ifnet *ifp = &sc->arpcom.ac_if;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
 
        emx_stop(sc);
 
@@ -694,7 +762,7 @@ emx_suspend(device_t dev)
                emx_enable_wol(dev);
         }
 
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 
        return bus_generic_suspend(dev);
 }
@@ -705,13 +773,13 @@ emx_resume(device_t dev)
        struct emx_softc *sc = device_get_softc(dev);
        struct ifnet *ifp = &sc->arpcom.ac_if;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
 
        emx_init(sc);
        emx_get_mgmt(sc);
        if_devstart(ifp);
 
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 
        return bus_generic_resume(dev);
 }
@@ -722,7 +790,7 @@ emx_start(struct ifnet *ifp)
        struct emx_softc *sc = ifp->if_softc;
        struct mbuf *m_head;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_SERIALIZED(&sc->tx_serialize);
 
        if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING)
                return;
@@ -771,7 +839,7 @@ emx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
        int max_frame_size, mask, reinit;
        int error = 0;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        switch (command) {
        case SIOCSIFMTU:
@@ -837,8 +905,8 @@ emx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
                if (ifp->if_flags & IFF_RUNNING) {
                        emx_disable_intr(sc);
                        emx_set_multi(sc);
-#ifdef DEVICE_POLLING
-                       if (!(ifp->if_flags & IFF_POLLING))
+#ifdef IFPOLL_ENABLE
+                       if (!(ifp->if_flags & IFF_NPOLLING))
 #endif
                                emx_enable_intr(sc);
                }
@@ -888,7 +956,7 @@ emx_watchdog(struct ifnet *ifp)
 {
        struct emx_softc *sc = ifp->if_softc;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        /*
         * The timer is set to 5 every time start queues a packet.
@@ -940,7 +1008,7 @@ emx_init(void *xsc)
        uint32_t pba;
        int i;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        emx_stop(sc);
 
@@ -1072,66 +1140,21 @@ emx_init(void *xsc)
                E1000_WRITE_REG(&sc->hw, E1000_IVAR, 0x800A0908);
        }
 
-#ifdef DEVICE_POLLING
+#ifdef IFPOLL_ENABLE
        /*
         * Only enable interrupts if we are not polling, make sure
         * they are off otherwise.
         */
-       if (ifp->if_flags & IFF_POLLING)
+       if (ifp->if_flags & IFF_NPOLLING)
                emx_disable_intr(sc);
        else
-#endif /* DEVICE_POLLING */
+#endif /* IFPOLL_ENABLE */
                emx_enable_intr(sc);
 
        /* Don't reset the phy next time init gets called */
        sc->hw.phy.reset_disable = TRUE;
 }
 
-#ifdef DEVICE_POLLING
-
-static void
-emx_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
-{
-       struct emx_softc *sc = ifp->if_softc;
-       uint32_t reg_icr;
-
-       ASSERT_SERIALIZED(ifp->if_serializer);
-
-       switch (cmd) {
-       case POLL_REGISTER:
-               emx_disable_intr(sc);
-               break;
-
-       case POLL_DEREGISTER:
-               emx_enable_intr(sc);
-               break;
-
-       case POLL_AND_CHECK_STATUS:
-               reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR);
-               if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
-                       callout_stop(&sc->timer);
-                       sc->hw.mac.get_link_status = 1;
-                       emx_update_link_status(sc);
-                       callout_reset(&sc->timer, hz, emx_timer, sc);
-               }
-               /* FALL THROUGH */
-       case POLL_ONLY:
-               if (ifp->if_flags & IFF_RUNNING) {
-                       int i;
-
-                       for (i = 0; i < sc->rx_ring_inuse; ++i)
-                               emx_rxeof(sc, i, count);
-
-                       emx_txeof(sc);
-                       if (!ifq_is_empty(&ifp->if_snd))
-                               if_devstart(ifp);
-               }
-               break;
-       }
-}
-
-#endif /* DEVICE_POLLING */
-
 static void
 emx_intr(void *xsc)
 {
@@ -1140,7 +1163,7 @@ emx_intr(void *xsc)
        uint32_t reg_icr;
 
        logif(intr_beg);
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_SERIALIZED(&sc->main_serialize);
 
        reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR);
 
@@ -1165,18 +1188,27 @@ emx_intr(void *xsc)
                    (E1000_ICR_RXT0 | E1000_ICR_RXDMT0 | E1000_ICR_RXO)) {
                        int i;
 
-                       for (i = 0; i < sc->rx_ring_inuse; ++i)
+                       for (i = 0; i < sc->rx_ring_inuse; ++i) {
+                               lwkt_serialize_enter(
+                               &sc->rx_data[i].rx_serialize);
                                emx_rxeof(sc, i, -1);
+                               lwkt_serialize_exit(
+                               &sc->rx_data[i].rx_serialize);
+                       }
                }
                if (reg_icr & E1000_ICR_TXDW) {
+                       lwkt_serialize_enter(&sc->tx_serialize);
                        emx_txeof(sc);
                        if (!ifq_is_empty(&ifp->if_snd))
                                if_devstart(ifp);
+                       lwkt_serialize_exit(&sc->tx_serialize);
                }
        }
 
        /* Link status change */
        if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
+               emx_serialize_skipmain(sc);
+
                callout_stop(&sc->timer);
                sc->hw.mac.get_link_status = 1;
                emx_update_link_status(sc);
@@ -1185,6 +1217,8 @@ emx_intr(void *xsc)
                emx_tx_purge(sc);
 
                callout_reset(&sc->timer, hz, emx_timer, sc);
+
+               emx_deserialize_skipmain(sc);
        }
 
        if (reg_icr & E1000_ICR_RXO)
@@ -1198,7 +1232,7 @@ emx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 {
        struct emx_softc *sc = ifp->if_softc;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        emx_update_link_status(sc);
 
@@ -1239,7 +1273,7 @@ emx_media_change(struct ifnet *ifp)
        struct emx_softc *sc = ifp->if_softc;
        struct ifmedia *ifm = &sc->media;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
                return (EINVAL);
@@ -1497,7 +1531,7 @@ emx_timer(void *xsc)
        struct emx_softc *sc = xsc;
        struct ifnet *ifp = &sc->arpcom.ac_if;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
 
        emx_update_link_status(sc);
        emx_update_stats(sc);
@@ -1513,7 +1547,7 @@ emx_timer(void *xsc)
 
        callout_reset(&sc->timer, hz, emx_timer, sc);
 
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 }
 
 static void
@@ -1605,7 +1639,7 @@ emx_stop(struct emx_softc *sc)
        struct ifnet *ifp = &sc->arpcom.ac_if;
        int i;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        emx_disable_intr(sc);
 
@@ -1725,10 +1759,16 @@ emx_setup_ifp(struct emx_softc *sc)
        ifp->if_init =  emx_init;
        ifp->if_ioctl = emx_ioctl;
        ifp->if_start = emx_start;
-#ifdef DEVICE_POLLING
-       ifp->if_poll = emx_poll;
+#ifdef IFPOLL_ENABLE
+       ifp->if_qpoll = emx_qpoll;
 #endif
        ifp->if_watchdog = emx_watchdog;
+       ifp->if_serialize = emx_serialize;
+       ifp->if_deserialize = emx_deserialize;
+       ifp->if_tryserialize = emx_tryserialize;
+#ifdef INVARIANTS
+       ifp->if_serialize_assert = emx_serialize_assert;
+#endif
        ifq_set_maxlen(&ifp->if_snd, sc->num_tx_desc - 1);
        ifq_set_ready(&ifp->if_snd);
 
@@ -2779,6 +2819,7 @@ emx_rxeof(struct emx_softc *sc, int ring_idx, int count)
        ether_input_chain_init(chain);
 
        while ((staterr & E1000_RXD_STAT_DD) && count != 0) {
+               struct pktinfo *pi = NULL, pi0;
                struct emx_rxbuf *rx_buf = &rdata->rx_buf[i];
                struct mbuf *m = NULL;
                int eop, len;
@@ -2857,6 +2898,10 @@ emx_rxeof(struct emx_softc *sc, int ring_idx, int count)
                                rdata->fmp = NULL;
                                rdata->lmp = NULL;
 
+                               if (ifp->if_capenable & IFCAP_RSS) {
+                                       pi = emx_rssinfo(m, &pi0, mrq,
+                                                        rss_hash, staterr);
+                               }
 #ifdef EMX_RSS_DEBUG
                                rdata->rx_pkts++;
 #endif
@@ -2874,7 +2919,7 @@ discard:
                }
 
                if (m != NULL)
-                       ether_input_chain(ifp, m, NULL, chain);
+                       ether_input_chain(ifp, m, pi, chain);
 
                /* Advance our pointers to the next descriptor. */
                if (++i == rdata->num_rx_desc)
@@ -2896,7 +2941,7 @@ discard:
 static void
 emx_enable_intr(struct emx_softc *sc)
 {
-       lwkt_serialize_handler_enable(sc->arpcom.ac_if.if_serializer);
+       lwkt_serialize_handler_enable(&sc->main_serialize);
        E1000_WRITE_REG(&sc->hw, E1000_IMS, IMS_ENABLE_MASK);
 }
 
@@ -2904,7 +2949,7 @@ static void
 emx_disable_intr(struct emx_softc *sc)
 {
        E1000_WRITE_REG(&sc->hw, E1000_IMC, 0xffffffff);
-       lwkt_serialize_handler_disable(sc->arpcom.ac_if.if_serializer);
+       lwkt_serialize_handler_disable(&sc->main_serialize);
 }
 
 /*
@@ -3277,7 +3322,7 @@ emx_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
        sc = (struct emx_softc *)arg1;
        ifp = &sc->arpcom.ac_if;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
 
        if (result == 1)
                emx_print_debug_info(sc);
@@ -3290,7 +3335,7 @@ emx_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
        if (result == 2)
                emx_print_nvm_info(sc);
 
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 
        return (error);
 }
@@ -3309,9 +3354,9 @@ emx_sysctl_stats(SYSCTL_HANDLER_ARGS)
                struct emx_softc *sc = (struct emx_softc *)arg1;
                struct ifnet *ifp = &sc->arpcom.ac_if;
 
-               lwkt_serialize_enter(ifp->if_serializer);
+               ifnet_serialize_all(ifp);
                emx_print_hw_stats(sc);
-               lwkt_serialize_exit(ifp->if_serializer);
+               ifnet_deserialize_all(ifp);
        }
        return (error);
 }
@@ -3351,6 +3396,7 @@ emx_add_sysctl(struct emx_softc *sc)
        SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree),
                       OID_AUTO, "txd", CTLFLAG_RD, &sc->num_tx_desc, 0, NULL);
 
+#ifdef notyet
 #ifdef PROFILE_SERIALIZER
        SYSCTL_ADD_UINT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree),
                        OID_AUTO, "serializer_sleep", CTLFLAG_RW,
@@ -3364,6 +3410,7 @@ emx_add_sysctl(struct emx_softc *sc)
        SYSCTL_ADD_UINT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree),
                        OID_AUTO, "serializer_try", CTLFLAG_RW,
                        &ifp->if_serializer->try_cnt, 0, NULL);
+#endif
 #endif
 
        SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree),
@@ -3419,7 +3466,7 @@ emx_sysctl_int_throttle(SYSCTL_HANDLER_ARGS)
                        return EINVAL;
        }
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
 
        if (throttle)
                sc->int_throttle_ceil = 1000000000 / 256 / throttle;
@@ -3429,7 +3476,7 @@ emx_sysctl_int_throttle(SYSCTL_HANDLER_ARGS)
        if (ifp->if_flags & IFF_RUNNING)
                E1000_WRITE_REG(&sc->hw, E1000_ITR, throttle);
 
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 
        if (bootverbose) {
                if_printf(ifp, "Interrupt moderation set to %d/sec\n",
@@ -3452,7 +3499,7 @@ emx_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS)
        if (segs <= 0)
                return EINVAL;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
 
        /*
         * Don't allow int_tx_nsegs to become:
@@ -3470,7 +3517,7 @@ emx_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS)
                sc->tx_int_nsegs = segs;
        }
 
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 
        return error;
 }
@@ -3532,3 +3579,245 @@ emx_dma_free(struct emx_softc *sc)
        if (sc->parent_dtag != NULL)
                bus_dma_tag_destroy(sc->parent_dtag);
 }
+
+static void
+emx_serialize(struct ifnet *ifp, enum ifnet_serialize slz)
+{
+       struct emx_softc *sc = ifp->if_softc;
+
+       switch (slz) {
+       case IFNET_SERIALIZE_ALL:
+               lwkt_serialize_array_enter(sc->serializes, EMX_NSERIALIZE, 0);
+               break;
+
+       case IFNET_SERIALIZE_MAIN:
+               lwkt_serialize_enter(&sc->main_serialize);
+               break;
+
+       case IFNET_SERIALIZE_TX:
+               lwkt_serialize_enter(&sc->tx_serialize);
+               break;
+
+       case IFNET_SERIALIZE_RX(0):
+               lwkt_serialize_enter(&sc->rx_data[0].rx_serialize);
+               break;
+
+       case IFNET_SERIALIZE_RX(1):
+               lwkt_serialize_enter(&sc->rx_data[1].rx_serialize);
+               break;
+
+       default:
+               panic("%s unsupported serialize type\n", ifp->if_xname);
+       }
+}
+
+static void
+emx_deserialize(struct ifnet *ifp, enum ifnet_serialize slz)
+{
+       struct emx_softc *sc = ifp->if_softc;
+
+       switch (slz) {
+       case IFNET_SERIALIZE_ALL:
+               lwkt_serialize_array_exit(sc->serializes, EMX_NSERIALIZE, 0);
+               break;
+
+       case IFNET_SERIALIZE_MAIN:
+               lwkt_serialize_exit(&sc->main_serialize);
+               break;
+
+       case IFNET_SERIALIZE_TX:
+               lwkt_serialize_exit(&sc->tx_serialize);
+               break;
+
+       case IFNET_SERIALIZE_RX(0):
+               lwkt_serialize_exit(&sc->rx_data[0].rx_serialize);
+               break;
+
+       case IFNET_SERIALIZE_RX(1):
+               lwkt_serialize_exit(&sc->rx_data[1].rx_serialize);
+               break;
+
+       default:
+               panic("%s unsupported serialize type\n", ifp->if_xname);
+       }
+}
+
+static int
+emx_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz)
+{
+       struct emx_softc *sc = ifp->if_softc;
+
+       switch (slz) {
+       case IFNET_SERIALIZE_ALL:
+               return lwkt_serialize_array_try(sc->serializes,
+                                               EMX_NSERIALIZE, 0);
+
+       case IFNET_SERIALIZE_MAIN:
+               return lwkt_serialize_try(&sc->main_serialize);
+
+       case IFNET_SERIALIZE_TX:
+               return lwkt_serialize_try(&sc->tx_serialize);
+
+       case IFNET_SERIALIZE_RX(0):
+               return lwkt_serialize_try(&sc->rx_data[0].rx_serialize);
+
+       case IFNET_SERIALIZE_RX(1):
+               return lwkt_serialize_try(&sc->rx_data[1].rx_serialize);
+
+       default:
+               panic("%s unsupported serialize type\n", ifp->if_xname);
+       }
+}
+
+static void
+emx_serialize_skipmain(struct emx_softc *sc)
+{
+       lwkt_serialize_array_enter(sc->serializes, EMX_NSERIALIZE, 1);
+}
+
+#ifdef IFPOLL_ENABLE
+static int
+emx_tryserialize_skipmain(struct emx_softc *sc)
+{
+       return lwkt_serialize_array_try(sc->serializes, EMX_NSERIALIZE, 1);
+}
+#endif
+
+static void
+emx_deserialize_skipmain(struct emx_softc *sc)
+{
+       lwkt_serialize_array_exit(sc->serializes, EMX_NSERIALIZE, 1);
+}
+
+#ifdef INVARIANTS
+
+static void
+emx_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz,
+                    boolean_t serialized)
+{
+       struct emx_softc *sc = ifp->if_softc;
+       int i;
+
+       switch (slz) {
+       case IFNET_SERIALIZE_ALL:
+               if (serialized) {
+                       for (i = 0; i < EMX_NSERIALIZE; ++i)
+                               ASSERT_SERIALIZED(sc->serializes[i]);
+               } else {
+                       for (i = 0; i < EMX_NSERIALIZE; ++i)
+                               ASSERT_NOT_SERIALIZED(sc->serializes[i]);
+               }
+               break;
+
+       case IFNET_SERIALIZE_MAIN:
+               if (serialized)
+                       ASSERT_SERIALIZED(&sc->main_serialize);
+               else
+                       ASSERT_NOT_SERIALIZED(&sc->main_serialize);
+               break;
+
+       case IFNET_SERIALIZE_TX:
+               if (serialized)
+                       ASSERT_SERIALIZED(&sc->tx_serialize);
+               else
+                       ASSERT_NOT_SERIALIZED(&sc->tx_serialize);
+               break;
+
+       case IFNET_SERIALIZE_RX(0):
+               if (serialized)
+                       ASSERT_SERIALIZED(&sc->rx_data[0].rx_serialize);
+               else
+                       ASSERT_NOT_SERIALIZED(&sc->rx_data[0].rx_serialize);
+               break;
+
+       case IFNET_SERIALIZE_RX(1):
+               if (serialized)
+                       ASSERT_SERIALIZED(&sc->rx_data[1].rx_serialize);
+               else
+                       ASSERT_NOT_SERIALIZED(&sc->rx_data[1].rx_serialize);
+               break;
+
+       default:
+               panic("%s unsupported serialize type\n", ifp->if_xname);
+       }
+}
+
+#endif /* INVARIANTS */
+
+#ifdef IFPOLL_ENABLE
+
+static void
+emx_qpoll_status(struct ifnet *ifp, int pollhz __unused)
+{
+       struct emx_softc *sc = ifp->if_softc;
+       uint32_t reg_icr;
+
+       ASSERT_SERIALIZED(&sc->main_serialize);
+
+       reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR);
+       if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
+               if (emx_tryserialize_skipmain(sc)) {
+                       callout_stop(&sc->timer);
+                       sc->hw.mac.get_link_status = 1;
+                       emx_update_link_status(sc);
+                       callout_reset(&sc->timer, hz, emx_timer, sc);
+                       emx_deserialize_skipmain(sc);
+               }
+       }
+}
+
+static void
+emx_qpoll_tx(struct ifnet *ifp, void *arg __unused, int cycle __unused)
+{
+       struct emx_softc *sc = ifp->if_softc;
+
+       ASSERT_SERIALIZED(&sc->tx_serialize);
+
+       emx_txeof(sc);
+       if (!ifq_is_empty(&ifp->if_snd))
+               if_devstart(ifp);
+}
+
+static void
+emx_qpoll_rx(struct ifnet *ifp, void *arg, int cycle)
+{
+       struct emx_softc *sc = ifp->if_softc;
+       struct emx_rxdata *rdata = arg;
+
+       ASSERT_SERIALIZED(&rdata->rx_serialize);
+
+       emx_rxeof(sc, rdata - sc->rx_data, cycle);
+}
+
+static void
+emx_qpoll(struct ifnet *ifp, struct ifpoll_info *info)
+{
+       struct emx_softc *sc = ifp->if_softc;
+
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
+
+       if (info) {
+               int i;
+
+               info->ifpi_status.status_func = emx_qpoll_status;
+               info->ifpi_status.serializer = &sc->main_serialize;
+
+               info->ifpi_tx[0].poll_func = emx_qpoll_tx;
+               info->ifpi_tx[0].arg = NULL;
+               info->ifpi_tx[0].serializer = &sc->tx_serialize;
+
+               for (i = 0; i < sc->rx_ring_cnt; ++i) {
+                       info->ifpi_rx[i].poll_func = emx_qpoll_rx;
+                       info->ifpi_rx[i].arg = &sc->rx_data[i];
+                       info->ifpi_rx[i].serializer =
+                               &sc->rx_data[i].rx_serialize;
+               }
+
+               if (ifp->if_flags & IFF_RUNNING)
+                       emx_disable_intr(sc);
+       } else if (ifp->if_flags & IFF_RUNNING) {
+               emx_enable_intr(sc);
+       }
+}
+
+#endif /* IFPOLL_ENABLE */