Add ifpoll, which support hardware TX/RX queues based polling.
[dragonfly.git] / sys / dev / netif / emx / if_emx.c
index 202dfdf..662a023 100644 (file)
@@ -64,7 +64,7 @@
  * SUCH DAMAGE.
  */
 
-#include "opt_polling.h"
+#include "opt_ifpoll.h"
 #include "opt_serializer.h"
 #include "opt_rss.h"
 #include "opt_emx.h"
@@ -80,6 +80,7 @@
 #include <sys/proc.h>
 #include <sys/rman.h>
 #include <sys/serialize.h>
+#include <sys/serialize2.h>
 #include <sys/socket.h>
 #include <sys/sockio.h>
 #include <sys/sysctl.h>
 #include <net/if_dl.h>
 #include <net/if_media.h>
 #include <net/ifq_var.h>
+#include <net/toeplitz.h>
+#include <net/toeplitz2.h>
 #include <net/vlan/if_vlan_var.h>
 #include <net/vlan/if_vlan_ether.h>
+#include <net/if_poll.h>
 
 #include <netinet/in_systm.h>
 #include <netinet/in.h>
 #ifdef EMX_RSS_DEBUG
 #define EMX_RSS_DPRINTF(sc, lvl, fmt, ...) \
 do { \
-       if (sc->rss_debug > lvl) \
+       if (sc->rss_debug >= lvl) \
                if_printf(&sc->arpcom.ac_if, fmt, __VA_ARGS__); \
 } while (0)
 #else  /* !EMX_RSS_DEBUG */
@@ -135,6 +139,7 @@ static const struct emx_device {
        EMX_DEVICE(82571EB_SERDES_DUAL),
        EMX_DEVICE(82571EB_SERDES_QUAD),
        EMX_DEVICE(82571EB_QUAD_COPPER),
+       EMX_DEVICE(82571EB_QUAD_COPPER_BP),
        EMX_DEVICE(82571EB_QUAD_COPPER_LP),
        EMX_DEVICE(82571EB_QUAD_FIBER),
        EMX_DEVICE(82571PT_QUAD_COPPER),
@@ -170,13 +175,20 @@ static void       emx_init(void *);
 static void    emx_stop(struct emx_softc *);
 static int     emx_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 static void    emx_start(struct ifnet *);
-#ifdef DEVICE_POLLING
-static void    emx_poll(struct ifnet *, enum poll_cmd, int);
+#ifdef IFPOLL_ENABLE
+static void    emx_qpoll(struct ifnet *, struct ifpoll_info *);
 #endif
 static void    emx_watchdog(struct ifnet *);
 static void    emx_media_status(struct ifnet *, struct ifmediareq *);
 static int     emx_media_change(struct ifnet *);
 static void    emx_timer(void *);
+static void    emx_serialize(struct ifnet *, enum ifnet_serialize);
+static void    emx_deserialize(struct ifnet *, enum ifnet_serialize);
+static int     emx_tryserialize(struct ifnet *, enum ifnet_serialize);
+#ifdef INVARIANTS
+static void    emx_serialize_assert(struct ifnet *, enum ifnet_serialize,
+                   boolean_t);
+#endif
 
 static void    emx_intr(void *);
 static void    emx_rxeof(struct emx_softc *, int, int);
@@ -224,6 +236,12 @@ static int emx_sysctl_int_throttle(SYSCTL_HANDLER_ARGS);
 static int     emx_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS);
 static void    emx_add_sysctl(struct emx_softc *);
 
+static void    emx_serialize_skipmain(struct emx_softc *);
+static void    emx_deserialize_skipmain(struct emx_softc *);
+#ifdef IFPOLL_ENABLE
+static int     emx_tryserialize_skipmain(struct emx_softc *);
+#endif
+
 /* Management and WOL Support */
 static void    emx_get_mgmt(struct emx_softc *);
 static void    emx_rel_mgmt(struct emx_softc *);
@@ -319,6 +337,45 @@ emx_rxcsum(uint32_t staterr, struct mbuf *mp)
        }
 }
 
+static __inline struct pktinfo *
+emx_rssinfo(struct mbuf *m, struct pktinfo *pi,
+           uint32_t mrq, uint32_t hash, uint32_t staterr)
+{
+       switch (mrq & EMX_RXDMRQ_RSSTYPE_MASK) {
+       case EMX_RXDMRQ_IPV4_TCP:
+               pi->pi_netisr = NETISR_IP;
+               pi->pi_flags = 0;
+               pi->pi_l3proto = IPPROTO_TCP;
+               break;
+
+       case EMX_RXDMRQ_IPV6_TCP:
+               pi->pi_netisr = NETISR_IPV6;
+               pi->pi_flags = 0;
+               pi->pi_l3proto = IPPROTO_TCP;
+               break;
+
+       case EMX_RXDMRQ_IPV4:
+               if (staterr & E1000_RXD_STAT_IXSM)
+                       return NULL;
+
+               if ((staterr &
+                    (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) ==
+                   E1000_RXD_STAT_TCPCS) {
+                       pi->pi_netisr = NETISR_IP;
+                       pi->pi_flags = 0;
+                       pi->pi_l3proto = IPPROTO_UDP;
+                       break;
+               }
+               /* FALL THROUGH */
+       default:
+               return NULL;
+       }
+
+       m->m_flags |= M_HASH;
+       m->m_pkthdr.hash = toeplitz_hash(hash);
+       return pi;
+}
+
 static int
 emx_probe(device_t dev)
 {
@@ -343,9 +400,21 @@ emx_attach(device_t dev)
 {
        struct emx_softc *sc = device_get_softc(dev);
        struct ifnet *ifp = &sc->arpcom.ac_if;
-       int error = 0;
+       int error = 0, i;
        uint16_t eeprom_data, device_id;
 
+       lwkt_serialize_init(&sc->main_serialize);
+       lwkt_serialize_init(&sc->tx_serialize);
+       for (i = 0; i < EMX_NRX_RING; ++i)
+               lwkt_serialize_init(&sc->rx_data[i].rx_serialize);
+
+       i = 0;
+       sc->serializes[i++] = &sc->main_serialize;
+       sc->serializes[i++] = &sc->tx_serialize;
+       sc->serializes[i++] = &sc->rx_data[0].rx_serialize;
+       sc->serializes[i++] = &sc->rx_data[1].rx_serialize;
+       KKASSERT(i == EMX_NSERIALIZE);
+
        callout_init(&sc->timer);
 
        sc->dev = sc->osdep.dev = dev;
@@ -597,7 +666,7 @@ emx_attach(device_t dev)
                sc->tx_int_nsegs = sc->oact_tx_desc;
 
        error = bus_setup_intr(dev, sc->intr_res, INTR_MPSAFE, emx_intr, sc,
-                              &sc->intr_tag, ifp->if_serializer);
+                              &sc->intr_tag, &sc->main_serialize);
        if (error) {
                device_printf(dev, "Failed to register interrupt handler");
                ether_ifdetach(&sc->arpcom.ac_if);
@@ -620,7 +689,7 @@ emx_detach(device_t dev)
        if (device_is_attached(dev)) {
                struct ifnet *ifp = &sc->arpcom.ac_if;
 
-               lwkt_serialize_enter(ifp->if_serializer);
+               ifnet_serialize_all(ifp);
 
                emx_stop(sc);
 
@@ -640,7 +709,7 @@ emx_detach(device_t dev)
 
                bus_teardown_intr(dev, sc->intr_res, sc->intr_tag);
 
-               lwkt_serialize_exit(ifp->if_serializer);
+               ifnet_deserialize_all(ifp);
 
                ether_ifdetach(ifp);
        }
@@ -677,7 +746,7 @@ emx_suspend(device_t dev)
        struct emx_softc *sc = device_get_softc(dev);
        struct ifnet *ifp = &sc->arpcom.ac_if;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
 
        emx_stop(sc);
 
@@ -693,7 +762,7 @@ emx_suspend(device_t dev)
                emx_enable_wol(dev);
         }
 
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 
        return bus_generic_suspend(dev);
 }
@@ -704,13 +773,13 @@ emx_resume(device_t dev)
        struct emx_softc *sc = device_get_softc(dev);
        struct ifnet *ifp = &sc->arpcom.ac_if;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
 
        emx_init(sc);
        emx_get_mgmt(sc);
        if_devstart(ifp);
 
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 
        return bus_generic_resume(dev);
 }
@@ -721,7 +790,7 @@ emx_start(struct ifnet *ifp)
        struct emx_softc *sc = ifp->if_softc;
        struct mbuf *m_head;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_SERIALIZED(&sc->tx_serialize);
 
        if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING)
                return;
@@ -770,7 +839,7 @@ emx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
        int max_frame_size, mask, reinit;
        int error = 0;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        switch (command) {
        case SIOCSIFMTU:
@@ -836,8 +905,8 @@ emx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
                if (ifp->if_flags & IFF_RUNNING) {
                        emx_disable_intr(sc);
                        emx_set_multi(sc);
-#ifdef DEVICE_POLLING
-                       if (!(ifp->if_flags & IFF_POLLING))
+#ifdef IFPOLL_ENABLE
+                       if (!(ifp->if_flags & IFF_NPOLLING))
 #endif
                                emx_enable_intr(sc);
                }
@@ -887,7 +956,7 @@ emx_watchdog(struct ifnet *ifp)
 {
        struct emx_softc *sc = ifp->if_softc;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        /*
         * The timer is set to 5 every time start queues a packet.
@@ -939,7 +1008,7 @@ emx_init(void *xsc)
        uint32_t pba;
        int i;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        emx_stop(sc);
 
@@ -1071,66 +1140,21 @@ emx_init(void *xsc)
                E1000_WRITE_REG(&sc->hw, E1000_IVAR, 0x800A0908);
        }
 
-#ifdef DEVICE_POLLING
+#ifdef IFPOLL_ENABLE
        /*
         * Only enable interrupts if we are not polling, make sure
         * they are off otherwise.
         */
-       if (ifp->if_flags & IFF_POLLING)
+       if (ifp->if_flags & IFF_NPOLLING)
                emx_disable_intr(sc);
        else
-#endif /* DEVICE_POLLING */
+#endif /* IFPOLL_ENABLE */
                emx_enable_intr(sc);
 
        /* Don't reset the phy next time init gets called */
        sc->hw.phy.reset_disable = TRUE;
 }
 
-#ifdef DEVICE_POLLING
-
-static void
-emx_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
-{
-       struct emx_softc *sc = ifp->if_softc;
-       uint32_t reg_icr;
-
-       ASSERT_SERIALIZED(ifp->if_serializer);
-
-       switch (cmd) {
-       case POLL_REGISTER:
-               emx_disable_intr(sc);
-               break;
-
-       case POLL_DEREGISTER:
-               emx_enable_intr(sc);
-               break;
-
-       case POLL_AND_CHECK_STATUS:
-               reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR);
-               if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
-                       callout_stop(&sc->timer);
-                       sc->hw.mac.get_link_status = 1;
-                       emx_update_link_status(sc);
-                       callout_reset(&sc->timer, hz, emx_timer, sc);
-               }
-               /* FALL THROUGH */
-       case POLL_ONLY:
-               if (ifp->if_flags & IFF_RUNNING) {
-                       int i;
-
-                       for (i = 0; i < sc->rx_ring_inuse; ++i)
-                               emx_rxeof(sc, i, count);
-
-                       emx_txeof(sc);
-                       if (!ifq_is_empty(&ifp->if_snd))
-                               if_devstart(ifp);
-               }
-               break;
-       }
-}
-
-#endif /* DEVICE_POLLING */
-
 static void
 emx_intr(void *xsc)
 {
@@ -1139,7 +1163,7 @@ emx_intr(void *xsc)
        uint32_t reg_icr;
 
        logif(intr_beg);
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_SERIALIZED(&sc->main_serialize);
 
        reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR);
 
@@ -1164,18 +1188,27 @@ emx_intr(void *xsc)
                    (E1000_ICR_RXT0 | E1000_ICR_RXDMT0 | E1000_ICR_RXO)) {
                        int i;
 
-                       for (i = 0; i < sc->rx_ring_inuse; ++i)
+                       for (i = 0; i < sc->rx_ring_inuse; ++i) {
+                               lwkt_serialize_enter(
+                               &sc->rx_data[i].rx_serialize);
                                emx_rxeof(sc, i, -1);
+                               lwkt_serialize_exit(
+                               &sc->rx_data[i].rx_serialize);
+                       }
                }
                if (reg_icr & E1000_ICR_TXDW) {
+                       lwkt_serialize_enter(&sc->tx_serialize);
                        emx_txeof(sc);
                        if (!ifq_is_empty(&ifp->if_snd))
                                if_devstart(ifp);
+                       lwkt_serialize_exit(&sc->tx_serialize);
                }
        }
 
        /* Link status change */
        if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
+               emx_serialize_skipmain(sc);
+
                callout_stop(&sc->timer);
                sc->hw.mac.get_link_status = 1;
                emx_update_link_status(sc);
@@ -1184,6 +1217,8 @@ emx_intr(void *xsc)
                emx_tx_purge(sc);
 
                callout_reset(&sc->timer, hz, emx_timer, sc);
+
+               emx_deserialize_skipmain(sc);
        }
 
        if (reg_icr & E1000_ICR_RXO)
@@ -1197,7 +1232,7 @@ emx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 {
        struct emx_softc *sc = ifp->if_softc;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        emx_update_link_status(sc);
 
@@ -1238,7 +1273,7 @@ emx_media_change(struct ifnet *ifp)
        struct emx_softc *sc = ifp->if_softc;
        struct ifmedia *ifm = &sc->media;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
                return (EINVAL);
@@ -1496,7 +1531,7 @@ emx_timer(void *xsc)
        struct emx_softc *sc = xsc;
        struct ifnet *ifp = &sc->arpcom.ac_if;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
 
        emx_update_link_status(sc);
        emx_update_stats(sc);
@@ -1512,7 +1547,7 @@ emx_timer(void *xsc)
 
        callout_reset(&sc->timer, hz, emx_timer, sc);
 
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 }
 
 static void
@@ -1604,7 +1639,7 @@ emx_stop(struct emx_softc *sc)
        struct ifnet *ifp = &sc->arpcom.ac_if;
        int i;
 
-       ASSERT_SERIALIZED(ifp->if_serializer);
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
 
        emx_disable_intr(sc);
 
@@ -1724,10 +1759,16 @@ emx_setup_ifp(struct emx_softc *sc)
        ifp->if_init =  emx_init;
        ifp->if_ioctl = emx_ioctl;
        ifp->if_start = emx_start;
-#ifdef DEVICE_POLLING
-       ifp->if_poll = emx_poll;
+#ifdef IFPOLL_ENABLE
+       ifp->if_qpoll = emx_qpoll;
 #endif
        ifp->if_watchdog = emx_watchdog;
+       ifp->if_serialize = emx_serialize;
+       ifp->if_deserialize = emx_deserialize;
+       ifp->if_tryserialize = emx_tryserialize;
+#ifdef INVARIANTS
+       ifp->if_serialize_assert = emx_serialize_assert;
+#endif
        ifq_set_maxlen(&ifp->if_snd, sc->num_tx_desc - 1);
        ifq_set_ready(&ifp->if_snd);
 
@@ -2559,7 +2600,7 @@ emx_init_rx_unit(struct emx_softc *sc)
 {
        struct ifnet *ifp = &sc->arpcom.ac_if;
        uint64_t bus_addr;
-       uint32_t rctl, rxcsum, rfctl, key, reta;
+       uint32_t rctl, rxcsum, rfctl;
        int i;
 
        /*
@@ -2646,6 +2687,13 @@ emx_init_rx_unit(struct emx_softc *sc)
         * Configure multiple receive queue (RSS)
         */
        if (ifp->if_capenable & IFCAP_RSS) {
+               uint8_t key[EMX_NRSSRK * EMX_RSSRK_SIZE];
+               uint32_t reta;
+
+               KASSERT(sc->rx_ring_inuse == EMX_NRX_RING,
+                       ("invalid number of RX ring (%d)",
+                        sc->rx_ring_inuse));
+
                /*
                 * NOTE:
                 * When we reach here, RSS has already been disabled
@@ -2656,14 +2704,29 @@ emx_init_rx_unit(struct emx_softc *sc)
                /*
                 * Configure RSS key
                 */
-               key = 0x5a6d5a6d; /* XXX */
-               for (i = 0; i < EMX_NRSSRK; ++i)
-                       E1000_WRITE_REG(&sc->hw, E1000_RSSRK(i), key);
+               toeplitz_get_key(key, sizeof(key));
+               for (i = 0; i < EMX_NRSSRK; ++i) {
+                       uint32_t rssrk;
+
+                       rssrk = EMX_RSSRK_VAL(key, i);
+                       EMX_RSS_DPRINTF(sc, 1, "rssrk%d 0x%08x\n", i, rssrk);
+
+                       E1000_WRITE_REG(&sc->hw, E1000_RSSRK(i), rssrk);
+               }
 
                /*
-                * Configure RSS redirect table
+                * Configure RSS redirect table in following fashion:
+                * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)]
                 */
-               reta = 0x80008000;
+               reta = 0;
+               for (i = 0; i < EMX_RETA_SIZE; ++i) {
+                       uint32_t q;
+
+                       q = (i % sc->rx_ring_inuse) << EMX_RETA_RINGIDX_SHIFT;
+                       reta |= q << (8 * i);
+               }
+               EMX_RSS_DPRINTF(sc, 1, "reta 0x%08x\n", reta);
+
                for (i = 0; i < EMX_NRETA; ++i)
                        E1000_WRITE_REG(&sc->hw, E1000_RETA(i), reta);
 
@@ -2756,6 +2819,7 @@ emx_rxeof(struct emx_softc *sc, int ring_idx, int count)
        ether_input_chain_init(chain);
 
        while ((staterr & E1000_RXD_STAT_DD) && count != 0) {
+               struct pktinfo *pi = NULL, pi0;
                struct emx_rxbuf *rx_buf = &rdata->rx_buf[i];
                struct mbuf *m = NULL;
                int eop, len;
@@ -2834,6 +2898,10 @@ emx_rxeof(struct emx_softc *sc, int ring_idx, int count)
                                rdata->fmp = NULL;
                                rdata->lmp = NULL;
 
+                               if (ifp->if_capenable & IFCAP_RSS) {
+                                       pi = emx_rssinfo(m, &pi0, mrq,
+                                                        rss_hash, staterr);
+                               }
 #ifdef EMX_RSS_DEBUG
                                rdata->rx_pkts++;
 #endif
@@ -2851,7 +2919,7 @@ discard:
                }
 
                if (m != NULL)
-                       ether_input_chain(ifp, m, chain);
+                       ether_input_chain(ifp, m, pi, chain);
 
                /* Advance our pointers to the next descriptor. */
                if (++i == rdata->num_rx_desc)
@@ -2873,7 +2941,7 @@ discard:
 static void
 emx_enable_intr(struct emx_softc *sc)
 {
-       lwkt_serialize_handler_enable(sc->arpcom.ac_if.if_serializer);
+       lwkt_serialize_handler_enable(&sc->main_serialize);
        E1000_WRITE_REG(&sc->hw, E1000_IMS, IMS_ENABLE_MASK);
 }
 
@@ -2881,7 +2949,7 @@ static void
 emx_disable_intr(struct emx_softc *sc)
 {
        E1000_WRITE_REG(&sc->hw, E1000_IMC, 0xffffffff);
-       lwkt_serialize_handler_disable(sc->arpcom.ac_if.if_serializer);
+       lwkt_serialize_handler_disable(&sc->main_serialize);
 }
 
 /*
@@ -3254,7 +3322,7 @@ emx_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
        sc = (struct emx_softc *)arg1;
        ifp = &sc->arpcom.ac_if;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
 
        if (result == 1)
                emx_print_debug_info(sc);
@@ -3267,7 +3335,7 @@ emx_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
        if (result == 2)
                emx_print_nvm_info(sc);
 
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 
        return (error);
 }
@@ -3286,9 +3354,9 @@ emx_sysctl_stats(SYSCTL_HANDLER_ARGS)
                struct emx_softc *sc = (struct emx_softc *)arg1;
                struct ifnet *ifp = &sc->arpcom.ac_if;
 
-               lwkt_serialize_enter(ifp->if_serializer);
+               ifnet_serialize_all(ifp);
                emx_print_hw_stats(sc);
-               lwkt_serialize_exit(ifp->if_serializer);
+               ifnet_deserialize_all(ifp);
        }
        return (error);
 }
@@ -3328,6 +3396,7 @@ emx_add_sysctl(struct emx_softc *sc)
        SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree),
                       OID_AUTO, "txd", CTLFLAG_RD, &sc->num_tx_desc, 0, NULL);
 
+#ifdef notyet
 #ifdef PROFILE_SERIALIZER
        SYSCTL_ADD_UINT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree),
                        OID_AUTO, "serializer_sleep", CTLFLAG_RW,
@@ -3341,6 +3410,7 @@ emx_add_sysctl(struct emx_softc *sc)
        SYSCTL_ADD_UINT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree),
                        OID_AUTO, "serializer_try", CTLFLAG_RW,
                        &ifp->if_serializer->try_cnt, 0, NULL);
+#endif
 #endif
 
        SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree),
@@ -3364,7 +3434,7 @@ emx_add_sysctl(struct emx_softc *sc)
                ksnprintf(rx_pkt, sizeof(rx_pkt), "rx%d_pkt", i);
                SYSCTL_ADD_UINT(&sc->sysctl_ctx,
                                SYSCTL_CHILDREN(sc->sysctl_tree), OID_AUTO,
-                               rx_pkt, CTLFLAG_RD,
+                               rx_pkt, CTLFLAG_RW,
                                &sc->rx_data[i].rx_pkts, 0, "RXed packets");
        }
 #endif
@@ -3396,7 +3466,7 @@ emx_sysctl_int_throttle(SYSCTL_HANDLER_ARGS)
                        return EINVAL;
        }
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
 
        if (throttle)
                sc->int_throttle_ceil = 1000000000 / 256 / throttle;
@@ -3406,7 +3476,7 @@ emx_sysctl_int_throttle(SYSCTL_HANDLER_ARGS)
        if (ifp->if_flags & IFF_RUNNING)
                E1000_WRITE_REG(&sc->hw, E1000_ITR, throttle);
 
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 
        if (bootverbose) {
                if_printf(ifp, "Interrupt moderation set to %d/sec\n",
@@ -3429,7 +3499,7 @@ emx_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS)
        if (segs <= 0)
                return EINVAL;
 
-       lwkt_serialize_enter(ifp->if_serializer);
+       ifnet_serialize_all(ifp);
 
        /*
         * Don't allow int_tx_nsegs to become:
@@ -3447,7 +3517,7 @@ emx_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS)
                sc->tx_int_nsegs = segs;
        }
 
-       lwkt_serialize_exit(ifp->if_serializer);
+       ifnet_deserialize_all(ifp);
 
        return error;
 }
@@ -3509,3 +3579,245 @@ emx_dma_free(struct emx_softc *sc)
        if (sc->parent_dtag != NULL)
                bus_dma_tag_destroy(sc->parent_dtag);
 }
+
+static void
+emx_serialize(struct ifnet *ifp, enum ifnet_serialize slz)
+{
+       struct emx_softc *sc = ifp->if_softc;
+
+       switch (slz) {
+       case IFNET_SERIALIZE_ALL:
+               lwkt_serialize_array_enter(sc->serializes, EMX_NSERIALIZE, 0);
+               break;
+
+       case IFNET_SERIALIZE_MAIN:
+               lwkt_serialize_enter(&sc->main_serialize);
+               break;
+
+       case IFNET_SERIALIZE_TX:
+               lwkt_serialize_enter(&sc->tx_serialize);
+               break;
+
+       case IFNET_SERIALIZE_RX(0):
+               lwkt_serialize_enter(&sc->rx_data[0].rx_serialize);
+               break;
+
+       case IFNET_SERIALIZE_RX(1):
+               lwkt_serialize_enter(&sc->rx_data[1].rx_serialize);
+               break;
+
+       default:
+               panic("%s unsupported serialize type\n", ifp->if_xname);
+       }
+}
+
+static void
+emx_deserialize(struct ifnet *ifp, enum ifnet_serialize slz)
+{
+       struct emx_softc *sc = ifp->if_softc;
+
+       switch (slz) {
+       case IFNET_SERIALIZE_ALL:
+               lwkt_serialize_array_exit(sc->serializes, EMX_NSERIALIZE, 0);
+               break;
+
+       case IFNET_SERIALIZE_MAIN:
+               lwkt_serialize_exit(&sc->main_serialize);
+               break;
+
+       case IFNET_SERIALIZE_TX:
+               lwkt_serialize_exit(&sc->tx_serialize);
+               break;
+
+       case IFNET_SERIALIZE_RX(0):
+               lwkt_serialize_exit(&sc->rx_data[0].rx_serialize);
+               break;
+
+       case IFNET_SERIALIZE_RX(1):
+               lwkt_serialize_exit(&sc->rx_data[1].rx_serialize);
+               break;
+
+       default:
+               panic("%s unsupported serialize type\n", ifp->if_xname);
+       }
+}
+
+static int
+emx_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz)
+{
+       struct emx_softc *sc = ifp->if_softc;
+
+       switch (slz) {
+       case IFNET_SERIALIZE_ALL:
+               return lwkt_serialize_array_try(sc->serializes,
+                                               EMX_NSERIALIZE, 0);
+
+       case IFNET_SERIALIZE_MAIN:
+               return lwkt_serialize_try(&sc->main_serialize);
+
+       case IFNET_SERIALIZE_TX:
+               return lwkt_serialize_try(&sc->tx_serialize);
+
+       case IFNET_SERIALIZE_RX(0):
+               return lwkt_serialize_try(&sc->rx_data[0].rx_serialize);
+
+       case IFNET_SERIALIZE_RX(1):
+               return lwkt_serialize_try(&sc->rx_data[1].rx_serialize);
+
+       default:
+               panic("%s unsupported serialize type\n", ifp->if_xname);
+       }
+}
+
+static void
+emx_serialize_skipmain(struct emx_softc *sc)
+{
+       lwkt_serialize_array_enter(sc->serializes, EMX_NSERIALIZE, 1);
+}
+
+#ifdef IFPOLL_ENABLE
+static int
+emx_tryserialize_skipmain(struct emx_softc *sc)
+{
+       return lwkt_serialize_array_try(sc->serializes, EMX_NSERIALIZE, 1);
+}
+#endif
+
+static void
+emx_deserialize_skipmain(struct emx_softc *sc)
+{
+       lwkt_serialize_array_exit(sc->serializes, EMX_NSERIALIZE, 1);
+}
+
+#ifdef INVARIANTS
+
+static void
+emx_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz,
+                    boolean_t serialized)
+{
+       struct emx_softc *sc = ifp->if_softc;
+       int i;
+
+       switch (slz) {
+       case IFNET_SERIALIZE_ALL:
+               if (serialized) {
+                       for (i = 0; i < EMX_NSERIALIZE; ++i)
+                               ASSERT_SERIALIZED(sc->serializes[i]);
+               } else {
+                       for (i = 0; i < EMX_NSERIALIZE; ++i)
+                               ASSERT_NOT_SERIALIZED(sc->serializes[i]);
+               }
+               break;
+
+       case IFNET_SERIALIZE_MAIN:
+               if (serialized)
+                       ASSERT_SERIALIZED(&sc->main_serialize);
+               else
+                       ASSERT_NOT_SERIALIZED(&sc->main_serialize);
+               break;
+
+       case IFNET_SERIALIZE_TX:
+               if (serialized)
+                       ASSERT_SERIALIZED(&sc->tx_serialize);
+               else
+                       ASSERT_NOT_SERIALIZED(&sc->tx_serialize);
+               break;
+
+       case IFNET_SERIALIZE_RX(0):
+               if (serialized)
+                       ASSERT_SERIALIZED(&sc->rx_data[0].rx_serialize);
+               else
+                       ASSERT_NOT_SERIALIZED(&sc->rx_data[0].rx_serialize);
+               break;
+
+       case IFNET_SERIALIZE_RX(1):
+               if (serialized)
+                       ASSERT_SERIALIZED(&sc->rx_data[1].rx_serialize);
+               else
+                       ASSERT_NOT_SERIALIZED(&sc->rx_data[1].rx_serialize);
+               break;
+
+       default:
+               panic("%s unsupported serialize type\n", ifp->if_xname);
+       }
+}
+
+#endif /* INVARIANTS */
+
+#ifdef IFPOLL_ENABLE
+
+static void
+emx_qpoll_status(struct ifnet *ifp, int pollhz __unused)
+{
+       struct emx_softc *sc = ifp->if_softc;
+       uint32_t reg_icr;
+
+       ASSERT_SERIALIZED(&sc->main_serialize);
+
+       reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR);
+       if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
+               if (emx_tryserialize_skipmain(sc)) {
+                       callout_stop(&sc->timer);
+                       sc->hw.mac.get_link_status = 1;
+                       emx_update_link_status(sc);
+                       callout_reset(&sc->timer, hz, emx_timer, sc);
+                       emx_deserialize_skipmain(sc);
+               }
+       }
+}
+
+static void
+emx_qpoll_tx(struct ifnet *ifp, void *arg __unused, int cycle __unused)
+{
+       struct emx_softc *sc = ifp->if_softc;
+
+       ASSERT_SERIALIZED(&sc->tx_serialize);
+
+       emx_txeof(sc);
+       if (!ifq_is_empty(&ifp->if_snd))
+               if_devstart(ifp);
+}
+
+static void
+emx_qpoll_rx(struct ifnet *ifp, void *arg, int cycle)
+{
+       struct emx_softc *sc = ifp->if_softc;
+       struct emx_rxdata *rdata = arg;
+
+       ASSERT_SERIALIZED(&rdata->rx_serialize);
+
+       emx_rxeof(sc, rdata - sc->rx_data, cycle);
+}
+
+static void
+emx_qpoll(struct ifnet *ifp, struct ifpoll_info *info)
+{
+       struct emx_softc *sc = ifp->if_softc;
+
+       ASSERT_IFNET_SERIALIZED_ALL(ifp);
+
+       if (info) {
+               int i;
+
+               info->ifpi_status.status_func = emx_qpoll_status;
+               info->ifpi_status.serializer = &sc->main_serialize;
+
+               info->ifpi_tx[0].poll_func = emx_qpoll_tx;
+               info->ifpi_tx[0].arg = NULL;
+               info->ifpi_tx[0].serializer = &sc->tx_serialize;
+
+               for (i = 0; i < sc->rx_ring_cnt; ++i) {
+                       info->ifpi_rx[i].poll_func = emx_qpoll_rx;
+                       info->ifpi_rx[i].arg = &sc->rx_data[i];
+                       info->ifpi_rx[i].serializer =
+                               &sc->rx_data[i].rx_serialize;
+               }
+
+               if (ifp->if_flags & IFF_RUNNING)
+                       emx_disable_intr(sc);
+       } else if (ifp->if_flags & IFF_RUNNING) {
+               emx_enable_intr(sc);
+       }
+}
+
+#endif /* IFPOLL_ENABLE */