X-Git-Url: https://gitweb.dragonflybsd.org/dragonfly.git/blobdiff_plain/6446af7bd901fbf9b200a4867083702021fe7cda..b3a7093f50ab968ffe0b9d2663cf41b81cc5431f:/sys/dev/netif/emx/if_emx.c diff --git a/sys/dev/netif/emx/if_emx.c b/sys/dev/netif/emx/if_emx.c index 4041510d7f..662a023df7 100644 --- a/sys/dev/netif/emx/if_emx.c +++ b/sys/dev/netif/emx/if_emx.c @@ -64,8 +64,10 @@ * SUCH DAMAGE. */ -#include "opt_polling.h" +#include "opt_ifpoll.h" #include "opt_serializer.h" +#include "opt_rss.h" +#include "opt_emx.h" #include #include @@ -78,6 +80,7 @@ #include #include #include +#include #include #include #include @@ -90,8 +93,11 @@ #include #include #include +#include +#include #include #include +#include #include #include @@ -106,6 +112,16 @@ #include #include +#ifdef EMX_RSS_DEBUG +#define EMX_RSS_DPRINTF(sc, lvl, fmt, ...) \ +do { \ + if (sc->rss_debug >= lvl) \ + if_printf(&sc->arpcom.ac_if, fmt, __VA_ARGS__); \ +} while (0) +#else /* !EMX_RSS_DEBUG */ +#define EMX_RSS_DPRINTF(sc, lvl, fmt, ...) ((void)0) +#endif /* EMX_RSS_DEBUG */ + #define EMX_NAME "Intel(R) PRO/1000 " #define EMX_DEVICE(id) \ @@ -123,6 +139,7 @@ static const struct emx_device { EMX_DEVICE(82571EB_SERDES_DUAL), EMX_DEVICE(82571EB_SERDES_QUAD), EMX_DEVICE(82571EB_QUAD_COPPER), + EMX_DEVICE(82571EB_QUAD_COPPER_BP), EMX_DEVICE(82571EB_QUAD_COPPER_LP), EMX_DEVICE(82571EB_QUAD_FIBER), EMX_DEVICE(82571PT_QUAD_COPPER), @@ -158,13 +175,20 @@ static void emx_init(void *); static void emx_stop(struct emx_softc *); static int emx_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void emx_start(struct ifnet *); -#ifdef DEVICE_POLLING -static void emx_poll(struct ifnet *, enum poll_cmd, int); +#ifdef IFPOLL_ENABLE +static void emx_qpoll(struct ifnet *, struct ifpoll_info *); #endif static void emx_watchdog(struct ifnet *); static void emx_media_status(struct ifnet *, struct ifmediareq *); static int emx_media_change(struct ifnet *); static void emx_timer(void *); +static void emx_serialize(struct ifnet *, enum ifnet_serialize); +static void emx_deserialize(struct ifnet *, enum ifnet_serialize); +static int emx_tryserialize(struct ifnet *, enum ifnet_serialize); +#ifdef INVARIANTS +static void emx_serialize_assert(struct ifnet *, enum ifnet_serialize, + boolean_t); +#endif static void emx_intr(void *); static void emx_rxeof(struct emx_softc *, int, int); @@ -212,6 +236,12 @@ static int emx_sysctl_int_throttle(SYSCTL_HANDLER_ARGS); static int emx_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS); static void emx_add_sysctl(struct emx_softc *); +static void emx_serialize_skipmain(struct emx_softc *); +static void emx_deserialize_skipmain(struct emx_softc *); +#ifdef IFPOLL_ENABLE +static int emx_tryserialize_skipmain(struct emx_softc *); +#endif + /* Management and WOL Support */ static void emx_get_mgmt(struct emx_softc *); static void emx_rel_mgmt(struct emx_softc *); @@ -283,7 +313,7 @@ static __inline void emx_setup_rxdesc(emx_rxdesc_t *rxd, const struct emx_rxbuf *rxbuf) { rxd->rxd_bufaddr = htole64(rxbuf->paddr); - /* DD bits must be cleared */ + /* DD bit must be cleared */ rxd->rxd_staterr = 0; } @@ -307,6 +337,45 @@ emx_rxcsum(uint32_t staterr, struct mbuf *mp) } } +static __inline struct pktinfo * +emx_rssinfo(struct mbuf *m, struct pktinfo *pi, + uint32_t mrq, uint32_t hash, uint32_t staterr) +{ + switch (mrq & EMX_RXDMRQ_RSSTYPE_MASK) { + case EMX_RXDMRQ_IPV4_TCP: + pi->pi_netisr = NETISR_IP; + pi->pi_flags = 0; + pi->pi_l3proto = IPPROTO_TCP; + break; + + case EMX_RXDMRQ_IPV6_TCP: + pi->pi_netisr = NETISR_IPV6; + pi->pi_flags = 0; + pi->pi_l3proto = IPPROTO_TCP; + break; + + case EMX_RXDMRQ_IPV4: + if (staterr & E1000_RXD_STAT_IXSM) + return NULL; + + if ((staterr & + (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) == + E1000_RXD_STAT_TCPCS) { + pi->pi_netisr = NETISR_IP; + pi->pi_flags = 0; + pi->pi_l3proto = IPPROTO_UDP; + break; + } + /* FALL THROUGH */ + default: + return NULL; + } + + m->m_flags |= M_HASH; + m->m_pkthdr.hash = toeplitz_hash(hash); + return pi; +} + static int emx_probe(device_t dev) { @@ -331,9 +400,21 @@ emx_attach(device_t dev) { struct emx_softc *sc = device_get_softc(dev); struct ifnet *ifp = &sc->arpcom.ac_if; - int error = 0; + int error = 0, i; uint16_t eeprom_data, device_id; + lwkt_serialize_init(&sc->main_serialize); + lwkt_serialize_init(&sc->tx_serialize); + for (i = 0; i < EMX_NRX_RING; ++i) + lwkt_serialize_init(&sc->rx_data[i].rx_serialize); + + i = 0; + sc->serializes[i++] = &sc->main_serialize; + sc->serializes[i++] = &sc->tx_serialize; + sc->serializes[i++] = &sc->rx_data[0].rx_serialize; + sc->serializes[i++] = &sc->rx_data[1].rx_serialize; + KKASSERT(i == EMX_NSERIALIZE); + callout_init(&sc->timer); sc->dev = sc->osdep.dev = dev; @@ -437,6 +518,15 @@ emx_attach(device_t dev) /* This controls when hardware reports transmit completion status. */ sc->hw.mac.report_tx_early = 1; +#ifdef RSS + /* Calculate # of RX rings */ + if (ncpus > 1) + sc->rx_ring_cnt = EMX_NRX_RING; + else +#endif + sc->rx_ring_cnt = 1; + sc->rx_ring_inuse = sc->rx_ring_cnt; + /* Allocate RX/TX rings' busdma(9) stuffs */ error = emx_dma_alloc(sc); if (error) @@ -576,7 +666,7 @@ emx_attach(device_t dev) sc->tx_int_nsegs = sc->oact_tx_desc; error = bus_setup_intr(dev, sc->intr_res, INTR_MPSAFE, emx_intr, sc, - &sc->intr_tag, ifp->if_serializer); + &sc->intr_tag, &sc->main_serialize); if (error) { device_printf(dev, "Failed to register interrupt handler"); ether_ifdetach(&sc->arpcom.ac_if); @@ -599,7 +689,7 @@ emx_detach(device_t dev) if (device_is_attached(dev)) { struct ifnet *ifp = &sc->arpcom.ac_if; - lwkt_serialize_enter(ifp->if_serializer); + ifnet_serialize_all(ifp); emx_stop(sc); @@ -619,7 +709,7 @@ emx_detach(device_t dev) bus_teardown_intr(dev, sc->intr_res, sc->intr_tag); - lwkt_serialize_exit(ifp->if_serializer); + ifnet_deserialize_all(ifp); ether_ifdetach(ifp); } @@ -656,7 +746,7 @@ emx_suspend(device_t dev) struct emx_softc *sc = device_get_softc(dev); struct ifnet *ifp = &sc->arpcom.ac_if; - lwkt_serialize_enter(ifp->if_serializer); + ifnet_serialize_all(ifp); emx_stop(sc); @@ -672,7 +762,7 @@ emx_suspend(device_t dev) emx_enable_wol(dev); } - lwkt_serialize_exit(ifp->if_serializer); + ifnet_deserialize_all(ifp); return bus_generic_suspend(dev); } @@ -683,13 +773,13 @@ emx_resume(device_t dev) struct emx_softc *sc = device_get_softc(dev); struct ifnet *ifp = &sc->arpcom.ac_if; - lwkt_serialize_enter(ifp->if_serializer); + ifnet_serialize_all(ifp); emx_init(sc); emx_get_mgmt(sc); if_devstart(ifp); - lwkt_serialize_exit(ifp->if_serializer); + ifnet_deserialize_all(ifp); return bus_generic_resume(dev); } @@ -700,7 +790,7 @@ emx_start(struct ifnet *ifp) struct emx_softc *sc = ifp->if_softc; struct mbuf *m_head; - ASSERT_SERIALIZED(ifp->if_serializer); + ASSERT_SERIALIZED(&sc->tx_serialize); if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING) return; @@ -749,7 +839,7 @@ emx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr) int max_frame_size, mask, reinit; int error = 0; - ASSERT_SERIALIZED(ifp->if_serializer); + ASSERT_IFNET_SERIALIZED_ALL(ifp); switch (command) { case SIOCSIFMTU: @@ -815,8 +905,8 @@ emx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr) if (ifp->if_flags & IFF_RUNNING) { emx_disable_intr(sc); emx_set_multi(sc); -#ifdef DEVICE_POLLING - if (!(ifp->if_flags & IFF_POLLING)) +#ifdef IFPOLL_ENABLE + if (!(ifp->if_flags & IFF_NPOLLING)) #endif emx_enable_intr(sc); } @@ -846,6 +936,10 @@ emx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr) ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; reinit = 1; } + if (mask & IFCAP_RSS) { + ifp->if_capenable ^= IFCAP_RSS; + reinit = 1; + } if (reinit && (ifp->if_flags & IFF_RUNNING)) emx_init(sc); break; @@ -862,7 +956,7 @@ emx_watchdog(struct ifnet *ifp) { struct emx_softc *sc = ifp->if_softc; - ASSERT_SERIALIZED(ifp->if_serializer); + ASSERT_IFNET_SERIALIZED_ALL(ifp); /* * The timer is set to 5 every time start queues a packet. @@ -912,8 +1006,9 @@ emx_init(void *xsc) struct ifnet *ifp = &sc->arpcom.ac_if; device_t dev = sc->dev; uint32_t pba; + int i; - ASSERT_SERIALIZED(ifp->if_serializer); + ASSERT_IFNET_SERIALIZED_ALL(ifp); emx_stop(sc); @@ -1000,11 +1095,22 @@ emx_init(void *xsc) /* Setup Multicast table */ emx_set_multi(sc); + /* + * Adjust # of RX ring to be used based on IFCAP_RSS + */ + if (ifp->if_capenable & IFCAP_RSS) + sc->rx_ring_inuse = sc->rx_ring_cnt; + else + sc->rx_ring_inuse = 1; + /* Prepare receive descriptors and buffers */ - if (emx_init_rx_ring(sc, &sc->rx_data[0])) { - device_printf(dev, "Could not setup receive structures\n"); - emx_stop(sc); - return; + for (i = 0; i < sc->rx_ring_inuse; ++i) { + if (emx_init_rx_ring(sc, &sc->rx_data[i])) { + device_printf(dev, + "Could not setup receive structures\n"); + emx_stop(sc); + return; + } } emx_init_rx_unit(sc); @@ -1034,63 +1140,21 @@ emx_init(void *xsc) E1000_WRITE_REG(&sc->hw, E1000_IVAR, 0x800A0908); } -#ifdef DEVICE_POLLING +#ifdef IFPOLL_ENABLE /* * Only enable interrupts if we are not polling, make sure * they are off otherwise. */ - if (ifp->if_flags & IFF_POLLING) + if (ifp->if_flags & IFF_NPOLLING) emx_disable_intr(sc); else -#endif /* DEVICE_POLLING */ +#endif /* IFPOLL_ENABLE */ emx_enable_intr(sc); /* Don't reset the phy next time init gets called */ sc->hw.phy.reset_disable = TRUE; } -#ifdef DEVICE_POLLING - -static void -emx_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) -{ - struct emx_softc *sc = ifp->if_softc; - uint32_t reg_icr; - - ASSERT_SERIALIZED(ifp->if_serializer); - - switch (cmd) { - case POLL_REGISTER: - emx_disable_intr(sc); - break; - - case POLL_DEREGISTER: - emx_enable_intr(sc); - break; - - case POLL_AND_CHECK_STATUS: - reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR); - if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { - callout_stop(&sc->timer); - sc->hw.mac.get_link_status = 1; - emx_update_link_status(sc); - callout_reset(&sc->timer, hz, emx_timer, sc); - } - /* FALL THROUGH */ - case POLL_ONLY: - if (ifp->if_flags & IFF_RUNNING) { - emx_rxeof(sc, 0, count); - emx_txeof(sc); - - if (!ifq_is_empty(&ifp->if_snd)) - if_devstart(ifp); - } - break; - } -} - -#endif /* DEVICE_POLLING */ - static void emx_intr(void *xsc) { @@ -1099,7 +1163,7 @@ emx_intr(void *xsc) uint32_t reg_icr; logif(intr_beg); - ASSERT_SERIALIZED(ifp->if_serializer); + ASSERT_SERIALIZED(&sc->main_serialize); reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR); @@ -1110,7 +1174,7 @@ emx_intr(void *xsc) /* * XXX: some laptops trigger several spurious interrupts - * on em(4) when in the resume cycle. The ICR register + * on emx(4) when in the resume cycle. The ICR register * reports all-ones value in this case. Processing such * interrupts would lead to a freeze. I don't know why. */ @@ -1121,17 +1185,30 @@ emx_intr(void *xsc) if (ifp->if_flags & IFF_RUNNING) { if (reg_icr & - (E1000_ICR_RXT0 | E1000_ICR_RXDMT0 | E1000_ICR_RXO)) - emx_rxeof(sc, 0, -1); + (E1000_ICR_RXT0 | E1000_ICR_RXDMT0 | E1000_ICR_RXO)) { + int i; + + for (i = 0; i < sc->rx_ring_inuse; ++i) { + lwkt_serialize_enter( + &sc->rx_data[i].rx_serialize); + emx_rxeof(sc, i, -1); + lwkt_serialize_exit( + &sc->rx_data[i].rx_serialize); + } + } if (reg_icr & E1000_ICR_TXDW) { + lwkt_serialize_enter(&sc->tx_serialize); emx_txeof(sc); if (!ifq_is_empty(&ifp->if_snd)) if_devstart(ifp); + lwkt_serialize_exit(&sc->tx_serialize); } } /* Link status change */ if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { + emx_serialize_skipmain(sc); + callout_stop(&sc->timer); sc->hw.mac.get_link_status = 1; emx_update_link_status(sc); @@ -1140,6 +1217,8 @@ emx_intr(void *xsc) emx_tx_purge(sc); callout_reset(&sc->timer, hz, emx_timer, sc); + + emx_deserialize_skipmain(sc); } if (reg_icr & E1000_ICR_RXO) @@ -1153,7 +1232,7 @@ emx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) { struct emx_softc *sc = ifp->if_softc; - ASSERT_SERIALIZED(ifp->if_serializer); + ASSERT_IFNET_SERIALIZED_ALL(ifp); emx_update_link_status(sc); @@ -1194,7 +1273,7 @@ emx_media_change(struct ifnet *ifp) struct emx_softc *sc = ifp->if_softc; struct ifmedia *ifm = &sc->media; - ASSERT_SERIALIZED(ifp->if_serializer); + ASSERT_IFNET_SERIALIZED_ALL(ifp); if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) return (EINVAL); @@ -1452,7 +1531,7 @@ emx_timer(void *xsc) struct emx_softc *sc = xsc; struct ifnet *ifp = &sc->arpcom.ac_if; - lwkt_serialize_enter(ifp->if_serializer); + ifnet_serialize_all(ifp); emx_update_link_status(sc); emx_update_stats(sc); @@ -1468,7 +1547,7 @@ emx_timer(void *xsc) callout_reset(&sc->timer, hz, emx_timer, sc); - lwkt_serialize_exit(ifp->if_serializer); + ifnet_deserialize_all(ifp); } static void @@ -1560,7 +1639,7 @@ emx_stop(struct emx_softc *sc) struct ifnet *ifp = &sc->arpcom.ac_if; int i; - ASSERT_SERIALIZED(ifp->if_serializer); + ASSERT_IFNET_SERIALIZED_ALL(ifp); emx_disable_intr(sc); @@ -1569,6 +1648,15 @@ emx_stop(struct emx_softc *sc) ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); ifp->if_timer = 0; + /* + * Disable multiple receive queues. + * + * NOTE: + * We should disable multiple receive queues before + * resetting the hardware. + */ + E1000_WRITE_REG(&sc->hw, E1000_MRQC, 0); + e1000_reset_hw(&sc->hw); E1000_WRITE_REG(&sc->hw, E1000_WUC, 0); @@ -1582,7 +1670,8 @@ emx_stop(struct emx_softc *sc) } } - emx_free_rx_ring(sc, &sc->rx_data[0]); + for (i = 0; i < sc->rx_ring_inuse; ++i) + emx_free_rx_ring(sc, &sc->rx_data[i]); sc->csum_flags = 0; sc->csum_ehlen = 0; @@ -1670,10 +1759,16 @@ emx_setup_ifp(struct emx_softc *sc) ifp->if_init = emx_init; ifp->if_ioctl = emx_ioctl; ifp->if_start = emx_start; -#ifdef DEVICE_POLLING - ifp->if_poll = emx_poll; +#ifdef IFPOLL_ENABLE + ifp->if_qpoll = emx_qpoll; #endif ifp->if_watchdog = emx_watchdog; + ifp->if_serialize = emx_serialize; + ifp->if_deserialize = emx_deserialize; + ifp->if_tryserialize = emx_tryserialize; +#ifdef INVARIANTS + ifp->if_serialize_assert = emx_serialize_assert; +#endif ifq_set_maxlen(&ifp->if_snd, sc->num_tx_desc - 1); ifq_set_ready(&ifp->if_snd); @@ -1682,6 +1777,8 @@ emx_setup_ifp(struct emx_softc *sc) ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; + if (sc->rx_ring_cnt > 1) + ifp->if_capabilities |= IFCAP_RSS; ifp->if_capenable = ifp->if_capabilities; ifp->if_hwassist = EMX_CSUM_FEATURES; @@ -2380,7 +2477,7 @@ emx_create_rx_ring(struct emx_softc *sc, struct emx_rxdata *rdata) * Validate number of receive descriptors. It must not exceed * hardware maximum, and must be multiple of E1000_DBA_ALIGN. */ - if ((emx_rxd * sizeof(struct e1000_rx_desc)) % EMX_DBA_ALIGN != 0 || + if ((emx_rxd * sizeof(emx_rxdesc_t)) % EMX_DBA_ALIGN != 0 || emx_rxd > EMX_MAX_RXD || emx_rxd < EMX_MIN_RXD) { device_printf(dev, "Using %d RX descriptors instead of %d!\n", EMX_DEFAULT_RXD, emx_rxd); @@ -2502,9 +2599,9 @@ static void emx_init_rx_unit(struct emx_softc *sc) { struct ifnet *ifp = &sc->arpcom.ac_if; - struct emx_rxdata *rdata = &sc->rx_data[0]; uint64_t bus_addr; uint32_t rctl, rxcsum, rfctl; + int i; /* * Make sure receives are disabled while setting @@ -2534,11 +2631,17 @@ emx_init_rx_unit(struct emx_softc *sc) E1000_WRITE_REG(&sc->hw, E1000_RFCTL, rfctl); /* Setup the Base and Length of the Rx Descriptor Ring */ - bus_addr = rdata->rx_desc_paddr; - E1000_WRITE_REG(&sc->hw, E1000_RDLEN(0), - rdata->num_rx_desc * sizeof(struct e1000_rx_desc)); - E1000_WRITE_REG(&sc->hw, E1000_RDBAH(0), (uint32_t)(bus_addr >> 32)); - E1000_WRITE_REG(&sc->hw, E1000_RDBAL(0), (uint32_t)bus_addr); + for (i = 0; i < sc->rx_ring_inuse; ++i) { + struct emx_rxdata *rdata = &sc->rx_data[i]; + + bus_addr = rdata->rx_desc_paddr; + E1000_WRITE_REG(&sc->hw, E1000_RDLEN(i), + rdata->num_rx_desc * sizeof(emx_rxdesc_t)); + E1000_WRITE_REG(&sc->hw, E1000_RDBAH(i), + (uint32_t)(bus_addr >> 32)); + E1000_WRITE_REG(&sc->hw, E1000_RDBAL(i), + (uint32_t)bus_addr); + } /* Setup the Receive Control Register */ rctl &= ~(3 << E1000_RCTL_MO_SHIFT); @@ -2560,13 +2663,84 @@ emx_init_rx_unit(struct emx_softc *sc) else rctl &= ~E1000_RCTL_LPE; - /* Receive Checksum Offload for TCP and UDP */ - if (ifp->if_capenable & IFCAP_RXCSUM) { + /* + * Receive Checksum Offload for TCP and UDP + * + * Checksum offloading is also enabled if multiple receive + * queue is to be supported, since we need it to figure out + * packet type. + */ + if (ifp->if_capenable & (IFCAP_RSS | IFCAP_RXCSUM)) { rxcsum = E1000_READ_REG(&sc->hw, E1000_RXCSUM); - rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); + + /* + * NOTE: + * PCSD must be enabled to enable multiple + * receive queues. + */ + rxcsum |= E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL | + E1000_RXCSUM_PCSD; E1000_WRITE_REG(&sc->hw, E1000_RXCSUM, rxcsum); } + /* + * Configure multiple receive queue (RSS) + */ + if (ifp->if_capenable & IFCAP_RSS) { + uint8_t key[EMX_NRSSRK * EMX_RSSRK_SIZE]; + uint32_t reta; + + KASSERT(sc->rx_ring_inuse == EMX_NRX_RING, + ("invalid number of RX ring (%d)", + sc->rx_ring_inuse)); + + /* + * NOTE: + * When we reach here, RSS has already been disabled + * in emx_stop(), so we could safely configure RSS key + * and redirect table. + */ + + /* + * Configure RSS key + */ + toeplitz_get_key(key, sizeof(key)); + for (i = 0; i < EMX_NRSSRK; ++i) { + uint32_t rssrk; + + rssrk = EMX_RSSRK_VAL(key, i); + EMX_RSS_DPRINTF(sc, 1, "rssrk%d 0x%08x\n", i, rssrk); + + E1000_WRITE_REG(&sc->hw, E1000_RSSRK(i), rssrk); + } + + /* + * Configure RSS redirect table in following fashion: + * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] + */ + reta = 0; + for (i = 0; i < EMX_RETA_SIZE; ++i) { + uint32_t q; + + q = (i % sc->rx_ring_inuse) << EMX_RETA_RINGIDX_SHIFT; + reta |= q << (8 * i); + } + EMX_RSS_DPRINTF(sc, 1, "reta 0x%08x\n", reta); + + for (i = 0; i < EMX_NRETA; ++i) + E1000_WRITE_REG(&sc->hw, E1000_RETA(i), reta); + + /* + * Enable multiple receive queues. + * Enable IPv4 RSS standard hash functions. + * Disable RSS interrupt. + */ + E1000_WRITE_REG(&sc->hw, E1000_MRQC, + E1000_MRQC_ENABLE_RSS_2Q | + E1000_MRQC_RSS_FIELD_IPV4_TCP | + E1000_MRQC_RSS_FIELD_IPV4); + } + /* * XXX TEMPORARY WORKAROUND: on some systems with 82573 * long latencies are observed, like Lenovo X60. This @@ -2579,14 +2753,17 @@ emx_init_rx_unit(struct emx_softc *sc) E1000_WRITE_REG(&sc->hw, E1000_RDTR, EMX_RDTR_82573); } - /* Enable Receives */ - E1000_WRITE_REG(&sc->hw, E1000_RCTL, rctl); - /* * Setup the HW Rx Head and Tail Descriptor Pointers */ - E1000_WRITE_REG(&sc->hw, E1000_RDH(0), 0); - E1000_WRITE_REG(&sc->hw, E1000_RDT(0), sc->rx_data[0].num_rx_desc - 1); + for (i = 0; i < sc->rx_ring_inuse; ++i) { + E1000_WRITE_REG(&sc->hw, E1000_RDH(i), 0); + E1000_WRITE_REG(&sc->hw, E1000_RDT(i), + sc->rx_data[i].num_rx_desc - 1); + } + + /* Enable Receives */ + E1000_WRITE_REG(&sc->hw, E1000_RCTL, rctl); } static void @@ -2642,6 +2819,7 @@ emx_rxeof(struct emx_softc *sc, int ring_idx, int count) ether_input_chain_init(chain); while ((staterr & E1000_RXD_STAT_DD) && count != 0) { + struct pktinfo *pi = NULL, pi0; struct emx_rxbuf *rx_buf = &rdata->rx_buf[i]; struct mbuf *m = NULL; int eop, len; @@ -2667,6 +2845,7 @@ emx_rxeof(struct emx_softc *sc, int ring_idx, int count) if (!(staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK)) { uint16_t vlan = 0; + uint32_t mrq, rss_hash; /* * Save several necessary information, @@ -2675,6 +2854,13 @@ emx_rxeof(struct emx_softc *sc, int ring_idx, int count) if ((staterr & E1000_RXD_STAT_VP) && eop) vlan = le16toh(current_desc->rxd_vlan); + mrq = le32toh(current_desc->rxd_mrq); + rss_hash = le32toh(current_desc->rxd_rss); + + EMX_RSS_DPRINTF(sc, 10, + "ring%d, mrq 0x%08x, rss_hash 0x%08x\n", + ring_idx, mrq, rss_hash); + if (emx_newbuf(sc, rdata, i, 0) != 0) { ifp->if_iqdrops++; goto discard; @@ -2711,6 +2897,14 @@ emx_rxeof(struct emx_softc *sc, int ring_idx, int count) m = rdata->fmp; rdata->fmp = NULL; rdata->lmp = NULL; + + if (ifp->if_capenable & IFCAP_RSS) { + pi = emx_rssinfo(m, &pi0, mrq, + rss_hash, staterr); + } +#ifdef EMX_RSS_DEBUG + rdata->rx_pkts++; +#endif } } else { ifp->if_ierrors++; @@ -2725,7 +2919,7 @@ discard: } if (m != NULL) - ether_input_chain(ifp, m, chain); + ether_input_chain(ifp, m, pi, chain); /* Advance our pointers to the next descriptor. */ if (++i == rdata->num_rx_desc) @@ -2738,16 +2932,16 @@ discard: ether_input_dispatch(chain); - /* Advance the E1000's Receive Queue #0 "Tail Pointer". */ + /* Advance the E1000's Receive Queue "Tail Pointer". */ if (--i < 0) i = rdata->num_rx_desc - 1; - E1000_WRITE_REG(&sc->hw, E1000_RDT(0), i); + E1000_WRITE_REG(&sc->hw, E1000_RDT(ring_idx), i); } static void emx_enable_intr(struct emx_softc *sc) { - lwkt_serialize_handler_enable(sc->arpcom.ac_if.if_serializer); + lwkt_serialize_handler_enable(&sc->main_serialize); E1000_WRITE_REG(&sc->hw, E1000_IMS, IMS_ENABLE_MASK); } @@ -2755,7 +2949,7 @@ static void emx_disable_intr(struct emx_softc *sc) { E1000_WRITE_REG(&sc->hw, E1000_IMC, 0xffffffff); - lwkt_serialize_handler_disable(sc->arpcom.ac_if.if_serializer); + lwkt_serialize_handler_disable(&sc->main_serialize); } /* @@ -3128,7 +3322,7 @@ emx_sysctl_debug_info(SYSCTL_HANDLER_ARGS) sc = (struct emx_softc *)arg1; ifp = &sc->arpcom.ac_if; - lwkt_serialize_enter(ifp->if_serializer); + ifnet_serialize_all(ifp); if (result == 1) emx_print_debug_info(sc); @@ -3141,7 +3335,7 @@ emx_sysctl_debug_info(SYSCTL_HANDLER_ARGS) if (result == 2) emx_print_nvm_info(sc); - lwkt_serialize_exit(ifp->if_serializer); + ifnet_deserialize_all(ifp); return (error); } @@ -3160,9 +3354,9 @@ emx_sysctl_stats(SYSCTL_HANDLER_ARGS) struct emx_softc *sc = (struct emx_softc *)arg1; struct ifnet *ifp = &sc->arpcom.ac_if; - lwkt_serialize_enter(ifp->if_serializer); + ifnet_serialize_all(ifp); emx_print_hw_stats(sc); - lwkt_serialize_exit(ifp->if_serializer); + ifnet_deserialize_all(ifp); } return (error); } @@ -3173,6 +3367,10 @@ emx_add_sysctl(struct emx_softc *sc) #ifdef PROFILE_SERIALIZER struct ifnet *ifp = &sc->arpcom.ac_if; #endif +#ifdef EMX_RSS_DEBUG + char rx_pkt[32]; + int i; +#endif sysctl_ctx_init(&sc->sysctl_ctx); sc->sysctl_tree = SYSCTL_ADD_NODE(&sc->sysctl_ctx, @@ -3198,6 +3396,7 @@ emx_add_sysctl(struct emx_softc *sc) SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), OID_AUTO, "txd", CTLFLAG_RD, &sc->num_tx_desc, 0, NULL); +#ifdef notyet #ifdef PROFILE_SERIALIZER SYSCTL_ADD_UINT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), OID_AUTO, "serializer_sleep", CTLFLAG_RW, @@ -3211,6 +3410,7 @@ emx_add_sysctl(struct emx_softc *sc) SYSCTL_ADD_UINT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), OID_AUTO, "serializer_try", CTLFLAG_RW, &ifp->if_serializer->try_cnt, 0, NULL); +#endif #endif SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), @@ -3221,6 +3421,23 @@ emx_add_sysctl(struct emx_softc *sc) OID_AUTO, "int_tx_nsegs", CTLTYPE_INT|CTLFLAG_RW, sc, 0, emx_sysctl_int_tx_nsegs, "I", "# segments per TX interrupt"); + + SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), + OID_AUTO, "rx_ring_inuse", CTLFLAG_RD, + &sc->rx_ring_inuse, 0, "RX ring in use"); + +#ifdef EMX_RSS_DEBUG + SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), + OID_AUTO, "rss_debug", CTLFLAG_RW, &sc->rss_debug, + 0, "RSS debug level"); + for (i = 0; i < sc->rx_ring_cnt; ++i) { + ksnprintf(rx_pkt, sizeof(rx_pkt), "rx%d_pkt", i); + SYSCTL_ADD_UINT(&sc->sysctl_ctx, + SYSCTL_CHILDREN(sc->sysctl_tree), OID_AUTO, + rx_pkt, CTLFLAG_RW, + &sc->rx_data[i].rx_pkts, 0, "RXed packets"); + } +#endif } static int @@ -3249,7 +3466,7 @@ emx_sysctl_int_throttle(SYSCTL_HANDLER_ARGS) return EINVAL; } - lwkt_serialize_enter(ifp->if_serializer); + ifnet_serialize_all(ifp); if (throttle) sc->int_throttle_ceil = 1000000000 / 256 / throttle; @@ -3259,7 +3476,7 @@ emx_sysctl_int_throttle(SYSCTL_HANDLER_ARGS) if (ifp->if_flags & IFF_RUNNING) E1000_WRITE_REG(&sc->hw, E1000_ITR, throttle); - lwkt_serialize_exit(ifp->if_serializer); + ifnet_deserialize_all(ifp); if (bootverbose) { if_printf(ifp, "Interrupt moderation set to %d/sec\n", @@ -3282,7 +3499,7 @@ emx_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS) if (segs <= 0) return EINVAL; - lwkt_serialize_enter(ifp->if_serializer); + ifnet_serialize_all(ifp); /* * Don't allow int_tx_nsegs to become: @@ -3300,7 +3517,7 @@ emx_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS) sc->tx_int_nsegs = segs; } - lwkt_serialize_exit(ifp->if_serializer); + ifnet_deserialize_all(ifp); return error; } @@ -3308,7 +3525,7 @@ emx_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS) static int emx_dma_alloc(struct emx_softc *sc) { - int error; + int error, i; /* * Create top level busdma tag @@ -3335,10 +3552,13 @@ emx_dma_alloc(struct emx_softc *sc) /* * Allocate receive descriptors ring and buffers */ - error = emx_create_rx_ring(sc, &sc->rx_data[0]); - if (error) { - device_printf(sc->dev, "Could not setup receive structures\n"); - return error; + for (i = 0; i < sc->rx_ring_cnt; ++i) { + error = emx_create_rx_ring(sc, &sc->rx_data[i]); + if (error) { + device_printf(sc->dev, + "Could not setup receive structures\n"); + return error; + } } return 0; } @@ -3346,10 +3566,258 @@ emx_dma_alloc(struct emx_softc *sc) static void emx_dma_free(struct emx_softc *sc) { + int i; + emx_destroy_tx_ring(sc, sc->num_tx_desc); - emx_destroy_rx_ring(sc, &sc->rx_data[0], sc->rx_data[0].num_rx_desc); + + for (i = 0; i < sc->rx_ring_cnt; ++i) { + emx_destroy_rx_ring(sc, &sc->rx_data[i], + sc->rx_data[i].num_rx_desc); + } /* Free top level busdma tag */ if (sc->parent_dtag != NULL) bus_dma_tag_destroy(sc->parent_dtag); } + +static void +emx_serialize(struct ifnet *ifp, enum ifnet_serialize slz) +{ + struct emx_softc *sc = ifp->if_softc; + + switch (slz) { + case IFNET_SERIALIZE_ALL: + lwkt_serialize_array_enter(sc->serializes, EMX_NSERIALIZE, 0); + break; + + case IFNET_SERIALIZE_MAIN: + lwkt_serialize_enter(&sc->main_serialize); + break; + + case IFNET_SERIALIZE_TX: + lwkt_serialize_enter(&sc->tx_serialize); + break; + + case IFNET_SERIALIZE_RX(0): + lwkt_serialize_enter(&sc->rx_data[0].rx_serialize); + break; + + case IFNET_SERIALIZE_RX(1): + lwkt_serialize_enter(&sc->rx_data[1].rx_serialize); + break; + + default: + panic("%s unsupported serialize type\n", ifp->if_xname); + } +} + +static void +emx_deserialize(struct ifnet *ifp, enum ifnet_serialize slz) +{ + struct emx_softc *sc = ifp->if_softc; + + switch (slz) { + case IFNET_SERIALIZE_ALL: + lwkt_serialize_array_exit(sc->serializes, EMX_NSERIALIZE, 0); + break; + + case IFNET_SERIALIZE_MAIN: + lwkt_serialize_exit(&sc->main_serialize); + break; + + case IFNET_SERIALIZE_TX: + lwkt_serialize_exit(&sc->tx_serialize); + break; + + case IFNET_SERIALIZE_RX(0): + lwkt_serialize_exit(&sc->rx_data[0].rx_serialize); + break; + + case IFNET_SERIALIZE_RX(1): + lwkt_serialize_exit(&sc->rx_data[1].rx_serialize); + break; + + default: + panic("%s unsupported serialize type\n", ifp->if_xname); + } +} + +static int +emx_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz) +{ + struct emx_softc *sc = ifp->if_softc; + + switch (slz) { + case IFNET_SERIALIZE_ALL: + return lwkt_serialize_array_try(sc->serializes, + EMX_NSERIALIZE, 0); + + case IFNET_SERIALIZE_MAIN: + return lwkt_serialize_try(&sc->main_serialize); + + case IFNET_SERIALIZE_TX: + return lwkt_serialize_try(&sc->tx_serialize); + + case IFNET_SERIALIZE_RX(0): + return lwkt_serialize_try(&sc->rx_data[0].rx_serialize); + + case IFNET_SERIALIZE_RX(1): + return lwkt_serialize_try(&sc->rx_data[1].rx_serialize); + + default: + panic("%s unsupported serialize type\n", ifp->if_xname); + } +} + +static void +emx_serialize_skipmain(struct emx_softc *sc) +{ + lwkt_serialize_array_enter(sc->serializes, EMX_NSERIALIZE, 1); +} + +#ifdef IFPOLL_ENABLE +static int +emx_tryserialize_skipmain(struct emx_softc *sc) +{ + return lwkt_serialize_array_try(sc->serializes, EMX_NSERIALIZE, 1); +} +#endif + +static void +emx_deserialize_skipmain(struct emx_softc *sc) +{ + lwkt_serialize_array_exit(sc->serializes, EMX_NSERIALIZE, 1); +} + +#ifdef INVARIANTS + +static void +emx_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz, + boolean_t serialized) +{ + struct emx_softc *sc = ifp->if_softc; + int i; + + switch (slz) { + case IFNET_SERIALIZE_ALL: + if (serialized) { + for (i = 0; i < EMX_NSERIALIZE; ++i) + ASSERT_SERIALIZED(sc->serializes[i]); + } else { + for (i = 0; i < EMX_NSERIALIZE; ++i) + ASSERT_NOT_SERIALIZED(sc->serializes[i]); + } + break; + + case IFNET_SERIALIZE_MAIN: + if (serialized) + ASSERT_SERIALIZED(&sc->main_serialize); + else + ASSERT_NOT_SERIALIZED(&sc->main_serialize); + break; + + case IFNET_SERIALIZE_TX: + if (serialized) + ASSERT_SERIALIZED(&sc->tx_serialize); + else + ASSERT_NOT_SERIALIZED(&sc->tx_serialize); + break; + + case IFNET_SERIALIZE_RX(0): + if (serialized) + ASSERT_SERIALIZED(&sc->rx_data[0].rx_serialize); + else + ASSERT_NOT_SERIALIZED(&sc->rx_data[0].rx_serialize); + break; + + case IFNET_SERIALIZE_RX(1): + if (serialized) + ASSERT_SERIALIZED(&sc->rx_data[1].rx_serialize); + else + ASSERT_NOT_SERIALIZED(&sc->rx_data[1].rx_serialize); + break; + + default: + panic("%s unsupported serialize type\n", ifp->if_xname); + } +} + +#endif /* INVARIANTS */ + +#ifdef IFPOLL_ENABLE + +static void +emx_qpoll_status(struct ifnet *ifp, int pollhz __unused) +{ + struct emx_softc *sc = ifp->if_softc; + uint32_t reg_icr; + + ASSERT_SERIALIZED(&sc->main_serialize); + + reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR); + if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { + if (emx_tryserialize_skipmain(sc)) { + callout_stop(&sc->timer); + sc->hw.mac.get_link_status = 1; + emx_update_link_status(sc); + callout_reset(&sc->timer, hz, emx_timer, sc); + emx_deserialize_skipmain(sc); + } + } +} + +static void +emx_qpoll_tx(struct ifnet *ifp, void *arg __unused, int cycle __unused) +{ + struct emx_softc *sc = ifp->if_softc; + + ASSERT_SERIALIZED(&sc->tx_serialize); + + emx_txeof(sc); + if (!ifq_is_empty(&ifp->if_snd)) + if_devstart(ifp); +} + +static void +emx_qpoll_rx(struct ifnet *ifp, void *arg, int cycle) +{ + struct emx_softc *sc = ifp->if_softc; + struct emx_rxdata *rdata = arg; + + ASSERT_SERIALIZED(&rdata->rx_serialize); + + emx_rxeof(sc, rdata - sc->rx_data, cycle); +} + +static void +emx_qpoll(struct ifnet *ifp, struct ifpoll_info *info) +{ + struct emx_softc *sc = ifp->if_softc; + + ASSERT_IFNET_SERIALIZED_ALL(ifp); + + if (info) { + int i; + + info->ifpi_status.status_func = emx_qpoll_status; + info->ifpi_status.serializer = &sc->main_serialize; + + info->ifpi_tx[0].poll_func = emx_qpoll_tx; + info->ifpi_tx[0].arg = NULL; + info->ifpi_tx[0].serializer = &sc->tx_serialize; + + for (i = 0; i < sc->rx_ring_cnt; ++i) { + info->ifpi_rx[i].poll_func = emx_qpoll_rx; + info->ifpi_rx[i].arg = &sc->rx_data[i]; + info->ifpi_rx[i].serializer = + &sc->rx_data[i].rx_serialize; + } + + if (ifp->if_flags & IFF_RUNNING) + emx_disable_intr(sc); + } else if (ifp->if_flags & IFF_RUNNING) { + emx_enable_intr(sc); + } +} + +#endif /* IFPOLL_ENABLE */