From 7d235eb54de15330b69dcdeca5c99604aeea12d1 Mon Sep 17 00:00:00 2001 From: Sepherosa Ziehau Date: Tue, 12 Jun 2012 19:03:18 +0800 Subject: [PATCH] igb: Split RX and TX serializer --- sys/dev/netif/igb/if_igb.c | 164 +++++++++++++++++++++++++++++++++---- sys/dev/netif/igb/if_igb.h | 11 +++ 2 files changed, 159 insertions(+), 16 deletions(-) diff --git a/sys/dev/netif/igb/if_igb.c b/sys/dev/netif/igb/if_igb.c index 2f9c575c74..2bf5d168de 100644 --- a/sys/dev/netif/igb/if_igb.c +++ b/sys/dev/netif/igb/if_igb.c @@ -167,6 +167,13 @@ static void igb_start(struct ifnet *); #ifdef DEVICE_POLLING static void igb_poll(struct ifnet *, enum poll_cmd, int); #endif +static void igb_serialize(struct ifnet *, enum ifnet_serialize); +static void igb_deserialize(struct ifnet *, enum ifnet_serialize); +static int igb_tryserialize(struct ifnet *, enum ifnet_serialize); +#ifdef INVARIANTS +static void igb_serialize_assert(struct ifnet *, enum ifnet_serialize, + boolean_t); +#endif static void igb_intr(void *); static void igb_shared_intr(void *); @@ -188,6 +195,9 @@ static void igb_get_hw_control(struct igb_softc *); static void igb_rel_hw_control(struct igb_softc *); static void igb_enable_wol(device_t); +static void igb_serialize_skipmain(struct igb_softc *); +static void igb_deserialize_skipmain(struct igb_softc *); + static device_method_t igb_methods[] = { /* Device interface */ DEVMETHOD(device_probe, igb_probe), @@ -278,7 +288,7 @@ igb_attach(device_t dev) struct igb_softc *sc = device_get_softc(dev); uint16_t eeprom_data; u_int intr_flags; - int error = 0; + int error = 0, i, j; #ifdef notyet /* SYSCTL stuff */ @@ -391,6 +401,24 @@ igb_attach(device_t dev) if (error) goto failed; + /* + * Setup serializers + */ + lwkt_serialize_init(&sc->main_serialize); + i = 0; + sc->serializes[i++] = &sc->main_serialize; + + sc->tx_serialize = i; + for (j = 0; j < sc->tx_ring_cnt; ++j) + sc->serializes[i++] = &sc->tx_rings[j].tx_serialize; + + sc->rx_serialize = i; + for (j = 0; j < sc->rx_ring_cnt; ++j) + sc->serializes[i++] = &sc->rx_rings[j].rx_serialize; + + sc->serialize_cnt = i; + KKASSERT(sc->serialize_cnt <= IGB_NSERIALIZE); + /* Allocate the appropriate stats memory */ if (sc->vf_ifp) { sc->stats = kmalloc(sizeof(struct e1000_vf_stats), M_DEVBUF, @@ -1278,6 +1306,12 @@ igb_setup_ifp(struct igb_softc *sc) ifp->if_init = igb_init; ifp->if_ioctl = igb_ioctl; ifp->if_start = igb_start; + ifp->if_serialize = igb_serialize; + ifp->if_deserialize = igb_deserialize; + ifp->if_tryserialize = igb_tryserialize; +#ifdef INVARIANTS + ifp->if_serialize_assert = igb_serialize_assert; +#endif #ifdef DEVICE_POLLING ifp->if_poll = igb_poll; #endif @@ -1381,6 +1415,7 @@ igb_alloc_rings(struct igb_softc *sc) /* Set up some basics */ txr->sc = sc; txr->me = i; + lwkt_serialize_init(&txr->tx_serialize); error = igb_create_tx_ring(txr); if (error) @@ -1398,6 +1433,7 @@ igb_alloc_rings(struct igb_softc *sc) /* Set up some basics */ rxr->sc = sc; rxr->me = i; + lwkt_serialize_init(&rxr->rx_serialize); error = igb_create_rx_ring(rxr); if (error) @@ -2376,7 +2412,7 @@ igb_set_vlan(struct igb_softc *sc) static void igb_enable_intr(struct igb_softc *sc) { - lwkt_serialize_handler_enable(sc->arpcom.ac_if.if_serializer); + lwkt_serialize_handler_enable(&sc->main_serialize); if ((sc->flags & IGB_FLAG_SHARED_INTR) == 0) { /* XXX MSI-X should use sc->intr_mask */ @@ -2400,7 +2436,7 @@ igb_disable_intr(struct igb_softc *sc) E1000_WRITE_REG(&sc->hw, E1000_IMC, 0xffffffff); E1000_WRITE_FLUSH(&sc->hw); - lwkt_serialize_handler_disable(sc->arpcom.ac_if.if_serializer); + lwkt_serialize_handler_disable(&sc->main_serialize); } /* @@ -2702,7 +2738,7 @@ igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) struct igb_softc *sc = ifp->if_softc; uint32_t reg_icr; - ASSERT_SERIALIZED(ifp->if_serializer); + ASSERT_SERIALIZED(&sc->main_serialize); switch (cmd) { case POLL_REGISTER: @@ -2716,17 +2752,31 @@ igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) case POLL_AND_CHECK_STATUS: reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR); if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { + igb_serialize_skipmain(sc); sc->hw.mac.get_link_status = 1; igb_update_link_status(sc); + igb_deserialize_skipmain(sc); } /* FALL THROUGH */ case POLL_ONLY: if (ifp->if_flags & IFF_RUNNING) { - igb_rxeof(&sc->rx_rings[0], count); + struct igb_tx_ring *txr; + int i; + + for (i = 0; i < sc->rx_ring_cnt; ++i) { + struct igb_rx_ring *rxr = &sc->rx_rings[i]; + + lwkt_serialize_enter(&rxr->rx_serialize); + igb_rxeof(rxr, count); + lwkt_serialize_exit(&rxr->rx_serialize); + } - igb_txeof(&sc->tx_rings[0]); + txr = &sc->tx_rings[0]; + lwkt_serialize_enter(&txr->tx_serialize); + igb_txeof(txr); if (!ifq_is_empty(&ifp->if_snd)) if_devstart(ifp); + lwkt_serialize_exit(&txr->tx_serialize); } break; } @@ -2741,7 +2791,7 @@ igb_intr(void *xsc) struct ifnet *ifp = &sc->arpcom.ac_if; uint32_t eicr; - ASSERT_IFNET_SERIALIZED_ALL(ifp); + ASSERT_SERIALIZED(&sc->main_serialize); eicr = E1000_READ_REG(&sc->hw, E1000_EICR); @@ -2749,13 +2799,26 @@ igb_intr(void *xsc) return; if (ifp->if_flags & IFF_RUNNING) { - if (eicr & sc->rx_rings[0].rx_intr_mask) - igb_rxeof(&sc->rx_rings[0], -1); + struct igb_tx_ring *txr; + int i; - if (eicr & sc->tx_rings[0].tx_intr_mask) { - igb_txeof(&sc->tx_rings[0]); + for (i = 0; i < sc->rx_ring_cnt; ++i) { + struct igb_rx_ring *rxr = &sc->rx_rings[i]; + + if (eicr & rxr->rx_intr_mask) { + lwkt_serialize_enter(&rxr->rx_serialize); + igb_rxeof(rxr, -1); + lwkt_serialize_exit(&rxr->rx_serialize); + } + } + + txr = &sc->tx_rings[0]; + if (eicr & txr->tx_intr_mask) { + lwkt_serialize_enter(&txr->tx_serialize); + igb_txeof(txr); if (!ifq_is_empty(&ifp->if_snd)) if_devstart(ifp); + lwkt_serialize_exit(&txr->tx_serialize); } } @@ -2764,8 +2827,10 @@ igb_intr(void *xsc) /* Link status change */ if (icr & E1000_ICR_LSC) { + igb_serialize_skipmain(sc); sc->hw.mac.get_link_status = 1; igb_update_link_status(sc); + igb_deserialize_skipmain(sc); } } @@ -2783,7 +2848,7 @@ igb_shared_intr(void *xsc) struct ifnet *ifp = &sc->arpcom.ac_if; uint32_t reg_icr; - ASSERT_IFNET_SERIALIZED_ALL(ifp); + ASSERT_SERIALIZED(&sc->main_serialize); reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR); @@ -2799,17 +2864,31 @@ igb_shared_intr(void *xsc) return; if (ifp->if_flags & IFF_RUNNING) { - igb_rxeof(&sc->rx_rings[0], -1); + struct igb_tx_ring *txr; + int i; + + for (i = 0; i < sc->rx_ring_cnt; ++i) { + struct igb_rx_ring *rxr = &sc->rx_rings[i]; - igb_txeof(&sc->tx_rings[0]); + lwkt_serialize_enter(&rxr->rx_serialize); + igb_rxeof(rxr, -1); + lwkt_serialize_exit(&rxr->rx_serialize); + } + + txr = &sc->tx_rings[0]; + lwkt_serialize_enter(&txr->tx_serialize); + igb_txeof(txr); if (!ifq_is_empty(&ifp->if_snd)) if_devstart(ifp); + lwkt_serialize_exit(&txr->tx_serialize); } /* Link status change */ if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { + igb_serialize_skipmain(sc); sc->hw.mac.get_link_status = 1; igb_update_link_status(sc); + igb_deserialize_skipmain(sc); } if (reg_icr & E1000_ICR_RXO) @@ -3032,7 +3111,7 @@ igb_start(struct ifnet *ifp) struct igb_tx_ring *txr = &sc->tx_rings[0]; struct mbuf *m_head; - ASSERT_IFNET_SERIALIZED_ALL(ifp); + ASSERT_SERIALIZED(&txr->tx_serialize); if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING) return; @@ -3363,7 +3442,7 @@ igb_setup_intr(struct igb_softc *sc) error = bus_setup_intr(sc->dev, sc->intr_res, INTR_MPSAFE, (sc->flags & IGB_FLAG_SHARED_INTR) ? igb_shared_intr : igb_intr, - sc, &sc->intr_tag, ifp->if_serializer); + sc, &sc->intr_tag, &sc->main_serialize); if (error) { device_printf(sc->dev, "Failed to register interrupt handler"); return error; @@ -3428,3 +3507,56 @@ igb_setup_rx_intr(struct igb_rx_ring *rxr) rxr->rx_intr_mask = 1 << rxr->rx_intr_bit; } } + +static void +igb_serialize(struct ifnet *ifp, enum ifnet_serialize slz) +{ + struct igb_softc *sc = ifp->if_softc; + + ifnet_serialize_array_enter(sc->serializes, sc->serialize_cnt, + sc->tx_serialize, sc->rx_serialize, slz); +} + +static void +igb_deserialize(struct ifnet *ifp, enum ifnet_serialize slz) +{ + struct igb_softc *sc = ifp->if_softc; + + ifnet_serialize_array_exit(sc->serializes, sc->serialize_cnt, + sc->tx_serialize, sc->rx_serialize, slz); +} + +static int +igb_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz) +{ + struct igb_softc *sc = ifp->if_softc; + + return ifnet_serialize_array_try(sc->serializes, sc->serialize_cnt, + sc->tx_serialize, sc->rx_serialize, slz); +} + +static void +igb_serialize_skipmain(struct igb_softc *sc) +{ + lwkt_serialize_array_enter(sc->serializes, sc->serialize_cnt, 1); +} + +static void +igb_deserialize_skipmain(struct igb_softc *sc) +{ + lwkt_serialize_array_exit(sc->serializes, sc->serialize_cnt, 1); +} + +#ifdef INVARIANTS + +static void +igb_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz, + boolean_t serialized) +{ + struct igb_softc *sc = ifp->if_softc; + + ifnet_serialize_array_assert(sc->serializes, sc->serialize_cnt, + sc->tx_serialize, sc->rx_serialize, slz, serialized); +} + +#endif /* INVARIANTS */ diff --git a/sys/dev/netif/igb/if_igb.h b/sys/dev/netif/igb/if_igb.h index 4afd114b15..27ba9ca21d 100644 --- a/sys/dev/netif/igb/if_igb.h +++ b/sys/dev/netif/igb/if_igb.h @@ -150,6 +150,9 @@ #define IGB_TX_OACTIVE_MAX 64 +/* main + 16x RX + 16x TX */ +#define IGB_NSERIALIZE 33 + struct igb_softc; /* @@ -166,6 +169,7 @@ struct igb_dma { * Transmit ring: one per queue */ struct igb_tx_ring { + struct lwkt_serialize tx_serialize; struct igb_softc *sc; uint32_t me; struct igb_dma txdma; @@ -204,6 +208,7 @@ struct igb_tx_ring { * Receive ring: one per queue */ struct igb_rx_ring { + struct lwkt_serialize rx_serialize; struct igb_softc *sc; uint32_t me; struct igb_dma rxdma; @@ -269,6 +274,12 @@ struct igb_softc { /* Multicast array pointer */ uint8_t *mta; + int serialize_cnt; + int tx_serialize; + int rx_serialize; + struct lwkt_serialize *serializes[IGB_NSERIALIZE]; + struct lwkt_serialize main_serialize; + int intr_rate; uint32_t intr_mask; -- 2.41.0