From f0a26983b6dd6b7880fa53aa7314918e56415aba Mon Sep 17 00:00:00 2001 From: Sepherosa Ziehau Date: Fri, 11 Jan 2013 13:31:30 +0800 Subject: [PATCH] if: Multiple TX queue support step 1 of many; introduce ifaltq subqueue Put the plain queue information, e.g. queue header and tail, serializer, packet staging scoreboard and ifnet.if_start schedule netmsg etc. into its own structure (subqueue). ifaltq structure could have multiple of subqueues based on the count that drivers can specify. Subqueue's enqueue, dequeue, purging and states updating are protected by the subqueue's serializer, so for hardwares supporting multiple TX queues, contention on queuing operation could be greatly reduced. The subqueue is passed to if_start to let the driver know which hardware TX queue to work on. Only the related driver's TX queue serializer will be held, so for hardwares supporting multiple TX queues, contention on driver's TX queue serializer could be greatly reduced. Bunch of ifsq_ prefixed functions are added, which is used to perform various operations on subqueues. Commonly used ifq_ prefixed functions are still kept mainly for the drivers which do not support multiple TX queues (well, these functions also ease the netif/ convertion in this step :). All of the pseudo network devices under sys/net are converted to use the new subqueue operation. netproto/802_11 is converted too. igb(4) is converted to use the new subqueue operation, the rest of the network drivers are only changed for the if_start interface modification. For ALTQs which have packet scheduler enabled, only the first subqueue is used (*). (*) Whether we should utilize multiple TX queues if ALTQ's packet scheduler is enabled is quite questionable. Mainly because hardware's multiple TX queue packet dequeue mechanism could have negative impact on ALTQ's packet scheduler's decision. --- sys/bus/u4b/net/if_usie.c | 6 +- sys/bus/u4b/net/uhso.c | 6 +- sys/bus/u4b/net/usb_ethernet.c | 8 +- sys/bus/u4b/net/usb_ethernet.h | 2 +- sys/dev/atm/en/midway.c | 4 +- sys/dev/netif/acx/if_acx.c | 5 +- sys/dev/netif/ae/if_ae.c | 5 +- sys/dev/netif/age/if_age.c | 5 +- sys/dev/netif/alc/if_alc.c | 5 +- sys/dev/netif/ale/if_ale.c | 5 +- sys/dev/netif/an/if_an.c | 6 +- sys/dev/netif/ar/if_ar.c | 11 +- sys/dev/netif/ath/ath/if_ath.c | 16 +- sys/dev/netif/aue/if_aue.c | 5 +- sys/dev/netif/axe/if_axe.c | 6 +- sys/dev/netif/bce/if_bce.c | 5 +- sys/dev/netif/bfe/if_bfe.c | 5 +- sys/dev/netif/bge/if_bge.c | 6 +- sys/dev/netif/bnx/if_bnx.c | 6 +- sys/dev/netif/bwi/if_bwi.c | 5 +- sys/dev/netif/cs/if_cs.c | 6 +- sys/dev/netif/cue/if_cue.c | 6 +- sys/dev/netif/dc/if_dc.c | 5 +- sys/dev/netif/de/if_de.c | 5 +- sys/dev/netif/ed/if_ed.c | 6 +- sys/dev/netif/em/if_em.c | 5 +- sys/dev/netif/emx/if_emx.c | 5 +- sys/dev/netif/ep/if_ep.c | 6 +- sys/dev/netif/et/if_et.c | 5 +- sys/dev/netif/ex/if_ex.c | 5 +- sys/dev/netif/fe/if_fe.c | 7 +- sys/dev/netif/fwe/if_fwe.c | 5 +- sys/dev/netif/fxp/if_fxp.c | 5 +- sys/dev/netif/igb/if_igb.c | 91 +++-- sys/dev/netif/igb/if_igb.h | 1 + sys/dev/netif/iwi/if_iwi.c | 5 +- sys/dev/netif/iwl/iwl2100.c | 5 +- sys/dev/netif/iwn/if_iwn.c | 5 +- sys/dev/netif/ixgbe/ixgbe.c | 6 +- sys/dev/netif/jme/if_jme.c | 5 +- sys/dev/netif/kue/if_kue.c | 6 +- sys/dev/netif/lge/if_lge.c | 6 +- sys/dev/netif/lgue/if_lgue.c | 6 +- sys/dev/netif/lnc/lance.c | 5 +- sys/dev/netif/msk/if_msk.c | 5 +- sys/dev/netif/mxge/if_mxge.c | 3 +- sys/dev/netif/my/if_my.c | 5 +- sys/dev/netif/ndis/if_ndis.c | 8 +- sys/dev/netif/nfe/if_nfe.c | 5 +- sys/dev/netif/nge/if_nge.c | 6 +- sys/dev/netif/pcn/if_pcn.c | 6 +- sys/dev/netif/ral/rt2560.c | 5 +- sys/dev/netif/ral/rt2661.c | 5 +- sys/dev/netif/re/if_re.c | 5 +- sys/dev/netif/rl/if_rl.c | 6 +- sys/dev/netif/rtw/rtw.c | 15 +- sys/dev/netif/rue/if_rue.c | 6 +- sys/dev/netif/rum/if_rum.c | 5 +- sys/dev/netif/sbni/if_sbni.c | 6 +- sys/dev/netif/sbsh/if_sbsh.c | 6 +- sys/dev/netif/sf/if_sf.c | 6 +- sys/dev/netif/sis/if_sis.c | 6 +- sys/dev/netif/sk/if_sk.c | 5 +- sys/dev/netif/sln/if_sln.c | 5 +- sys/dev/netif/sn/if_sn.c | 6 +- sys/dev/netif/sr/if_sr.c | 12 +- sys/dev/netif/ste/if_ste.c | 6 +- sys/dev/netif/stge/if_stge.c | 5 +- sys/dev/netif/ti/if_ti.c | 6 +- sys/dev/netif/tl/if_tl.c | 6 +- sys/dev/netif/tx/if_tx.c | 6 +- sys/dev/netif/txp/if_txp.c | 6 +- sys/dev/netif/ural/if_ural.c | 5 +- sys/dev/netif/vge/if_vge.c | 5 +- sys/dev/netif/vr/if_vr.c | 6 +- sys/dev/netif/vx/if_vx.c | 6 +- sys/dev/netif/wb/if_wb.c | 6 +- sys/dev/netif/wi/if_wi.c | 5 +- sys/dev/netif/wi/if_wi_pci.c | 2 +- sys/dev/netif/wpi/if_wpi.c | 7 +- sys/dev/netif/xe/if_xe.c | 6 +- sys/dev/netif/xl/if_xl.c | 15 +- sys/dev/virtual/vkernel/net/if_vke.c | 15 +- sys/net/altq/altq_cbq.c | 88 +++-- sys/net/altq/altq_fairq.c | 75 +++- sys/net/altq/altq_hfsc.c | 73 +++- sys/net/altq/altq_priq.c | 74 +++- sys/net/altq/altq_rmclass.c | 5 +- sys/net/altq/altq_subr.c | 64 ++-- sys/net/altq/if_altq.h | 109 ++++-- sys/net/bridge/if_bridge.c | 11 +- sys/net/ef/if_ef.c | 18 +- sys/net/if.c | 396 +++++++++++--------- sys/net/if_loop.c | 11 +- sys/net/if_mib.c | 7 +- sys/net/if_var.h | 6 +- sys/net/ifq_var.h | 335 ++++++++++++----- sys/net/pf/if_pflog.c | 7 +- sys/net/pf/if_pfsync.c | 7 +- sys/net/ppp/if_ppp.c | 18 +- sys/net/sl/if_sl.c | 6 +- sys/net/sppp/if_spppsubr.c | 36 +- sys/net/tap/if_tap.c | 23 +- sys/net/tun/if_tun.c | 18 +- sys/net/vlan/if_vlan.c | 15 +- sys/netgraph/eiface/ng_eiface.c | 4 +- sys/netgraph/fec/ng_fec.c | 4 +- sys/netgraph/iface/ng_iface.c | 4 +- sys/netgraph7/dragonfly.h | 4 +- sys/netgraph7/iface/ng_iface.c | 4 +- sys/netgraph7/ng_eiface.c | 4 +- sys/netgraph7/ng_fec.c | 4 +- sys/netgraph7/ng_sppp.c | 4 +- sys/netinet/ip_carp.c | 4 +- sys/netproto/802_11/ieee80211_proto.h | 2 +- sys/netproto/802_11/wlan/ieee80211_hostap.c | 7 +- sys/netproto/802_11/wlan/ieee80211_output.c | 16 +- sys/netproto/802_11/wlan/ieee80211_power.c | 35 +- sys/netproto/802_11/wlan/ieee80211_proto.c | 7 +- sys/netproto/ipx/ipx_ip.c | 4 +- usr.bin/netstat/if.c | 8 +- 121 files changed, 1347 insertions(+), 754 deletions(-) diff --git a/sys/bus/u4b/net/if_usie.c b/sys/bus/u4b/net/if_usie.c index ed9054fc1f..f9201280ee 100644 --- a/sys/bus/u4b/net/if_usie.c +++ b/sys/bus/u4b/net/if_usie.c @@ -118,7 +118,7 @@ static void usie_if_sync_to(void *); static void usie_if_sync_cb(void *, int); static void usie_if_status_cb(void *, int); -static void usie_if_start(struct ifnet *); +static void usie_if_start(struct ifnet *, struct ifaltq_subque *); static int usie_if_output(struct ifnet *, struct mbuf *, struct sockaddr *, struct route *); static void usie_if_init(void *); static void usie_if_stop(struct usie_softc *); @@ -1141,10 +1141,12 @@ usie_if_status_cb(void *arg, int pending) } static void -usie_if_start(struct ifnet *ifp) +usie_if_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct usie_softc *sc = ifp->if_softc; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { DPRINTF("Not running\n"); return; diff --git a/sys/bus/u4b/net/uhso.c b/sys/bus/u4b/net/uhso.c index 821117083f..2c778409e3 100644 --- a/sys/bus/u4b/net/uhso.c +++ b/sys/bus/u4b/net/uhso.c @@ -463,7 +463,7 @@ static void uhso_ucom_cfg_get_status(struct ucom_softc *, uint8_t *, uint8_t *); static void uhso_ucom_cfg_set_dtr(struct ucom_softc *, uint8_t); static void uhso_ucom_cfg_set_rts(struct ucom_softc *, uint8_t); static void uhso_if_init(void *); -static void uhso_if_start(struct ifnet *); +static void uhso_if_start(struct ifnet *, struct ifaltq_subque *); static void uhso_if_stop(struct uhso_softc *); static int uhso_if_ioctl(struct ifnet *, u_long, caddr_t); static int uhso_if_output(struct ifnet *, struct mbuf *, struct sockaddr *, @@ -1883,10 +1883,12 @@ uhso_if_output(struct ifnet *ifp, struct mbuf *m0, struct sockaddr *dst, } static void -uhso_if_start(struct ifnet *ifp) +uhso_if_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct uhso_softc *sc = ifp->if_softc; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { UHSO_DPRINTF(1, "Not running\n"); return; diff --git a/sys/bus/u4b/net/usb_ethernet.c b/sys/bus/u4b/net/usb_ethernet.c index 439b70110a..6fd4d30215 100644 --- a/sys/bus/u4b/net/usb_ethernet.c +++ b/sys/bus/u4b/net/usb_ethernet.c @@ -79,7 +79,7 @@ static usb_proc_callback_t ue_start_task; static usb_proc_callback_t ue_stop_task; static void ue_init(void *); -static void ue_start(struct ifnet *); +static void ue_start(struct ifnet *, struct ifaltq_subque *); static int ue_ifmedia_upd(struct ifnet *); static void ue_watchdog(void *); @@ -393,17 +393,19 @@ ue_stop_task(struct usb_proc_msg *_task) } void -uether_start(struct ifnet *ifp) +uether_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { ue_start(ifp); } static void -ue_start(struct ifnet *ifp) +ue_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct usb_ether *ue = ifp->if_softc; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if ((ifp->if_flags & IFF_RUNNING) == 0) return; UE_LOCK(ue); diff --git a/sys/bus/u4b/net/usb_ethernet.h b/sys/bus/u4b/net/usb_ethernet.h index 012c8c1a07..4757586c9a 100644 --- a/sys/bus/u4b/net/usb_ethernet.h +++ b/sys/bus/u4b/net/usb_ethernet.h @@ -124,5 +124,5 @@ int uether_rxbuf(struct usb_ether *, unsigned int, unsigned int); void uether_rxflush(struct usb_ether *); uint8_t uether_is_gone(struct usb_ether *); -void uether_start(struct ifnet *); +void uether_start(struct ifnet *, struct ifaltq_subque *); #endif /* _USB_ETHERNET_H_ */ diff --git a/sys/dev/atm/en/midway.c b/sys/dev/atm/en/midway.c index acb224946c..ee0ddde2be 100644 --- a/sys/dev/atm/en/midway.c +++ b/sys/dev/atm/en/midway.c @@ -263,7 +263,7 @@ STATIC void en_txdma (struct en_softc *, int); STATIC void en_txlaunch (struct en_softc *, int, struct en_launch *); STATIC void en_service (struct en_softc *); -STATIC void en_start (struct ifnet *); +STATIC void en_start (struct ifnet *, struct ifaltq_subque *); STATIC INLINE int en_sz2b (int); STATIC INLINE void en_write (struct en_softc *, u_int32_t, u_int32_t); @@ -1463,7 +1463,7 @@ en_loadvc(struct en_softc *sc, int vc) */ STATIC void -en_start(struct ifnet *ifp) +en_start(struct ifnet *ifp, struct ifaltq_subque *ifsq __unused) { struct en_softc *sc = (struct en_softc *) ifp->if_softc; struct mbuf *m, *lastm, *prev; diff --git a/sys/dev/netif/acx/if_acx.c b/sys/dev/netif/acx/if_acx.c index 7820cd24fe..269131d7d8 100644 --- a/sys/dev/netif/acx/if_acx.c +++ b/sys/dev/netif/acx/if_acx.c @@ -114,7 +114,7 @@ static int acx_detach(device_t); static int acx_shutdown(device_t); static void acx_init(void *); -static void acx_start(struct ifnet *); +static void acx_start(struct ifnet *, struct ifaltq_subque *); static int acx_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void acx_watchdog(struct ifnet *); @@ -1093,7 +1093,7 @@ acx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cr) } static void -acx_start(struct ifnet *ifp) +acx_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct acx_softc *sc = ifp->if_softc; struct ieee80211com *ic = &sc->sc_ic; @@ -1101,6 +1101,7 @@ acx_start(struct ifnet *ifp) struct acx_txbuf *buf; int trans, idx; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); if ((sc->sc_flags & ACX_FLAG_FW_LOADED) == 0) { diff --git a/sys/dev/netif/ae/if_ae.c b/sys/dev/netif/ae/if_ae.c index 8243c86a40..63209a406f 100644 --- a/sys/dev/netif/ae/if_ae.c +++ b/sys/dev/netif/ae/if_ae.c @@ -92,7 +92,7 @@ static void ae_miibus_statchg(device_t); static int ae_mediachange(struct ifnet *); static void ae_mediastatus(struct ifnet *, struct ifmediareq *); static void ae_init(void *); -static void ae_start(struct ifnet *); +static void ae_start(struct ifnet *, struct ifaltq_subque *); static int ae_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void ae_watchdog(struct ifnet *); static void ae_stop(struct ae_softc *); @@ -1152,11 +1152,12 @@ ae_encap(struct ae_softc *sc, struct mbuf **m_head) } static void -ae_start(struct ifnet *ifp) +ae_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct ae_softc *sc = ifp->if_softc; int error, trans; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); #ifdef AE_DEBUG diff --git a/sys/dev/netif/age/if_age.c b/sys/dev/netif/age/if_age.c index 44959fb431..164eead4e3 100644 --- a/sys/dev/netif/age/if_age.c +++ b/sys/dev/netif/age/if_age.c @@ -85,7 +85,7 @@ static void age_miibus_statchg(device_t); static void age_init(void *); static int age_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); -static void age_start(struct ifnet *); +static void age_start(struct ifnet *, struct ifaltq_subque *); static void age_watchdog(struct ifnet *); static void age_mediastatus(struct ifnet *, struct ifmediareq *); static int age_mediachange(struct ifnet *); @@ -1574,12 +1574,13 @@ age_encap(struct age_softc *sc, struct mbuf **m_head) } static void -age_start(struct ifnet *ifp) +age_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct age_softc *sc = ifp->if_softc; struct mbuf *m_head; int enq; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); if ((sc->age_flags & AGE_FLAG_LINK) == 0) { diff --git a/sys/dev/netif/alc/if_alc.c b/sys/dev/netif/alc/if_alc.c index 763018ff3b..572c075917 100644 --- a/sys/dev/netif/alc/if_alc.c +++ b/sys/dev/netif/alc/if_alc.c @@ -110,7 +110,7 @@ static void alc_miibus_statchg(device_t); static int alc_miibus_writereg(device_t, int, int, int); static void alc_init(void *); -static void alc_start(struct ifnet *); +static void alc_start(struct ifnet *, struct ifaltq_subque *); static void alc_watchdog(struct alc_softc *); static int alc_mediachange(struct ifnet *); static void alc_mediastatus(struct ifnet *, struct ifmediareq *); @@ -2173,12 +2173,13 @@ alc_encap(struct alc_softc *sc, struct mbuf **m_head) } static void -alc_start(struct ifnet *ifp) +alc_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct alc_softc *sc = ifp->if_softc; struct mbuf *m_head; int enq; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); /* Reclaim transmitted frames. */ diff --git a/sys/dev/netif/ale/if_ale.c b/sys/dev/netif/ale/if_ale.c index 2d922d9607..edd92a3780 100644 --- a/sys/dev/netif/ale/if_ale.c +++ b/sys/dev/netif/ale/if_ale.c @@ -88,7 +88,7 @@ static int ale_miibus_writereg(device_t, int, int, int); static void ale_miibus_statchg(device_t); static void ale_init(void *); -static void ale_start(struct ifnet *); +static void ale_start(struct ifnet *, struct ifaltq_subque *); static int ale_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void ale_watchdog(struct ifnet *); static int ale_mediachange(struct ifnet *); @@ -1660,12 +1660,13 @@ ale_encap(struct ale_softc *sc, struct mbuf **m_head) } static void -ale_start(struct ifnet *ifp) +ale_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct ale_softc *sc = ifp->if_softc; struct mbuf *m_head; int enq; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); if ((sc->ale_flags & ALE_FLAG_LINK) == 0) { diff --git a/sys/dev/netif/an/if_an.c b/sys/dev/netif/an/if_an.c index 99c4ee1e04..08a9524075 100644 --- a/sys/dev/netif/an/if_an.c +++ b/sys/dev/netif/an/if_an.c @@ -139,7 +139,7 @@ static int an_ioctl (struct ifnet *, u_long, caddr_t, struct ucred *); static void an_init (void *); static int an_init_tx_ring (struct an_softc *); -static void an_start (struct ifnet *); +static void an_start (struct ifnet *, struct ifaltq_subque *); static void an_watchdog (struct ifnet *); static void an_rxeof (struct an_softc *); static void an_txeof (struct an_softc *, int); @@ -2474,7 +2474,7 @@ an_init(void *xsc) } static void -an_start(struct ifnet *ifp) +an_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct an_softc *sc; struct mbuf *m0 = NULL; @@ -2485,6 +2485,8 @@ an_start(struct ifnet *ifp) struct an_card_tx_desc an_tx_desc; u_int8_t *buf; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + sc = ifp->if_softc; if (ifq_is_oactive(&ifp->if_snd)) diff --git a/sys/dev/netif/ar/if_ar.c b/sys/dev/netif/ar/if_ar.c index 97a24d6d12..a2fded9395 100644 --- a/sys/dev/netif/ar/if_ar.c +++ b/sys/dev/netif/ar/if_ar.c @@ -190,7 +190,7 @@ MODULE_DEPEND(ng_sync_ar, netgraph, NG_ABI_VERSION, NG_ABI_VERSION, NG_ABI_VERSI static void arintr(void *arg); static void ar_xmit(struct ar_softc *sc); #ifndef NETGRAPH -static void arstart(struct ifnet *ifp); +static void arstart(struct ifnet *ifp, struct ifaltq_subque *); static int arioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *); static void arwatchdog(struct ifnet *ifp); #else /* NETGRAPH */ @@ -622,7 +622,7 @@ ar_xmit(struct ar_softc *sc) */ #ifndef NETGRAPH static void -arstart(struct ifnet *ifp) +arstart(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct ar_softc *sc = ifp->if_softc; #else /* NETGRAPH */ @@ -803,7 +803,7 @@ arioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cr) if(!was_up && should_be_up) { /* Interface should be up -- start it. */ ar_up(sc); - arstart(ifp); + arstart(ifp, ifq_get_subq_default(&ifp->if_snd)); /* XXX Maybe clear the IFF_UP flag so that the link * will only go up after sppp lcp and ipcp negotiation. */ @@ -864,7 +864,7 @@ arwatchdog(struct ar_softc *sc) ar_xmit(sc); #ifndef NETGRAPH - arstart(ifp); + arstart(ifp, ifq_get_subq_default(&ifp->if_snd)); #else /* NETGRAPH */ arstart(sc); #endif /* NETGRAPH */ @@ -2020,7 +2020,8 @@ ar_dmac_intr(struct ar_hardc *hc, int scano, u_char isr1) if(dotxstart & 0x0C) { sc = &hc->sc[mch + (NCHAN * scano)]; #ifndef NETGRAPH - arstart(&sc->ifsppp.pp_if); + arstart(&sc->ifsppp.pp_if, + ifq_get_subq_default(&sc->ifsppp.pp_if.if_snd)); #else /* NETGRAPH */ arstart(sc); #endif /* NETGRAPH */ diff --git a/sys/dev/netif/ath/ath/if_ath.c b/sys/dev/netif/ath/ath/if_ath.c index 5a88cf2035..92cb46f534 100644 --- a/sys/dev/netif/ath/ath/if_ath.c +++ b/sys/dev/netif/ath/ath/if_ath.c @@ -122,7 +122,7 @@ static void ath_vap_delete(struct ieee80211vap *); static void ath_init(void *); static void ath_stop_locked(struct ifnet *); static void ath_stop(struct ifnet *); -static void ath_start(struct ifnet *); +static void ath_start(struct ifnet *, struct ifaltq_subque *); static int ath_reset(struct ifnet *); static int ath_reset_vap(struct ieee80211vap *, u_long); static int ath_media_change(struct ifnet *); @@ -1700,7 +1700,7 @@ ath_reset(struct ifnet *ifp) } ath_hal_intrset(ah, sc->sc_imask); - ath_start(ifp); /* restart xmit */ + if_devstart(ifp); /* restart xmit */ return 0; } @@ -1806,7 +1806,7 @@ ath_txfrag_setup(struct ath_softc *sc, ath_bufhead *frags, } static void -ath_start(struct ifnet *ifp) +ath_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct ath_softc *sc = ifp->if_softc; struct ieee80211_node *ni; @@ -1814,6 +1814,8 @@ ath_start(struct ifnet *ifp) struct mbuf *m, *next; ath_bufhead frags; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if ((ifp->if_flags & IFF_RUNNING) == 0 || sc->sc_invalid) { ifq_purge(&ifp->if_snd); return; @@ -4057,7 +4059,7 @@ rx_next: ieee80211_ff_age_all(ic, 100); #endif if (!ifq_is_empty(&ifp->if_snd)) - ath_start(ifp); + if_devstart(ifp); } wlan_serialize_exit(); #undef PA2DESC @@ -5045,7 +5047,7 @@ ath_tx_task_q0(void *arg, int npending) if (sc->sc_softled) ath_led_event(sc, sc->sc_txrix); - ath_start(ifp); + if_devstart(ifp); wlan_serialize_exit(); } @@ -5084,7 +5086,7 @@ ath_tx_task_q0123(void *arg, int npending) if (sc->sc_softled) ath_led_event(sc, sc->sc_txrix); - ath_start(ifp); + if_devstart(ifp); wlan_serialize_exit(); } @@ -5117,7 +5119,7 @@ ath_tx_task(void *arg, int npending) if (sc->sc_softled) ath_led_event(sc, sc->sc_txrix); - ath_start(ifp); + if_devstart(ifp); wlan_serialize_exit(); } diff --git a/sys/dev/netif/aue/if_aue.c b/sys/dev/netif/aue/if_aue.c index 4f75f115a2..4b0a97e8b3 100644 --- a/sys/dev/netif/aue/if_aue.c +++ b/sys/dev/netif/aue/if_aue.c @@ -180,7 +180,7 @@ static void aue_rxeof(usbd_xfer_handle, usbd_private_handle, usbd_status); static void aue_txeof(usbd_xfer_handle, usbd_private_handle, usbd_status); static void aue_tick(void *); static void aue_rxstart(struct ifnet *); -static void aue_start(struct ifnet *); +static void aue_start(struct ifnet *, struct ifaltq_subque *); static int aue_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void aue_init(void *); static void aue_stop(struct aue_softc *); @@ -1084,11 +1084,12 @@ aue_encap(struct aue_softc *sc, struct mbuf *m, int idx) } static void -aue_start(struct ifnet *ifp) +aue_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct aue_softc *sc = ifp->if_softc; struct mbuf *m_head = NULL; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); AUE_LOCK(sc); if (!sc->aue_link) { diff --git a/sys/dev/netif/axe/if_axe.c b/sys/dev/netif/axe/if_axe.c index 5c27c1029d..6fd394f1b9 100644 --- a/sys/dev/netif/axe/if_axe.c +++ b/sys/dev/netif/axe/if_axe.c @@ -124,7 +124,7 @@ static void axe_rxeof(usbd_xfer_handle, usbd_private_handle, usbd_status); static void axe_txeof(usbd_xfer_handle, usbd_private_handle, usbd_status); static void axe_tick(void *); static void axe_rxstart(struct ifnet *); -static void axe_start(struct ifnet *); +static void axe_start(struct ifnet *, struct ifaltq_subque *); static int axe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void axe_init(void *); static void axe_stop(struct axe_softc *); @@ -755,11 +755,13 @@ axe_encap(struct axe_softc *sc, struct mbuf *m, int idx) } static void -axe_start(struct ifnet *ifp) +axe_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct axe_softc *sc = ifp->if_softc; struct mbuf *m_head = NULL; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if (!sc->axe_link) { ifq_purge(&ifp->if_snd); return; diff --git a/sys/dev/netif/bce/if_bce.c b/sys/dev/netif/bce/if_bce.c index b05af93fdb..952fa9f371 100644 --- a/sys/dev/netif/bce/if_bce.c +++ b/sys/dev/netif/bce/if_bce.c @@ -416,7 +416,7 @@ static void bce_free_tx_chain(struct bce_softc *); static int bce_encap(struct bce_softc *, struct mbuf **, int *); static int bce_tso_setup(struct bce_softc *, struct mbuf **, uint16_t *, uint16_t *); -static void bce_start(struct ifnet *); +static void bce_start(struct ifnet *, struct ifaltq_subque *); static int bce_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void bce_watchdog(struct ifnet *); static int bce_ifmedia_upd(struct ifnet *); @@ -4987,11 +4987,12 @@ back: /* Nothing. */ /****************************************************************************/ static void -bce_start(struct ifnet *ifp) +bce_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct bce_softc *sc = ifp->if_softc; int count = 0; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); /* If there's no link or the transmit queue is empty then just exit. */ diff --git a/sys/dev/netif/bfe/if_bfe.c b/sys/dev/netif/bfe/if_bfe.c index 5cbc2abba0..cff656a247 100644 --- a/sys/dev/netif/bfe/if_bfe.c +++ b/sys/dev/netif/bfe/if_bfe.c @@ -91,7 +91,7 @@ static int bfe_probe(device_t); static int bfe_attach(device_t); static int bfe_detach(device_t); static void bfe_intr(void *); -static void bfe_start(struct ifnet *); +static void bfe_start(struct ifnet *, struct ifaltq_subque *); static int bfe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void bfe_init(void *); static void bfe_stop(struct bfe_softc *); @@ -1275,12 +1275,13 @@ fail: * Set up to transmit a packet */ static void -bfe_start(struct ifnet *ifp) +bfe_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct bfe_softc *sc = ifp->if_softc; struct mbuf *m_head = NULL; int idx, need_trans; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); /* diff --git a/sys/dev/netif/bge/if_bge.c b/sys/dev/netif/bge/if_bge.c index 0717804faa..1971c49296 100644 --- a/sys/dev/netif/bge/if_bge.c +++ b/sys/dev/netif/bge/if_bge.c @@ -320,7 +320,7 @@ static void bge_msi_oneshot(void *); static void bge_intr(struct bge_softc *); static void bge_enable_intr(struct bge_softc *); static void bge_disable_intr(struct bge_softc *); -static void bge_start(struct ifnet *); +static void bge_start(struct ifnet *, struct ifaltq_subque *); static int bge_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void bge_init(void *); static void bge_stop(struct bge_softc *); @@ -3497,13 +3497,15 @@ bge_xmit(struct bge_softc *sc, uint32_t prodidx) * to the mbuf data regions directly in the transmit descriptors. */ static void -bge_start(struct ifnet *ifp) +bge_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct bge_softc *sc = ifp->if_softc; struct mbuf *m_head = NULL; uint32_t prodidx; int nsegs = 0; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd)) return; diff --git a/sys/dev/netif/bnx/if_bnx.c b/sys/dev/netif/bnx/if_bnx.c index bedbe86602..fef6ef25be 100644 --- a/sys/dev/netif/bnx/if_bnx.c +++ b/sys/dev/netif/bnx/if_bnx.c @@ -152,7 +152,7 @@ static void bnx_disable_intr(struct bnx_softc *); static void bnx_txeof(struct bnx_softc *, uint16_t); static void bnx_rxeof(struct bnx_softc *, uint16_t, int); -static void bnx_start(struct ifnet *); +static void bnx_start(struct ifnet *, struct ifaltq_subque *); static int bnx_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void bnx_init(void *); static void bnx_stop(struct bnx_softc *); @@ -2975,13 +2975,15 @@ back: * to the mbuf data regions directly in the transmit descriptors. */ static void -bnx_start(struct ifnet *ifp) +bnx_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct bnx_softc *sc = ifp->if_softc; struct mbuf *m_head = NULL; uint32_t prodidx; int nsegs = 0; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd)) return; diff --git a/sys/dev/netif/bwi/if_bwi.c b/sys/dev/netif/bwi/if_bwi.c index bf52caf64a..6e2b077bce 100644 --- a/sys/dev/netif/bwi/if_bwi.c +++ b/sys/dev/netif/bwi/if_bwi.c @@ -84,7 +84,7 @@ static int bwi_shutdown(device_t); static void bwi_init(void *); static int bwi_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); -static void bwi_start(struct ifnet *); +static void bwi_start(struct ifnet *, struct ifaltq_subque *); static void bwi_watchdog(struct ifnet *); static int bwi_newstate(struct ieee80211com *, enum ieee80211_state, int); static void bwi_updateslot(struct ifnet *); @@ -1568,13 +1568,14 @@ bwi_ioctl(struct ifnet *ifp, u_long cmd, caddr_t req, struct ucred *cr) } static void -bwi_start(struct ifnet *ifp) +bwi_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct bwi_softc *sc = ifp->if_softc; struct ieee80211com *ic = &sc->sc_ic; struct bwi_txbuf_data *tbd = &sc->sc_tx_bdata[BWI_TX_DATA_RING]; int trans, idx; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); if (ifq_is_oactive(&ifp->if_snd) || (ifp->if_flags & IFF_RUNNING) == 0) diff --git a/sys/dev/netif/cs/if_cs.c b/sys/dev/netif/cs/if_cs.c index e6ebb7dcdf..44862bf19b 100644 --- a/sys/dev/netif/cs/if_cs.c +++ b/sys/dev/netif/cs/if_cs.c @@ -75,7 +75,7 @@ SYSCTL_INT(_machdep, OID_AUTO, cs_recv_delay, CTLFLAG_RW, &cs_recv_delay, 0, "") static void cs_init(void *); static int cs_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); -static void cs_start(struct ifnet *); +static void cs_start(struct ifnet *, struct ifaltq_subque *); static void cs_stop(struct cs_softc *); static void cs_reset(struct cs_softc *); static void cs_watchdog(struct ifnet *); @@ -981,12 +981,14 @@ cs_xmit_buf( struct cs_softc *sc ) } static void -cs_start(struct ifnet *ifp) +cs_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { int length; struct mbuf *m, *mp; struct cs_softc *sc = ifp->if_softc; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + for (;;) { if (sc->buf_len) length = sc->buf_len; diff --git a/sys/dev/netif/cue/if_cue.c b/sys/dev/netif/cue/if_cue.c index 2d9a50b417..e00f4ee20f 100644 --- a/sys/dev/netif/cue/if_cue.c +++ b/sys/dev/netif/cue/if_cue.c @@ -95,7 +95,7 @@ static void cue_rxeof(usbd_xfer_handle, usbd_private_handle, usbd_status); static void cue_txeof(usbd_xfer_handle, usbd_private_handle, usbd_status); static void cue_tick(void *); static void cue_rxstart(struct ifnet *); -static void cue_start(struct ifnet *); +static void cue_start(struct ifnet *, struct ifaltq_subque *); static int cue_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void cue_init(void *); static void cue_stop(struct cue_softc *); @@ -812,11 +812,13 @@ cue_encap(struct cue_softc *sc, struct mbuf *m, int idx) } static void -cue_start(struct ifnet *ifp) +cue_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct cue_softc *sc; struct mbuf *m_head = NULL; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + sc = ifp->if_softc; CUE_LOCK(sc); diff --git a/sys/dev/netif/dc/if_dc.c b/sys/dev/netif/dc/if_dc.c index 66346fbf12..e8566e8e0b 100644 --- a/sys/dev/netif/dc/if_dc.c +++ b/sys/dev/netif/dc/if_dc.c @@ -215,7 +215,7 @@ static void dc_txeof (struct dc_softc *); static void dc_tick (void *); static void dc_tx_underrun (struct dc_softc *); static void dc_intr (void *); -static void dc_start (struct ifnet *); +static void dc_start (struct ifnet *, struct ifaltq_subque *); static int dc_ioctl (struct ifnet *, u_long, caddr_t, struct ucred *); #ifdef IFPOLL_ENABLE @@ -3073,12 +3073,13 @@ dc_encap(struct dc_softc *sc, struct mbuf *m_head, u_int32_t *txidx) */ static void -dc_start(struct ifnet *ifp) +dc_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct dc_softc *sc; struct mbuf *m_head, *m_defragged; int idx, need_trans; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); sc = ifp->if_softc; if (!sc->dc_link) { diff --git a/sys/dev/netif/de/if_de.c b/sys/dev/netif/de/if_de.c index d323bde388..39b1553094 100644 --- a/sys/dev/netif/de/if_de.c +++ b/sys/dev/netif/de/if_de.c @@ -103,7 +103,7 @@ static void tulip_intr_shared(void *); static void tulip_intr_normal(void *); static void tulip_init(tulip_softc_t *); static void tulip_reset(tulip_softc_t *); -static void tulip_ifstart(struct ifnet *); +static void tulip_ifstart(struct ifnet *, struct ifaltq_subque *); static struct mbuf *tulip_txput(tulip_softc_t *, struct mbuf *); static void tulip_txput_setup(tulip_softc_t *); static void tulip_rx_intr(tulip_softc_t *); @@ -3786,10 +3786,11 @@ tulip_ifioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred * cr) } static void -tulip_ifstart(struct ifnet *ifp) +tulip_ifstart(struct ifnet *ifp, struct ifaltq_subque *ifsq) { tulip_softc_t *sc = (tulip_softc_t *)ifp->if_softc; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); if (sc->tulip_if.if_flags & IFF_RUNNING) { if ((sc->tulip_flags & (TULIP_WANTSETUP|TULIP_TXPROBE_ACTIVE)) == TULIP_WANTSETUP) diff --git a/sys/dev/netif/ed/if_ed.c b/sys/dev/netif/ed/if_ed.c index cbcc3449bf..50196e6290 100644 --- a/sys/dev/netif/ed/if_ed.c +++ b/sys/dev/netif/ed/if_ed.c @@ -76,7 +76,7 @@ devclass_t ed_devclass; static void ed_init (void *); static int ed_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); -static void ed_start (struct ifnet *); +static void ed_start (struct ifnet *, struct ifaltq_subque *); static void ed_reset (struct ifnet *); static void ed_watchdog (struct ifnet *); #ifndef ED_NO_MIIBUS @@ -2065,13 +2065,15 @@ ed_xmit(struct ed_softc *sc) * (i.e. that the output part of the interface is idle) */ static void -ed_start(struct ifnet *ifp) +ed_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct ed_softc *sc = ifp->if_softc; struct mbuf *m0, *m; caddr_t buffer; int len; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if (sc->gone) { kprintf("ed_start(%p) GONE\n",ifp); ifq_purge(&ifp->if_snd); diff --git a/sys/dev/netif/em/if_em.c b/sys/dev/netif/em/if_em.c index 97222dab08..5e223631da 100644 --- a/sys/dev/netif/em/if_em.c +++ b/sys/dev/netif/em/if_em.c @@ -265,7 +265,7 @@ static int em_resume(device_t); static void em_init(void *); static void em_stop(struct adapter *); static int em_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); -static void em_start(struct ifnet *); +static void em_start(struct ifnet *, struct ifaltq_subque *); #ifdef IFPOLL_ENABLE static void em_npoll(struct ifnet *, struct ifpoll_info *); static void em_npoll_compat(struct ifnet *, void *, int); @@ -987,12 +987,13 @@ em_resume(device_t dev) } static void -em_start(struct ifnet *ifp) +em_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct adapter *adapter = ifp->if_softc; struct mbuf *m_head; int idx = -1, nsegs = 0; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd)) diff --git a/sys/dev/netif/emx/if_emx.c b/sys/dev/netif/emx/if_emx.c index b3f68d6367..653d93ec0f 100644 --- a/sys/dev/netif/emx/if_emx.c +++ b/sys/dev/netif/emx/if_emx.c @@ -176,7 +176,7 @@ static int emx_resume(device_t); static void emx_init(void *); static void emx_stop(struct emx_softc *); static int emx_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); -static void emx_start(struct ifnet *); +static void emx_start(struct ifnet *, struct ifaltq_subque *); #ifdef IFPOLL_ENABLE static void emx_npoll(struct ifnet *, struct ifpoll_info *); static void emx_npoll_status(struct ifnet *); @@ -935,13 +935,14 @@ emx_resume(device_t dev) } static void -emx_start(struct ifnet *ifp) +emx_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct emx_softc *sc = ifp->if_softc; struct emx_txdata *tdata = &sc->tx_data; struct mbuf *m_head; int idx = -1, nsegs = 0; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(&sc->tx_data.tx_serialize); if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd)) diff --git a/sys/dev/netif/ep/if_ep.c b/sys/dev/netif/ep/if_ep.c index 3a75732d84..186b9c78b6 100644 --- a/sys/dev/netif/ep/if_ep.c +++ b/sys/dev/netif/ep/if_ep.c @@ -103,7 +103,7 @@ static int ep_media2if_media[] = /* if functions */ static void ep_if_init (void *); static int ep_if_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); -static void ep_if_start (struct ifnet *); +static void ep_if_start (struct ifnet *, struct ifaltq_subque *); static void ep_if_watchdog (struct ifnet *); /* if_media functions */ @@ -413,7 +413,7 @@ ep_if_init(void *xsc) static const char padmap[] = {0, 3, 2, 1}; static void -ep_if_start(struct ifnet *ifp) +ep_if_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct ep_softc *sc = ifp->if_softc; u_int len; @@ -421,6 +421,8 @@ ep_if_start(struct ifnet *ifp) struct mbuf *top; int pad; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if (sc->gone) { ifq_purge(&ifp->if_snd); return; diff --git a/sys/dev/netif/et/if_et.c b/sys/dev/netif/et/if_et.c index bc7bd39613..2f0d1a79b7 100644 --- a/sys/dev/netif/et/if_et.c +++ b/sys/dev/netif/et/if_et.c @@ -77,7 +77,7 @@ static void et_miibus_statchg(device_t); static void et_init(void *); static int et_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); -static void et_start(struct ifnet *); +static void et_start(struct ifnet *, struct ifaltq_subque *); static void et_watchdog(struct ifnet *); static int et_ifmedia_upd(struct ifnet *); static void et_ifmedia_sts(struct ifnet *, struct ifmediareq *); @@ -1195,12 +1195,13 @@ et_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cr) } static void -et_start(struct ifnet *ifp) +et_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct et_softc *sc = ifp->if_softc; struct et_txbuf_data *tbd = &sc->sc_tx_data; int trans, oactive; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); if ((sc->sc_flags & ET_FLAG_TXRX_ENABLED) == 0) { diff --git a/sys/dev/netif/ex/if_ex.c b/sys/dev/netif/ex/if_ex.c index e487c1fbdb..782175ab7a 100644 --- a/sys/dev/netif/ex/if_ex.c +++ b/sys/dev/netif/ex/if_ex.c @@ -97,7 +97,7 @@ u_char plus_ee2irqmap[] = /* Network Interface Functions */ static void ex_init (void *); -static void ex_start (struct ifnet *); +static void ex_start (struct ifnet *, struct ifaltq_subque *); static int ex_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void ex_watchdog (struct ifnet *); @@ -360,7 +360,7 @@ ex_init(void *xsc) static void -ex_start(struct ifnet *ifp) +ex_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct ex_softc * sc = ifp->if_softc; int iobase = sc->iobase; @@ -369,6 +369,7 @@ ex_start(struct ifnet *ifp) struct mbuf * opkt; struct mbuf * m; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); DODEBUG(Start_End, kprintf("ex_start%d: start\n", unit);); /* diff --git a/sys/dev/netif/fe/if_fe.c b/sys/dev/netif/fe/if_fe.c index f68d7b721c..e90df0feb7 100644 --- a/sys/dev/netif/fe/if_fe.c +++ b/sys/dev/netif/fe/if_fe.c @@ -146,7 +146,8 @@ static void fe_init (void *); static void fe_intr (void *); static int fe_ioctl (struct ifnet *, u_long, caddr_t, struct ucred *); -static void fe_start (struct ifnet *); +static void fe_start (struct ifnet *, + struct ifaltq_subque *); static void fe_watchdog (struct ifnet *); static int fe_medchange (struct ifnet *); static void fe_medstat (struct ifnet *, struct ifmediareq *); @@ -1146,11 +1147,13 @@ fe_xmit (struct fe_softc *sc) * (i.e. that the output part of the interface is idle) */ void -fe_start (struct ifnet *ifp) +fe_start (struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct fe_softc *sc = ifp->if_softc; struct mbuf *m; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + #ifdef DIAGNOSTIC /* Just a sanity check. */ if ((sc->txb_count == 0) != (sc->txb_free == sc->txb_size)) { diff --git a/sys/dev/netif/fwe/if_fwe.c b/sys/dev/netif/fwe/if_fwe.c index 39a2605d31..3dada38ac1 100644 --- a/sys/dev/netif/fwe/if_fwe.c +++ b/sys/dev/netif/fwe/if_fwe.c @@ -73,7 +73,7 @@ #define TX_MAX_QUEUE (FWMAXQUEUE - 1) /* network interface */ -static void fwe_start (struct ifnet *); +static void fwe_start (struct ifnet *, struct ifaltq_subque *); static int fwe_ioctl (struct ifnet *, u_long, caddr_t, struct ucred *); static void fwe_init (void *); @@ -454,10 +454,11 @@ fwe_output_callback(struct fw_xfer *xfer) } static void -fwe_start(struct ifnet *ifp) +fwe_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct fwe_softc *fwe = ((struct fwe_eth_softc *)ifp->if_softc)->fwe; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); FWEDEBUG(ifp, "starting\n"); if (fwe->dma_ch < 0) { diff --git a/sys/dev/netif/fxp/if_fxp.c b/sys/dev/netif/fxp/if_fxp.c index 0d76f4f63c..dc4101cc3b 100644 --- a/sys/dev/netif/fxp/if_fxp.c +++ b/sys/dev/netif/fxp/if_fxp.c @@ -205,7 +205,7 @@ static void fxp_intr_body(struct fxp_softc *sc, static void fxp_init(void *xsc); static void fxp_tick(void *xsc); static void fxp_powerstate_d0(device_t dev); -static void fxp_start(struct ifnet *ifp); +static void fxp_start(struct ifnet *ifp, struct ifaltq_subque *); static void fxp_stop(struct fxp_softc *sc); static void fxp_release(device_t dev); static int fxp_ioctl(struct ifnet *ifp, u_long command, @@ -1046,11 +1046,12 @@ fxp_write_eeprom(struct fxp_softc *sc, u_short *data, int offset, int words) * Start packet transmission on the interface. */ static void -fxp_start(struct ifnet *ifp) +fxp_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct fxp_softc *sc = ifp->if_softc; struct fxp_cb_tx *txp; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); /* diff --git a/sys/dev/netif/igb/if_igb.c b/sys/dev/netif/igb/if_igb.c index 5cdc88f737..00c4ecd985 100644 --- a/sys/dev/netif/igb/if_igb.c +++ b/sys/dev/netif/igb/if_igb.c @@ -182,7 +182,7 @@ static void igb_media_status(struct ifnet *, struct ifmediareq *); static int igb_media_change(struct ifnet *); static void igb_timer(void *); static void igb_watchdog(struct ifnet *); -static void igb_start(struct ifnet *); +static void igb_start(struct ifnet *, struct ifaltq_subque *); #ifdef IFPOLL_ENABLE static void igb_npoll(struct ifnet *, struct ifpoll_info *); static void igb_npoll_rx(struct ifnet *, void *, int); @@ -647,7 +647,16 @@ igb_attach(device_t dev) ether_ifdetach(&sc->arpcom.ac_if); goto failed; } - ifq_set_cpuid(&sc->arpcom.ac_if.if_snd, sc->tx_rings[0].tx_intr_cpuid); + + for (i = 0; i < sc->tx_ring_cnt; ++i) { + struct ifaltq_subque *ifsq = + ifq_get_subq(&sc->arpcom.ac_if.if_snd, i); + struct igb_tx_ring *txr = &sc->tx_rings[i]; + + ifsq_set_cpuid(ifsq, txr->tx_intr_cpuid); + ifsq_set_priv(ifsq, txr); + txr->ifsq = ifsq; + } return 0; @@ -749,13 +758,15 @@ igb_resume(device_t dev) { struct igb_softc *sc = device_get_softc(dev); struct ifnet *ifp = &sc->arpcom.ac_if; + int i; ifnet_serialize_all(ifp); igb_init(sc); igb_get_mgmt(sc); - if_devstart(ifp); + for (i = 0; i < sc->tx_ring_cnt; ++i) + ifsq_devstart(sc->tx_rings[i].ifsq); ifnet_deserialize_all(ifp); @@ -955,7 +966,8 @@ igb_init(void *xsc) igb_set_promisc(sc); ifp->if_flags |= IFF_RUNNING; - ifq_clr_oactive(&ifp->if_snd); + for (i = 0; i < sc->tx_ring_cnt; ++i) + ifsq_clr_oactive(sc->tx_rings[i].ifsq); if (polling || sc->intr_type == PCI_INTR_TYPE_MSIX) sc->timer_cpuid = 0; /* XXX fixed */ @@ -1259,7 +1271,8 @@ igb_stop(struct igb_softc *sc) callout_stop(&sc->timer); ifp->if_flags &= ~IFF_RUNNING; - ifq_clr_oactive(&ifp->if_snd); + for (i = 0; i < sc->tx_ring_cnt; ++i) + ifsq_clr_oactive(sc->tx_rings[i].ifsq); ifp->if_timer = 0; e1000_reset_hw(&sc->hw); @@ -1992,7 +2005,7 @@ igb_txeof(struct igb_tx_ring *txr) * to tell the stack that it is OK to send packets. */ if (IGB_IS_NOT_OACTIVE(txr)) { - ifq_clr_oactive(&ifp->if_snd); + ifsq_clr_oactive(txr->ifsq); /* * We have enough TX descriptors, turn off @@ -2986,8 +2999,8 @@ igb_npoll_tx(struct ifnet *ifp, void *arg, int cycle __unused) ASSERT_SERIALIZED(&txr->tx_serialize); igb_txeof(txr); - if (!ifq_is_empty(&ifp->if_snd)) - if_devstart(ifp); + if (!ifsq_is_empty(txr->ifsq)) + ifsq_devstart(txr->ifsq); } static void @@ -3004,22 +3017,27 @@ static void igb_npoll(struct ifnet *ifp, struct ifpoll_info *info) { struct igb_softc *sc = ifp->if_softc; + int i; ASSERT_IFNET_SERIALIZED_ALL(ifp); if (info) { - struct igb_tx_ring *txr; - int i, off; + int off; info->ifpi_status.status_func = igb_npoll_status; info->ifpi_status.serializer = &sc->main_serialize; off = sc->tx_npoll_off; - KKASSERT(off < ncpus2); - txr = &sc->tx_rings[0]; - info->ifpi_tx[off].poll_func = igb_npoll_tx; - info->ifpi_tx[off].arg = txr; - info->ifpi_tx[off].serializer = &txr->tx_serialize; + for (i = 0; i < sc->tx_ring_cnt; ++i) { + struct igb_tx_ring *txr = &sc->tx_rings[i]; + int idx = i + off; + + KKASSERT(idx < ncpus2); + info->ifpi_tx[idx].poll_func = igb_npoll_tx; + info->ifpi_tx[idx].arg = txr; + info->ifpi_tx[idx].serializer = &txr->tx_serialize; + ifsq_set_cpuid(txr->ifsq, idx); + } off = sc->rx_npoll_off; for (i = 0; i < sc->rx_ring_cnt; ++i) { @@ -3038,7 +3056,6 @@ igb_npoll(struct ifnet *ifp, struct ifpoll_info *info) else igb_init(sc); } - ifq_set_cpuid(&ifp->if_snd, sc->tx_npoll_off); } else { if (ifp->if_flags & IFF_RUNNING) { if (sc->rx_ring_inuse == sc->rx_ring_cnt) @@ -3046,7 +3063,12 @@ igb_npoll(struct ifnet *ifp, struct ifpoll_info *info) else igb_init(sc); } - ifq_set_cpuid(&ifp->if_snd, sc->tx_rings[0].tx_intr_cpuid); + + for (i = 0; i < sc->tx_ring_cnt; ++i) { + struct igb_tx_ring *txr = &sc->tx_rings[i]; + + ifsq_set_cpuid(txr->ifsq, txr->tx_intr_cpuid); + } } } @@ -3067,7 +3089,7 @@ igb_intr(void *xsc) return; if (ifp->if_flags & IFF_RUNNING) { - struct igb_tx_ring *txr; + struct igb_tx_ring *txr = &sc->tx_rings[0]; int i; for (i = 0; i < sc->rx_ring_inuse; ++i) { @@ -3080,12 +3102,11 @@ igb_intr(void *xsc) } } - txr = &sc->tx_rings[0]; if (eicr & txr->tx_intr_mask) { lwkt_serialize_enter(&txr->tx_serialize); igb_txeof(txr); - if (!ifq_is_empty(&ifp->if_snd)) - if_devstart(ifp); + if (!ifsq_is_empty(txr->ifsq)) + ifsq_devstart(txr->ifsq); lwkt_serialize_exit(&txr->tx_serialize); } } @@ -3148,8 +3169,8 @@ igb_intr_shared(void *xsc) lwkt_serialize_enter(&txr->tx_serialize); igb_txeof(txr); - if (!ifq_is_empty(&ifp->if_snd)) - if_devstart(ifp); + if (!ifsq_is_empty(txr->ifsq)) + ifsq_devstart(txr->ifsq); lwkt_serialize_exit(&txr->tx_serialize); } } @@ -3303,35 +3324,36 @@ igb_encap(struct igb_tx_ring *txr, struct mbuf **m_headp, } static void -igb_start(struct ifnet *ifp) +igb_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct igb_softc *sc = ifp->if_softc; - struct igb_tx_ring *txr = &sc->tx_rings[0]; + struct igb_tx_ring *txr = ifsq_get_priv(ifsq); struct mbuf *m_head; int idx = -1, nsegs = 0; + KKASSERT(txr->ifsq == ifsq); ASSERT_SERIALIZED(&txr->tx_serialize); - if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd)) + if ((ifp->if_flags & IFF_RUNNING) == 0 || ifsq_is_oactive(ifsq)) return; if (!sc->link_active) { - ifq_purge(&ifp->if_snd); + ifsq_purge(ifsq); return; } if (!IGB_IS_NOT_OACTIVE(txr)) igb_txeof(txr); - while (!ifq_is_empty(&ifp->if_snd)) { + while (!ifsq_is_empty(ifsq)) { if (IGB_IS_OACTIVE(txr)) { - ifq_set_oactive(&ifp->if_snd); + ifsq_set_oactive(ifsq); /* Set watchdog on */ ifp->if_timer = 5; break; } - m_head = ifq_dequeue(&ifp->if_snd, NULL); + m_head = ifsq_dequeue(ifsq, NULL); if (m_head == NULL) break; @@ -3383,8 +3405,8 @@ igb_watchdog(struct ifnet *ifp) sc->watchdog_events++; igb_init(sc); - if (!ifq_is_empty(&ifp->if_snd)) - if_devstart(ifp); + if (!ifsq_is_empty(txr->ifsq)) + ifsq_devstart(txr->ifsq); } static void @@ -4319,13 +4341,12 @@ static void igb_msix_tx(void *arg) { struct igb_tx_ring *txr = arg; - struct ifnet *ifp = &txr->sc->arpcom.ac_if; ASSERT_SERIALIZED(&txr->tx_serialize); igb_txeof(txr); - if (!ifq_is_empty(&ifp->if_snd)) - if_devstart(ifp); + if (!ifsq_is_empty(txr->ifsq)) + ifsq_devstart(txr->ifsq); E1000_WRITE_REG(&txr->sc->hw, E1000_EIMS, txr->tx_intr_mask); } diff --git a/sys/dev/netif/igb/if_igb.h b/sys/dev/netif/igb/if_igb.h index 2864c120dd..72b22c01aa 100644 --- a/sys/dev/netif/igb/if_igb.h +++ b/sys/dev/netif/igb/if_igb.h @@ -214,6 +214,7 @@ struct igb_dma { struct igb_tx_ring { struct lwkt_serialize tx_serialize; struct igb_softc *sc; + struct ifaltq_subque *ifsq; uint32_t me; struct e1000_tx_desc *tx_base; int num_tx_desc; diff --git a/sys/dev/netif/iwi/if_iwi.c b/sys/dev/netif/iwi/if_iwi.c index 95e797ccfd..602a5ba7bc 100644 --- a/sys/dev/netif/iwi/if_iwi.c +++ b/sys/dev/netif/iwi/if_iwi.c @@ -169,7 +169,7 @@ static int iwi_tx_start(struct ifnet *, struct mbuf *, static int iwi_raw_xmit(struct ieee80211_node *, struct mbuf *, const struct ieee80211_bpf_params *); static void iwi_start_locked(struct ifnet *); -static void iwi_start(struct ifnet *); +static void iwi_start(struct ifnet *, struct ifaltq_subque *); static void iwi_watchdog(void *); static int iwi_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *ucred); static void iwi_stop_master(struct iwi_softc *); @@ -1958,8 +1958,9 @@ iwi_start_locked(struct ifnet *ifp) } static void -iwi_start(struct ifnet *ifp) +iwi_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); iwi_start_locked(ifp); } diff --git a/sys/dev/netif/iwl/iwl2100.c b/sys/dev/netif/iwl/iwl2100.c index 81bda7c4b9..e2348a404c 100644 --- a/sys/dev/netif/iwl/iwl2100.c +++ b/sys/dev/netif/iwl/iwl2100.c @@ -72,7 +72,7 @@ static void iwl2100_init(void *); static int iwl2100_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); -static void iwl2100_start(struct ifnet *); +static void iwl2100_start(struct ifnet *, struct ifaltq_subque *); static void iwl2100_watchdog(struct ifnet *); static int iwl2100_newstate(struct ieee80211com *, enum ieee80211_state, int); static int iwl2100_media_change(struct ifnet *); @@ -839,13 +839,14 @@ iwl2100_ioctl(struct ifnet *ifp, u_long cmd, caddr_t req, struct ucred *cr) } static void -iwl2100_start(struct ifnet *ifp) +iwl2100_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct iwl2100_softc *sc = ifp->if_softc; struct ieee80211com *ic = &sc->sc_ic; struct iwl2100_tx_ring *tr = &sc->sc_txring; int trans = 0; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); if (sc->sc_flags & IWL2100_F_DETACH) { diff --git a/sys/dev/netif/iwn/if_iwn.c b/sys/dev/netif/iwn/if_iwn.c index 37d5c520c4..fa62129a49 100644 --- a/sys/dev/netif/iwn/if_iwn.c +++ b/sys/dev/netif/iwn/if_iwn.c @@ -164,7 +164,7 @@ static int iwn_tx_data(struct iwn_softc *, struct mbuf *, struct ieee80211_node *, struct iwn_tx_ring *); static int iwn_raw_xmit(struct ieee80211_node *, struct mbuf *, const struct ieee80211_bpf_params *); -static void iwn_start(struct ifnet *); +static void iwn_start(struct ifnet *, struct ifaltq_subque *); static void iwn_start_locked(struct ifnet *); static void iwn_watchdog(struct iwn_softc *sc); static int iwn_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); @@ -3359,8 +3359,9 @@ iwn_raw_xmit(struct ieee80211_node *ni, struct mbuf *m, } static void -iwn_start(struct ifnet *ifp) +iwn_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); wlan_serialize_enter(); iwn_start_locked(ifp); wlan_serialize_exit(); diff --git a/sys/dev/netif/ixgbe/ixgbe.c b/sys/dev/netif/ixgbe/ixgbe.c index f6a1022c81..8b345777d1 100644 --- a/sys/dev/netif/ixgbe/ixgbe.c +++ b/sys/dev/netif/ixgbe/ixgbe.c @@ -102,7 +102,7 @@ static int ixgbe_probe(device_t); static int ixgbe_attach(device_t); static int ixgbe_detach(device_t); static int ixgbe_shutdown(device_t); -static void ixgbe_start(struct ifnet *); +static void ixgbe_start(struct ifnet *, struct ifaltq_subque *); static void ixgbe_start_locked(struct tx_ring *, struct ifnet *); #if 0 /* __FreeBSD_version >= 800000 */ static int ixgbe_mq_start(struct ifnet *, struct mbuf *); @@ -731,11 +731,13 @@ ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp) * not be used with multiqueue tx enabled. */ static void -ixgbe_start(struct ifnet *ifp) +ixgbe_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct adapter *adapter = ifp->if_softc; struct tx_ring *txr = adapter->tx_rings; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if (ifp->if_flags & IFF_RUNNING) { IXGBE_TX_LOCK(txr); ixgbe_start_locked(txr, ifp); diff --git a/sys/dev/netif/jme/if_jme.c b/sys/dev/netif/jme/if_jme.c index 03b386d640..629bb687f1 100644 --- a/sys/dev/netif/jme/if_jme.c +++ b/sys/dev/netif/jme/if_jme.c @@ -102,7 +102,7 @@ static void jme_miibus_statchg(device_t); static void jme_init(void *); static int jme_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); -static void jme_start(struct ifnet *); +static void jme_start(struct ifnet *, struct ifaltq_subque *); static void jme_watchdog(struct ifnet *); static void jme_mediastatus(struct ifnet *, struct ifmediareq *); static int jme_mediachange(struct ifnet *); @@ -1797,13 +1797,14 @@ fail: } static void -jme_start(struct ifnet *ifp) +jme_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct jme_softc *sc = ifp->if_softc; struct jme_txdata *tdata = &sc->jme_cdata.jme_tx_data; struct mbuf *m_head; int enq = 0; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(&tdata->jme_tx_serialize); if (!sc->jme_has_link) { diff --git a/sys/dev/netif/kue/if_kue.c b/sys/dev/netif/kue/if_kue.c index 416250d4bb..ce2fac28dd 100644 --- a/sys/dev/netif/kue/if_kue.c +++ b/sys/dev/netif/kue/if_kue.c @@ -128,7 +128,7 @@ static int kue_newbuf(struct kue_softc *, struct kue_chain *, struct mbuf *); static int kue_encap(struct kue_softc *, struct mbuf *, int); static void kue_rxeof(usbd_xfer_handle, usbd_private_handle, usbd_status); static void kue_txeof(usbd_xfer_handle, usbd_private_handle, usbd_status); -static void kue_start(struct ifnet *); +static void kue_start(struct ifnet *, struct ifaltq_subque *); static void kue_rxstart(struct ifnet *); static int kue_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void kue_init(void *); @@ -786,11 +786,13 @@ kue_encap(struct kue_softc *sc, struct mbuf *m, int idx) } static void -kue_start(struct ifnet *ifp) +kue_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct kue_softc *sc; struct mbuf *m_head = NULL; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + sc = ifp->if_softc; KUE_LOCK(sc); diff --git a/sys/dev/netif/lge/if_lge.c b/sys/dev/netif/lge/if_lge.c index d7e9305396..38c26d779f 100644 --- a/sys/dev/netif/lge/if_lge.c +++ b/sys/dev/netif/lge/if_lge.c @@ -141,7 +141,7 @@ static void lge_txeof(struct lge_softc *); static void lge_intr(void *); static void lge_tick(void *); static void lge_tick_serialized(void *); -static void lge_start(struct ifnet *); +static void lge_start(struct ifnet *, struct ifaltq_subque *); static int lge_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void lge_init(void *); static void lge_stop(struct lge_softc *); @@ -1119,13 +1119,15 @@ lge_encap(struct lge_softc *sc, struct mbuf *m_head, uint32_t *txidx) */ static void -lge_start(struct ifnet *ifp) +lge_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct lge_softc *sc = ifp->if_softc; struct mbuf *m_head = NULL, *m_defragged; uint32_t idx; int need_timer; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if (!sc->lge_link) { ifq_purge(&ifp->if_snd); return; diff --git a/sys/dev/netif/lgue/if_lgue.c b/sys/dev/netif/lgue/if_lgue.c index 0721cc7d1b..d62be4112a 100644 --- a/sys/dev/netif/lgue/if_lgue.c +++ b/sys/dev/netif/lgue/if_lgue.c @@ -47,7 +47,7 @@ static int lgue_match(device_t); static int lgue_attach(device_t); static int lgue_detach(device_t); -static void lgue_start(struct ifnet *); +static void lgue_start(struct ifnet *, struct ifaltq_subque *); static void lgue_stop(struct lgue_softc *); static void lgue_init(void *); static void lgue_watchdog(struct ifnet *); @@ -494,11 +494,13 @@ lgue_txeof(usbd_xfer_handle xfer, usbd_private_handle priv, usbd_status status) * Start transfer */ static void -lgue_start(struct ifnet *ifp) +lgue_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct lgue_softc *sc; struct mbuf *m_head; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + sc = ifp->if_softc; if (sc->lgue_dying) return; diff --git a/sys/dev/netif/lnc/lance.c b/sys/dev/netif/lnc/lance.c index 2f80de67b8..0d24774e92 100644 --- a/sys/dev/netif/lnc/lance.c +++ b/sys/dev/netif/lnc/lance.c @@ -96,7 +96,7 @@ devclass_t le_devclass; -static void lance_start(struct ifnet *); +static void lance_start(struct ifnet *, struct ifaltq_subque *); static void lance_init(void *); static void lance_watchdog(struct ifnet *); static int lance_mediachange(struct ifnet *); @@ -214,10 +214,11 @@ lance_resume(struct lance_softc *sc) } static void -lance_start(struct ifnet *ifp) +lance_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct lance_softc *sc = ifp->if_softc; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); (*sc->sc_start_locked)(sc); } diff --git a/sys/dev/netif/msk/if_msk.c b/sys/dev/netif/msk/if_msk.c index 43c00010cf..24aa34335e 100644 --- a/sys/dev/netif/msk/if_msk.c +++ b/sys/dev/netif/msk/if_msk.c @@ -256,7 +256,7 @@ static void msk_miibus_statchg(device_t); static void msk_init(void *); static int msk_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); -static void msk_start(struct ifnet *); +static void msk_start(struct ifnet *, struct ifaltq_subque *); static void msk_watchdog(struct ifnet *); static int msk_mediachange(struct ifnet *); static void msk_mediastatus(struct ifnet *, struct ifmediareq *); @@ -2596,7 +2596,7 @@ msk_encap(struct msk_if_softc *sc_if, struct mbuf **m_head) } static void -msk_start(struct ifnet *ifp) +msk_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct msk_if_softc *sc_if; struct mbuf *m_head; @@ -2604,6 +2604,7 @@ msk_start(struct ifnet *ifp) sc_if = ifp->if_softc; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); if (!sc_if->msk_link) { diff --git a/sys/dev/netif/mxge/if_mxge.c b/sys/dev/netif/mxge/if_mxge.c index e26bae92df..37baddacc6 100644 --- a/sys/dev/netif/mxge/if_mxge.c +++ b/sys/dev/netif/mxge/if_mxge.c @@ -2195,11 +2195,12 @@ mxge_start_locked(struct mxge_slice_state *ss) } static void -mxge_start(struct ifnet *ifp) +mxge_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { mxge_softc_t *sc = ifp->if_softc; struct mxge_slice_state *ss; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(sc->ifp->if_serializer); /* only use the first slice for now */ ss = &sc->ss[0]; diff --git a/sys/dev/netif/my/if_my.c b/sys/dev/netif/my/if_my.c index 69e5b61ba0..53e1b139e7 100644 --- a/sys/dev/netif/my/if_my.c +++ b/sys/dev/netif/my/if_my.c @@ -113,7 +113,7 @@ static void my_rxeof(struct my_softc *); static void my_txeof(struct my_softc *); static void my_txeoc(struct my_softc *); static void my_intr(void *); -static void my_start(struct ifnet *); +static void my_start(struct ifnet *, struct ifaltq_subque *); static int my_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void my_init(void *); static void my_stop(struct my_softc *); @@ -1327,12 +1327,13 @@ my_encap(struct my_softc * sc, struct my_chain * c, struct mbuf * m_head) * physical addresses. */ static void -my_start(struct ifnet * ifp) +my_start(struct ifnet * ifp, struct ifaltq_subque *ifsq) { struct my_softc *sc = ifp->if_softc; struct mbuf *m_head = NULL; struct my_chain *cur_tx = NULL, *start_tx; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); crit_enter(); if (sc->my_autoneg) { diff --git a/sys/dev/netif/ndis/if_ndis.c b/sys/dev/netif/ndis/if_ndis.c index a371b42bf9..8a2d303174 100644 --- a/sys/dev/netif/ndis/if_ndis.c +++ b/sys/dev/netif/ndis/if_ndis.c @@ -170,7 +170,7 @@ static int ndis_raw_xmit (struct ieee80211_node *, struct mbuf *, const struct ieee80211_bpf_params *); static void ndis_update_mcast (struct ifnet *ifp); static void ndis_update_promisc (struct ifnet *ifp); -static void ndis_start (struct ifnet *); +static void ndis_start (struct ifnet *, struct ifaltq_subque *); static void ndis_starttask (device_object *, void *); static void ndis_resettask (device_object *, void *); static void ndis_inputtask (device_object *, void *); @@ -1777,7 +1777,7 @@ ndis_starttask(device_object *d, void *arg) ifp = arg; if (!ifq_is_empty(&ifp->if_snd)) - ndis_start(ifp); + if_devstart(ifp); } /* @@ -1793,7 +1793,7 @@ ndis_starttask(device_object *d, void *arg) * will do the mapping themselves on a buffer by buffer basis. */ static void -ndis_start(struct ifnet *ifp) +ndis_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct ndis_softc *sc; struct mbuf *m = NULL; @@ -1801,6 +1801,8 @@ ndis_start(struct ifnet *ifp) ndis_tcpip_csum *csum; int pcnt = 0, status; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + sc = ifp->if_softc; NDIS_LOCK(sc); diff --git a/sys/dev/netif/nfe/if_nfe.c b/sys/dev/netif/nfe/if_nfe.c index 73473327dd..ab11def666 100644 --- a/sys/dev/netif/nfe/if_nfe.c +++ b/sys/dev/netif/nfe/if_nfe.c @@ -118,7 +118,7 @@ static int nfe_rxeof(struct nfe_softc *); static int nfe_txeof(struct nfe_softc *, int); static int nfe_encap(struct nfe_softc *, struct nfe_tx_ring *, struct mbuf *); -static void nfe_start(struct ifnet *); +static void nfe_start(struct ifnet *, struct ifaltq_subque *); static void nfe_watchdog(struct ifnet *); static void nfe_init(void *); static void nfe_stop(struct nfe_softc *); @@ -1356,13 +1356,14 @@ back: } static void -nfe_start(struct ifnet *ifp) +nfe_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct nfe_softc *sc = ifp->if_softc; struct nfe_tx_ring *ring = &sc->txq; int count = 0, oactive = 0; struct mbuf *m0; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd)) diff --git a/sys/dev/netif/nge/if_nge.c b/sys/dev/netif/nge/if_nge.c index c80dfc903a..b8a932a783 100644 --- a/sys/dev/netif/nge/if_nge.c +++ b/sys/dev/netif/nge/if_nge.c @@ -161,7 +161,7 @@ static void nge_rxeof(struct nge_softc *); static void nge_txeof(struct nge_softc *); static void nge_intr(void *); static void nge_tick(void *); -static void nge_start(struct ifnet *); +static void nge_start(struct ifnet *, struct ifaltq_subque *); static int nge_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void nge_init(void *); static void nge_stop(struct nge_softc *); @@ -1623,13 +1623,15 @@ nge_encap(struct nge_softc *sc, struct mbuf *m_head, uint32_t *txidx) */ static void -nge_start(struct ifnet *ifp) +nge_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct nge_softc *sc = ifp->if_softc; struct mbuf *m_head = NULL, *m_defragged; uint32_t idx; int need_trans; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if (!sc->nge_link) { ifq_purge(&ifp->if_snd); return; diff --git a/sys/dev/netif/pcn/if_pcn.c b/sys/dev/netif/pcn/if_pcn.c index b1f5589b6e..5c9d215599 100644 --- a/sys/dev/netif/pcn/if_pcn.c +++ b/sys/dev/netif/pcn/if_pcn.c @@ -125,7 +125,7 @@ static void pcn_rxeof (struct pcn_softc *); static void pcn_txeof (struct pcn_softc *); static void pcn_intr (void *); static void pcn_tick (void *); -static void pcn_start (struct ifnet *); +static void pcn_start (struct ifnet *, struct ifaltq_subque *); static int pcn_ioctl (struct ifnet *, u_long, caddr_t, struct ucred *); static void pcn_init (void *); @@ -991,13 +991,15 @@ pcn_encap(struct pcn_softc *sc, struct mbuf *m_head, u_int32_t *txidx) * physical addresses. */ static void -pcn_start(struct ifnet *ifp) +pcn_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct pcn_softc *sc; struct mbuf *m_head = NULL, *m_defragged; u_int32_t idx; int need_trans; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + sc = ifp->if_softc; if (!sc->pcn_link) { diff --git a/sys/dev/netif/ral/rt2560.c b/sys/dev/netif/ral/rt2560.c index b027d993d8..a3a58ed4d0 100644 --- a/sys/dev/netif/ral/rt2560.c +++ b/sys/dev/netif/ral/rt2560.c @@ -125,7 +125,7 @@ static int rt2560_tx_mgt(struct rt2560_softc *, struct mbuf *, static int rt2560_tx_data(struct rt2560_softc *, struct mbuf *, struct ieee80211_node *); static void rt2560_start_locked(struct ifnet *); -static void rt2560_start(struct ifnet *); +static void rt2560_start(struct ifnet *, struct ifaltq_subque *); static void rt2560_watchdog_callout(void *); static int rt2560_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); @@ -1937,8 +1937,9 @@ rt2560_start_locked(struct ifnet *ifp) } static void -rt2560_start(struct ifnet *ifp) +rt2560_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); rt2560_start_locked(ifp); } diff --git a/sys/dev/netif/ral/rt2661.c b/sys/dev/netif/ral/rt2661.c index cf7ac86495..99711b3249 100644 --- a/sys/dev/netif/ral/rt2661.c +++ b/sys/dev/netif/ral/rt2661.c @@ -119,7 +119,7 @@ static int rt2661_tx_data(struct rt2661_softc *, struct mbuf *, static int rt2661_tx_mgt(struct rt2661_softc *, struct mbuf *, struct ieee80211_node *); static void rt2661_start_locked(struct ifnet *); -static void rt2661_start(struct ifnet *); +static void rt2661_start(struct ifnet *, struct ifaltq_subque *); static int rt2661_raw_xmit(struct ieee80211_node *, struct mbuf *, const struct ieee80211_bpf_params *); static void rt2661_watchdog_callout(void *); @@ -1642,8 +1642,9 @@ rt2661_start_locked(struct ifnet *ifp) } static void -rt2661_start(struct ifnet *ifp) +rt2661_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); rt2661_start_locked(ifp); } diff --git a/sys/dev/netif/re/if_re.c b/sys/dev/netif/re/if_re.c index 64def44055..6999abd23f 100644 --- a/sys/dev/netif/re/if_re.c +++ b/sys/dev/netif/re/if_re.c @@ -304,7 +304,7 @@ static void re_intr(void *); static void re_tick(void *); static void re_tick_serialized(void *); -static void re_start(struct ifnet *); +static void re_start(struct ifnet *, struct ifaltq_subque *); static int re_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void re_init(void *); static void re_stop(struct re_softc *); @@ -2437,12 +2437,13 @@ back: */ static void -re_start(struct ifnet *ifp) +re_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct re_softc *sc = ifp->if_softc; struct mbuf *m_head; int idx, need_trans, oactive, error; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); if ((sc->re_flags & RE_F_LINKED) == 0) { diff --git a/sys/dev/netif/rl/if_rl.c b/sys/dev/netif/rl/if_rl.c index 4d449f3883..d85568d9a3 100644 --- a/sys/dev/netif/rl/if_rl.c +++ b/sys/dev/netif/rl/if_rl.c @@ -179,7 +179,7 @@ static void rl_rxeof(struct rl_softc *); static void rl_txeof(struct rl_softc *); static void rl_intr(void *); static void rl_tick(void *); -static void rl_start(struct ifnet *); +static void rl_start(struct ifnet *, struct ifaltq_subque *); static int rl_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void rl_init(void *); static void rl_stop (struct rl_softc *); @@ -1340,11 +1340,13 @@ rl_encap(struct rl_softc *sc, struct mbuf *m_head) */ static void -rl_start(struct ifnet *ifp) +rl_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct rl_softc *sc = ifp->if_softc; struct mbuf *m_head = NULL; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd)) return; diff --git a/sys/dev/netif/rtw/rtw.c b/sys/dev/netif/rtw/rtw.c index 24f4900d6c..d3226741c2 100644 --- a/sys/dev/netif/rtw/rtw.c +++ b/sys/dev/netif/rtw/rtw.c @@ -165,7 +165,7 @@ static void rtw_led_fastblink(void *); static void rtw_led_set(struct rtw_softc *); static void rtw_init(void *); -static void rtw_start(struct ifnet *); +static void rtw_start(struct ifnet *, struct ifaltq_subque *); static int rtw_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void rtw_watchdog(struct ifnet *); static void rtw_intr(void *); @@ -1684,8 +1684,10 @@ rtw_intr_tx(struct rtw_softc *sc, uint16_t isr) rtw_collect_txring(sc, &sc->sc_txsoft_blk[pri], &sc->sc_txdesc_blk[pri], 0); } - if (isr) - rtw_start(&sc->sc_ic.ic_if); + if (isr) { + rtw_start(&sc->sc_ic.ic_if, + ifq_get_subq_default(&sc->sc_ic.ic_if.if_snd)); + } } static __inline struct mbuf * @@ -1758,7 +1760,7 @@ rtw_intr_beacon(struct rtw_softc *sc, uint16_t isr) IF_ENQUEUE(&sc->sc_beaconq, m); - rtw_start(&ic->ic_if); + rtw_start(&ic->ic_if, ifq_get_subq_default(&ic->ic_if.if_snd)); } } @@ -3130,7 +3132,7 @@ rtw_print_txdesc(struct rtw_softc *sc, const char *action, #endif /* RTW_DEBUG */ static void -rtw_start(struct ifnet *ifp) +rtw_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct rtw_softc *sc = ifp->if_softc; struct ieee80211com *ic = &sc->sc_ic; @@ -3139,6 +3141,7 @@ rtw_start(struct ifnet *ifp) struct mbuf *m0; uint32_t proto_ctl0; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); DPRINTF(sc, RTW_DEBUG_XMIT, ("%s: enter %s\n", ifp->if_xname, __func__)); @@ -3429,7 +3432,7 @@ rtw_watchdog(struct ifnet *ifp) rtw_txdesc_blk_reset_all(sc); rtw_io_enable(sc, RTW_CR_TE, 1); rtw_txring_fixup(sc); - rtw_start(ifp); + rtw_start(ifp, ifq_get_subq_default(&ifp->if_snd)); } ieee80211_watchdog(&sc->sc_ic); } diff --git a/sys/dev/netif/rue/if_rue.c b/sys/dev/netif/rue/if_rue.c index ce98e906a0..0d1893a3b5 100644 --- a/sys/dev/netif/rue/if_rue.c +++ b/sys/dev/netif/rue/if_rue.c @@ -134,7 +134,7 @@ static void rue_rxeof(usbd_xfer_handle, usbd_private_handle, usbd_status); static void rue_txeof(usbd_xfer_handle, usbd_private_handle, usbd_status); static void rue_tick(void *); static void rue_rxstart(struct ifnet *); -static void rue_start(struct ifnet *); +static void rue_start(struct ifnet *, struct ifaltq_subque *); static int rue_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void rue_init(void *); static void rue_stop(struct rue_softc *); @@ -979,11 +979,13 @@ rue_encap(struct rue_softc *sc, struct mbuf *m, int idx) } static void -rue_start(struct ifnet *ifp) +rue_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct rue_softc *sc = ifp->if_softc; struct mbuf *m_head = NULL; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + RUE_LOCK(sc); if (!sc->rue_link) { diff --git a/sys/dev/netif/rum/if_rum.c b/sys/dev/netif/rum/if_rum.c index 1381a59c1f..93cf0c1be0 100644 --- a/sys/dev/netif/rum/if_rum.c +++ b/sys/dev/netif/rum/if_rum.c @@ -137,7 +137,7 @@ static void rum_setup_tx_desc(struct rum_softc *, int); static int rum_tx_data(struct rum_softc *, struct mbuf *, struct ieee80211_node *); -static void rum_start(struct ifnet *); +static void rum_start(struct ifnet *, struct ifaltq_subque *); static void rum_watchdog(struct ifnet *); static int rum_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); @@ -1117,11 +1117,12 @@ rum_tx_data(struct rum_softc *sc, struct mbuf *m0, struct ieee80211_node *ni) } static void -rum_start(struct ifnet *ifp) +rum_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct rum_softc *sc = ifp->if_softc; struct ieee80211com *ic = &sc->sc_ic; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); if (sc->sc_stopped) { diff --git a/sys/dev/netif/sbni/if_sbni.c b/sys/dev/netif/sbni/if_sbni.c index 74ae2488b7..ebd3cd1f65 100644 --- a/sys/dev/netif/sbni/if_sbni.c +++ b/sys/dev/netif/sbni/if_sbni.c @@ -88,7 +88,7 @@ #endif static void sbni_init(void *); -static void sbni_start(struct ifnet *); +static void sbni_start(struct ifnet *, struct ifaltq_subque *); static int sbni_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void sbni_watchdog(struct ifnet *); static void sbni_stop(struct sbni_softc *); @@ -283,9 +283,11 @@ sbni_init(void *xsc) static void -sbni_start(struct ifnet *ifp) +sbni_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct sbni_softc *sc = ifp->if_softc; + + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); if (sc->tx_frameno == 0) prepare_to_send(sc); } diff --git a/sys/dev/netif/sbsh/if_sbsh.c b/sys/dev/netif/sbsh/if_sbsh.c index 2f91f82390..597884b761 100644 --- a/sys/dev/netif/sbsh/if_sbsh.c +++ b/sys/dev/netif/sbsh/if_sbsh.c @@ -159,7 +159,7 @@ static int sbsh_suspend(device_t); static int sbsh_resume(device_t); static void sbsh_watchdog(struct ifnet *); -static void sbsh_start(struct ifnet *); +static void sbsh_start(struct ifnet *, struct ifaltq_subque *); static void sbsh_init(void *); static void sbsh_stop(struct sbsh_softc *); static void init_card(struct sbsh_softc *); @@ -311,10 +311,12 @@ sbsh_detach(device_t dev) static void -sbsh_start(struct ifnet *ifp) +sbsh_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct sbsh_softc *sc = ifp->if_softc; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if (sc->state == ACTIVE) start_xmit_frames(ifp->if_softc); } diff --git a/sys/dev/netif/sf/if_sf.c b/sys/dev/netif/sf/if_sf.c index f3b02cd199..998f37ec1f 100644 --- a/sys/dev/netif/sf/if_sf.c +++ b/sys/dev/netif/sf/if_sf.c @@ -136,7 +136,7 @@ static void sf_txeof (struct sf_softc *); static int sf_encap (struct sf_softc *, struct sf_tx_bufdesc_type0 *, struct mbuf *); -static void sf_start (struct ifnet *); +static void sf_start (struct ifnet *, struct ifaltq_subque *); static int sf_ioctl (struct ifnet *, u_long, caddr_t, struct ucred *); static void sf_init (void *); @@ -1235,13 +1235,15 @@ sf_encap(struct sf_softc *sc, struct sf_tx_bufdesc_type0 *c, } static void -sf_start(struct ifnet *ifp) +sf_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct sf_softc *sc; struct sf_tx_bufdesc_type0 *cur_tx = NULL; struct mbuf *m_head = NULL, *m_defragged; int i, txprod, need_trans = 0; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + sc = ifp->if_softc; if (!sc->sf_link) { diff --git a/sys/dev/netif/sis/if_sis.c b/sys/dev/netif/sis/if_sis.c index 50d6fd2ab7..ff152a677c 100644 --- a/sys/dev/netif/sis/if_sis.c +++ b/sys/dev/netif/sis/if_sis.c @@ -121,7 +121,7 @@ static void sis_rxeoc(struct sis_softc *); static void sis_txeof(struct sis_softc *); static void sis_intr(void *); static void sis_tick(void *); -static void sis_start(struct ifnet *); +static void sis_start(struct ifnet *, struct ifaltq_subque *); static int sis_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void sis_init(void *); static void sis_stop(struct sis_softc *); @@ -1690,12 +1690,14 @@ sis_encap(struct sis_softc *sc, struct mbuf **m_head, uint32_t *txidx) */ static void -sis_start(struct ifnet *ifp) +sis_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct sis_softc *sc = ifp->if_softc; int need_trans, error; uint32_t idx; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if (!sc->sis_link) { ifq_purge(&ifp->if_snd); return; diff --git a/sys/dev/netif/sk/if_sk.c b/sys/dev/netif/sk/if_sk.c index 97d0d1ff21..8167fdc622 100644 --- a/sys/dev/netif/sk/if_sk.c +++ b/sys/dev/netif/sk/if_sk.c @@ -190,7 +190,7 @@ static void sk_intr_yukon(struct sk_if_softc *); static void sk_rxeof(struct sk_if_softc *); static void sk_txeof(struct sk_if_softc *); static int sk_encap(struct sk_if_softc *, struct mbuf **, uint32_t *); -static void sk_start(struct ifnet *); +static void sk_start(struct ifnet *, struct ifaltq_subque *); static int sk_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void sk_init(void *); static void sk_init_xmac(struct sk_if_softc *); @@ -1698,13 +1698,14 @@ sk_encap(struct sk_if_softc *sc_if, struct mbuf **m_head0, uint32_t *txidx) } static void -sk_start(struct ifnet *ifp) +sk_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct sk_if_softc *sc_if = ifp->if_softc; struct sk_softc *sc = sc_if->sk_softc; uint32_t idx = sc_if->sk_cdata.sk_tx_prod; int trans = 0; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); DPRINTFN(2, ("sk_start\n")); if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd)) diff --git a/sys/dev/netif/sln/if_sln.c b/sys/dev/netif/sln/if_sln.c index 3ad3fb9b7d..7029186ec4 100644 --- a/sys/dev/netif/sln/if_sln.c +++ b/sys/dev/netif/sln/if_sln.c @@ -96,7 +96,7 @@ static int sln_resume(device_t); static void sln_reset(struct sln_softc *); static void sln_init(void *); -static void sln_tx(struct ifnet *); +static void sln_tx(struct ifnet *, struct ifaltq_subque *); static void sln_rx(struct sln_softc *); static void sln_tx_intr(struct sln_softc *); static void sln_media_intr(struct sln_softc *); @@ -727,13 +727,14 @@ sln_init(void *x) /* Transmit Packet */ static void -sln_tx(struct ifnet *ifp) +sln_tx(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct sln_softc *sc = ifp->if_softc; struct mbuf *m_head = NULL; struct mbuf *m_new = NULL; int entry; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); if (!sc->connect) { diff --git a/sys/dev/netif/sn/if_sn.c b/sys/dev/netif/sn/if_sn.c index efe31fb1c4..09d5e82be1 100644 --- a/sys/dev/netif/sn/if_sn.c +++ b/sys/dev/netif/sn/if_sn.c @@ -130,7 +130,7 @@ static void snresume(struct ifnet *); void sninit(void *); void snread(struct ifnet *); void snreset(struct sn_softc *); -void snstart(struct ifnet *); +void snstart(struct ifnet *, struct ifaltq_subque *); void snstop(struct sn_softc *); void snwatchdog(struct ifnet *); @@ -334,7 +334,7 @@ sninit(void *xsc) void -snstart(struct ifnet *ifp) +snstart(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct sn_softc *sc = ifp->if_softc; u_int len; @@ -347,6 +347,8 @@ snstart(struct ifnet *ifp) u_char packet_no; int time_out; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd)) return; diff --git a/sys/dev/netif/sr/if_sr.c b/sys/dev/netif/sr/if_sr.c index d1accd4180..a7040241a6 100644 --- a/sys/dev/netif/sr/if_sr.c +++ b/sys/dev/netif/sr/if_sr.c @@ -227,7 +227,7 @@ MODULE_DEPEND(ng_sync_sr, netgraph, NG_ABI_VERSION, NG_ABI_VERSION, NG_ABI_VERSI static void srintr(void *arg); static void sr_xmit(struct sr_softc *sc); #ifndef NETGRAPH -static void srstart(struct ifnet *ifp); +static void srstart(struct ifnet *ifp, struct ifaltq_subque *); static int srioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *); static void srwatchdog(struct ifnet *ifp); @@ -716,7 +716,7 @@ sr_xmit(struct sr_softc *sc) */ #ifndef NETGRAPH static void -srstart(struct ifnet *ifp) +srstart(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct sr_softc *sc; /* channel control structure */ #else @@ -1013,7 +1013,7 @@ srioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cr) * Interface should be up -- start it. */ sr_up(sc); - srstart(ifp); + srstart(ifp, ifq_get_subq_default(&ifp->if_snd)); /* * XXX Clear the IFF_UP flag so that the link will only go @@ -1102,7 +1102,8 @@ srwatchdog(struct sr_softc *sc) sr_xmit(sc); #ifndef NETGRAPH - srstart(ifp); /* restart transmitter */ + srstart(ifp, ifq_get_subq_default(&ifp->if_snd)); + /* restart transmitter */ #else srstart(sc); /* restart transmitter */ #endif /* NETGRAPH */ @@ -2399,7 +2400,8 @@ sr_dmac_intr(struct sr_hardc *hc, u_char isr1) if (dotxstart & 0x0C) { /* TX initiation enabled? */ sc = &hc->sc[mch]; #ifndef NETGRAPH - srstart(&sc->ifsppp.pp_if); + srstart(&sc->ifsppp.pp_if, + ifq_get_subq_default(&sc->ifsppp.pp_if.if_snd)); #else srstart(sc); #endif /* NETGRAPH */ diff --git a/sys/dev/netif/ste/if_ste.c b/sys/dev/netif/ste/if_ste.c index 02717c7349..c182228b43 100644 --- a/sys/dev/netif/ste/if_ste.c +++ b/sys/dev/netif/ste/if_ste.c @@ -98,7 +98,7 @@ static int ste_ioctl (struct ifnet *, u_long, caddr_t, struct ucred *); static int ste_encap (struct ste_softc *, struct ste_chain *, struct mbuf *); -static void ste_start (struct ifnet *); +static void ste_start (struct ifnet *, struct ifaltq_subque *); static void ste_watchdog (struct ifnet *); static void ste_shutdown (device_t); static int ste_newbuf (struct ste_softc *, @@ -1405,13 +1405,15 @@ encap_retry: } static void -ste_start(struct ifnet *ifp) +ste_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct ste_softc *sc; struct mbuf *m_head = NULL; struct ste_chain *cur_tx = NULL; int idx; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + sc = ifp->if_softc; if (!sc->ste_link) { diff --git a/sys/dev/netif/stge/if_stge.c b/sys/dev/netif/stge/if_stge.c index 5ffeddc920..73ea22c9a3 100644 --- a/sys/dev/netif/stge/if_stge.c +++ b/sys/dev/netif/stge/if_stge.c @@ -131,7 +131,7 @@ static int stge_suspend(device_t); static int stge_resume(device_t); static int stge_encap(struct stge_softc *, struct mbuf **); -static void stge_start(struct ifnet *); +static void stge_start(struct ifnet *, struct ifaltq_subque *); static void stge_watchdog(struct ifnet *); static int stge_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void stge_init(void *); @@ -1183,7 +1183,7 @@ stge_encap(struct stge_softc *sc, struct mbuf **m_head) * Start packet transmission on the interface. */ static void -stge_start(struct ifnet *ifp) +stge_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct stge_softc *sc; struct mbuf *m_head; @@ -1191,6 +1191,7 @@ stge_start(struct ifnet *ifp) sc = ifp->if_softc; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd)) diff --git a/sys/dev/netif/ti/if_ti.c b/sys/dev/netif/ti/if_ti.c index a4b7407443..d75d810ce6 100644 --- a/sys/dev/netif/ti/if_ti.c +++ b/sys/dev/netif/ti/if_ti.c @@ -164,7 +164,7 @@ static void ti_stats_update(struct ti_softc *); static int ti_encap(struct ti_softc *, struct mbuf *, uint32_t *); static void ti_intr(void *); -static void ti_start(struct ifnet *); +static void ti_start(struct ifnet *, struct ifaltq_subque *); static int ti_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void ti_init(void *); static void ti_init2(struct ti_softc *); @@ -1904,13 +1904,15 @@ ti_encap(struct ti_softc *sc, struct mbuf *m_head, uint32_t *txidx) * to the mbuf data regions directly in the transmit descriptors. */ static void -ti_start(struct ifnet *ifp) +ti_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct ti_softc *sc = ifp->if_softc; struct mbuf *m_head = NULL; uint32_t prodidx = 0; int need_trans; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + prodidx = CSR_READ_4(sc, TI_MB_SENDPROD_IDX); need_trans = 0; diff --git a/sys/dev/netif/tl/if_tl.c b/sys/dev/netif/tl/if_tl.c index 6d770ea62e..176dd35495 100644 --- a/sys/dev/netif/tl/if_tl.c +++ b/sys/dev/netif/tl/if_tl.c @@ -275,7 +275,7 @@ static int tl_encap (struct tl_softc *, struct tl_chain *, struct mbuf *); static void tl_intr (void *); -static void tl_start (struct ifnet *); +static void tl_start (struct ifnet *, struct ifaltq_subque *); static int tl_ioctl (struct ifnet *, u_long, caddr_t, struct ucred *); static void tl_init (void *); @@ -1820,13 +1820,15 @@ tl_encap(struct tl_softc *sc, struct tl_chain *c, struct mbuf *m_head) * physical addresses. */ static void -tl_start(struct ifnet *ifp) +tl_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct tl_softc *sc; struct mbuf *m_head = NULL; u_int32_t cmd; struct tl_chain *prev = NULL, *cur_tx = NULL, *start_tx; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + sc = ifp->if_softc; /* diff --git a/sys/dev/netif/tx/if_tx.c b/sys/dev/netif/tx/if_tx.c index e175a25393..d5c0ae9ce0 100644 --- a/sys/dev/netif/tx/if_tx.c +++ b/sys/dev/netif/tx/if_tx.c @@ -84,7 +84,7 @@ static int epic_ifioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void epic_intr(void *); static void epic_tx_underrun(epic_softc_t *); static int epic_common_attach(epic_softc_t *); -static void epic_ifstart(struct ifnet *); +static void epic_ifstart(struct ifnet *, struct ifaltq_subque *); static void epic_ifwatchdog(struct ifnet *); static void epic_stats_update(void *); static int epic_init(epic_softc_t *); @@ -474,7 +474,7 @@ epic_common_attach(epic_softc_t *sc) * or queue become empty. */ static void -epic_ifstart(struct ifnet *ifp) +epic_ifstart(struct ifnet *ifp, struct ifaltq_subque *ifsq) { epic_softc_t *sc = ifp->if_softc; struct epic_tx_buffer *buf; @@ -484,6 +484,8 @@ epic_ifstart(struct ifnet *ifp) struct mbuf *m; int i; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + while (sc->pending_txs < TX_RING_SIZE) { buf = sc->tx_buffer + sc->cur_tx; desc = sc->tx_desc + sc->cur_tx; diff --git a/sys/dev/netif/txp/if_txp.c b/sys/dev/netif/txp/if_txp.c index 5ed24a4246..8503f00faa 100644 --- a/sys/dev/netif/txp/if_txp.c +++ b/sys/dev/netif/txp/if_txp.c @@ -114,7 +114,7 @@ static void txp_intr (void *); static void txp_tick (void *); static int txp_shutdown (device_t); static int txp_ioctl (struct ifnet *, u_long, caddr_t, struct ucred *); -static void txp_start (struct ifnet *); +static void txp_start (struct ifnet *, struct ifaltq_subque *); static void txp_stop (struct txp_softc *); static void txp_init (void *); static void txp_watchdog (struct ifnet *); @@ -1183,7 +1183,7 @@ out: } static void -txp_start(struct ifnet *ifp) +txp_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct txp_softc *sc = ifp->if_softc; struct txp_tx_ring *r = &sc->sc_txhir; @@ -1193,6 +1193,8 @@ txp_start(struct ifnet *ifp) struct txp_swdesc *sd; u_int32_t firstprod, firstcnt, prod, cnt; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd)) return; diff --git a/sys/dev/netif/ural/if_ural.c b/sys/dev/netif/ural/if_ural.c index cf5a21d64f..319415af31 100644 --- a/sys/dev/netif/ural/if_ural.c +++ b/sys/dev/netif/ural/if_ural.c @@ -120,7 +120,7 @@ static int ural_tx_mgt(struct ural_softc *, struct mbuf *, struct ieee80211_node *); static int ural_tx_data(struct ural_softc *, struct mbuf *, struct ieee80211_node *); -static void ural_start(struct ifnet *); +static void ural_start(struct ifnet *, struct ifaltq_subque *); static void ural_watchdog(struct ifnet *); static int ural_reset(struct ifnet *); static int ural_ioctl(struct ifnet *, u_long, caddr_t, @@ -1361,11 +1361,12 @@ ural_tx_data(struct ural_softc *sc, struct mbuf *m0, struct ieee80211_node *ni) } static void -ural_start(struct ifnet *ifp) +ural_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct ural_softc *sc = ifp->if_softc; struct ieee80211com *ic = &sc->sc_ic; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); if (sc->sc_stopped) { diff --git a/sys/dev/netif/vge/if_vge.c b/sys/dev/netif/vge/if_vge.c index 57350da94f..2abf7be140 100644 --- a/sys/dev/netif/vge/if_vge.c +++ b/sys/dev/netif/vge/if_vge.c @@ -156,7 +156,7 @@ static void vge_rxeof (struct vge_softc *, int); static void vge_txeof (struct vge_softc *); static void vge_intr (void *); static void vge_tick (struct vge_softc *); -static void vge_start (struct ifnet *); +static void vge_start (struct ifnet *, struct ifaltq_subque *); static int vge_ioctl (struct ifnet *, u_long, caddr_t, struct ucred *); static void vge_init (void *); @@ -1686,12 +1686,13 @@ fail: */ static void -vge_start(struct ifnet *ifp) +vge_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct vge_softc *sc = ifp->if_softc; struct mbuf *m_head = NULL; int idx, pidx = 0; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); if (!sc->vge_link) { diff --git a/sys/dev/netif/vr/if_vr.c b/sys/dev/netif/vr/if_vr.c index 0176ed88cf..149b841675 100644 --- a/sys/dev/netif/vr/if_vr.c +++ b/sys/dev/netif/vr/if_vr.c @@ -138,7 +138,7 @@ static void vr_txeof(struct vr_softc *); static void vr_txeoc(struct vr_softc *); static void vr_tick(void *); static void vr_intr(void *); -static void vr_start(struct ifnet *); +static void vr_start(struct ifnet *, struct ifaltq_subque *); static int vr_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void vr_init(void *); static void vr_stop(struct vr_softc *); @@ -1319,13 +1319,15 @@ vr_encap(struct vr_softc *sc, int chain_idx, struct mbuf *m_head) * physical addresses. */ static void -vr_start(struct ifnet *ifp) +vr_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct vr_softc *sc; struct vr_chain_data *cd; struct vr_chain *tx_chain; int cur_tx_idx, start_tx_idx, prev_tx_idx; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd)) return; diff --git a/sys/dev/netif/vx/if_vx.c b/sys/dev/netif/vx/if_vx.c index d6495e7d85..9dfe3c9526 100644 --- a/sys/dev/netif/vx/if_vx.c +++ b/sys/dev/netif/vx/if_vx.c @@ -104,7 +104,7 @@ static void vxtxstat (struct vx_softc *); static int vxstatus (struct vx_softc *); static void vxinit (void *); static int vxioctl (struct ifnet *, u_long, caddr_t, struct ucred *); -static void vxstart (struct ifnet *ifp); +static void vxstart (struct ifnet *ifp, struct ifaltq_subque *); static void vxwatchdog (struct ifnet *); static void vxreset (struct vx_softc *); /* void vxstop (struct vx_softc *); */ @@ -382,12 +382,14 @@ vxsetlink(struct vx_softc *sc) } static void -vxstart(struct ifnet *ifp) +vxstart(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct vx_softc *sc = ifp->if_softc; struct mbuf *m0; int len, pad; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + /* Don't transmit if interface is busy or not running */ if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd)) return; diff --git a/sys/dev/netif/wb/if_wb.c b/sys/dev/netif/wb/if_wb.c index 497e478398..70d185a515 100644 --- a/sys/dev/netif/wb/if_wb.c +++ b/sys/dev/netif/wb/if_wb.c @@ -149,7 +149,7 @@ static void wb_txeof(struct wb_softc *); static void wb_txeoc(struct wb_softc *); static void wb_intr(void *); static void wb_tick(void *); -static void wb_start(struct ifnet *); +static void wb_start(struct ifnet *, struct ifaltq_subque *); static int wb_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void wb_init(void *); static void wb_stop(struct wb_softc *); @@ -1293,12 +1293,14 @@ wb_encap(struct wb_softc *sc, struct wb_chain *c, struct mbuf *m_head) * physical addresses. */ static void -wb_start(struct ifnet *ifp) +wb_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct wb_softc *sc = ifp->if_softc; struct mbuf *m_head = NULL; struct wb_chain *cur_tx = NULL, *start_tx; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + /* * Check for an available queue slot. If there are none, * punt. diff --git a/sys/dev/netif/wi/if_wi.c b/sys/dev/netif/wi/if_wi.c index a861e42772..5c1477829e 100644 --- a/sys/dev/netif/wi/if_wi.c +++ b/sys/dev/netif/wi/if_wi.c @@ -114,7 +114,7 @@ static struct ieee80211vap *wi_vap_create(struct ieee80211com *ic, static void wi_vap_delete(struct ieee80211vap *vap); static void wi_stop_locked(struct wi_softc *sc, int disable); static void wi_start_locked(struct ifnet *); -static void wi_start(struct ifnet *); +static void wi_start(struct ifnet *, struct ifaltq_subque *); static int wi_start_tx(struct ifnet *ifp, struct wi_frame *frmhdr, struct mbuf *m0); static int wi_raw_xmit(struct ieee80211_node *, struct mbuf *, @@ -1005,8 +1005,9 @@ wi_start_locked(struct ifnet *ifp) } static void -wi_start(struct ifnet *ifp) +wi_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); wi_start_locked(ifp); } diff --git a/sys/dev/netif/wi/if_wi_pci.c b/sys/dev/netif/wi/if_wi_pci.c index 193a1b0107..73fe241e3f 100644 --- a/sys/dev/netif/wi/if_wi_pci.c +++ b/sys/dev/netif/wi/if_wi_pci.c @@ -279,7 +279,7 @@ wi_pci_resume(device_t dev) if (ifp->if_flags & IFF_UP) { ifp->if_init(ifp->if_softc); if (ifp->if_flags & IFF_RUNNING) - ifp->if_start(ifp); + if_devstart(ifp); } wlan_serialize_exit(); diff --git a/sys/dev/netif/wpi/if_wpi.c b/sys/dev/netif/wpi/if_wpi.c index 8092ebe835..3ee653bcb1 100644 --- a/sys/dev/netif/wpi/if_wpi.c +++ b/sys/dev/netif/wpi/if_wpi.c @@ -202,7 +202,7 @@ static uint8_t wpi_plcp_signal(int); static void wpi_watchdog_callout(void *); static int wpi_tx_data(struct wpi_softc *, struct mbuf *, struct ieee80211_node *, int); -static void wpi_start(struct ifnet *); +static void wpi_start(struct ifnet *, struct ifaltq_subque *); static void wpi_start_locked(struct ifnet *); static int wpi_raw_xmit(struct ieee80211_node *, struct mbuf *, const struct ieee80211_bpf_params *); @@ -1243,7 +1243,7 @@ wpi_resume(device_t dev) if (ifp->if_flags & IFF_UP) { wpi_init(ifp->if_softc); if (ifp->if_flags & IFF_RUNNING) - wpi_start(ifp); + if_devstart(ifp); } wlan_serialize_exit(); return 0; @@ -2012,8 +2012,9 @@ wpi_tx_data(struct wpi_softc *sc, struct mbuf *m0, struct ieee80211_node *ni, * Process data waiting to be sent on the IFNET output queue */ static void -wpi_start(struct ifnet *ifp) +wpi_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); wpi_start_locked(ifp); } diff --git a/sys/dev/netif/xe/if_xe.c b/sys/dev/netif/xe/if_xe.c index a995918af2..17152a2ac3 100644 --- a/sys/dev/netif/xe/if_xe.c +++ b/sys/dev/netif/xe/if_xe.c @@ -153,7 +153,7 @@ struct xe_mii_frame { */ static void xe_init (void *xscp); static void xe_intr (void *xscp); -static void xe_start (struct ifnet *ifp); +static void xe_start (struct ifnet *ifp, struct ifaltq_subque *); static int xe_ioctl (struct ifnet *ifp, u_long command, caddr_t data, struct ucred *); static void xe_watchdog (struct ifnet *ifp); static int xe_media_change (struct ifnet *ifp); @@ -468,10 +468,12 @@ xe_init(void *xscp) { * and return immediately. */ static void -xe_start(struct ifnet *ifp) { +xe_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct xe_softc *scp = ifp->if_softc; struct mbuf *mbp; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if (scp->autoneg_status != XE_AUTONEG_NONE) { ifq_set_oactive(&ifp->if_snd); return; diff --git a/sys/dev/netif/xl/if_xl.c b/sys/dev/netif/xl/if_xl.c index 27a6066132..f3a3327a06 100644 --- a/sys/dev/netif/xl/if_xl.c +++ b/sys/dev/netif/xl/if_xl.c @@ -218,15 +218,15 @@ static void xl_txeof_90xB (struct xl_softc *); static void xl_txeoc (struct xl_softc *); static void xl_intr (void *); static void xl_start_body (struct ifnet *, int); -static void xl_start (struct ifnet *); -static void xl_start_90xB (struct ifnet *); +static void xl_start (struct ifnet *, struct ifaltq_subque *); +static void xl_start_90xB (struct ifnet *, struct ifaltq_subque *); static int xl_ioctl (struct ifnet *, u_long, caddr_t, struct ucred *); static void xl_init (void *); static void xl_stop (struct xl_softc *); static void xl_watchdog (struct ifnet *); #ifdef IFPOLL_ENABLE -static void xl_start_poll (struct ifnet *); +static void xl_start_poll (struct ifnet *, struct ifaltq_subque *); static void xl_npoll (struct ifnet *, struct ifpoll_info *); static void xl_npoll_compat (struct ifnet *, void *, int); #endif @@ -2229,8 +2229,9 @@ xl_txeoc(struct xl_softc *sc) #ifdef IFPOLL_ENABLE static void -xl_start_poll(struct ifnet *ifp) +xl_start_poll(struct ifnet *ifp, struct ifaltq_subque *ifsq) { + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); xl_start_body(ifp, 0); } @@ -2478,8 +2479,9 @@ xl_encap(struct xl_softc *sc, struct xl_chain *c, struct mbuf *m_head) } static void -xl_start(struct ifnet *ifp) +xl_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); xl_start_body(ifp, 1); } @@ -2614,7 +2616,7 @@ xl_start_body(struct ifnet *ifp, int proc_rx) } static void -xl_start_90xB(struct ifnet *ifp) +xl_start_90xB(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct xl_softc *sc; struct mbuf *m_head = NULL; @@ -2622,6 +2624,7 @@ xl_start_90xB(struct ifnet *ifp) struct xl_chain *prev_tx; int error, idx; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); sc = ifp->if_softc; diff --git a/sys/dev/virtual/vkernel/net/if_vke.c b/sys/dev/virtual/vkernel/net/if_vke.c index 83945e4037..66ae46e66b 100644 --- a/sys/dev/virtual/vkernel/net/if_vke.c +++ b/sys/dev/virtual/vkernel/net/if_vke.c @@ -107,7 +107,7 @@ struct vke_softc { in_addr_t sc_mask; /* netmask */ }; -static void vke_start(struct ifnet *); +static void vke_start(struct ifnet *, struct ifaltq_subque *); static void vke_init(void *); static int vke_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); @@ -281,7 +281,7 @@ vke_init(void *xsc) vke_stop(sc); ifp->if_flags |= IFF_RUNNING; - ifq_clr_oactive(&ifp->if_snd); + ifsq_clr_oactive(ifq_get_subq_default(&ifp->if_snd)); sc->sc_txfifo = kmalloc(sizeof(*sc->sc_txfifo), M_DEVBUF, M_WAITOK); sc->sc_txfifo_done = kmalloc(sizeof(*sc->sc_txfifo_done), M_DEVBUF, M_WAITOK); @@ -324,20 +324,21 @@ vke_init(void *xsc) * (so mplock, tokens, etc will not be released). */ static void -vke_start(struct ifnet *ifp) +vke_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct vke_softc *sc = ifp->if_softc; struct mbuf *m; cothread_t cotd = sc->cotd_tx; int count; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_SERIALIZED(ifp->if_serializer); - if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd)) + if ((ifp->if_flags & IFF_RUNNING) == 0 || ifsq_is_oactive(ifsq)) return; count = 0; - while ((m = ifq_dequeue(&ifp->if_snd, NULL)) != NULL) { + while ((m = ifsq_dequeue(ifsq, NULL)) != NULL) { if (vke_txfifo_enqueue(sc, m) != -1) { if (count++ == VKE_CHUNK) { cothread_lock(cotd, 0); @@ -417,7 +418,7 @@ vke_stop(struct vke_softc *sc) ASSERT_SERIALIZED(ifp->if_serializer); ifp->if_flags &= ~IFF_RUNNING; - ifq_clr_oactive(&ifp->if_snd); + ifsq_clr_oactive(ifq_get_subq_default(&ifp->if_snd)); if (sc) { if (sc->cotd_tx) { @@ -547,7 +548,7 @@ vke_tx_intr(cothread_t cotd) } if ((ifp->if_flags & IFF_RUNNING) == 0) - ifp->if_start(ifp); + if_devstart(ifp); ifnet_deserialize_all(ifp); } diff --git a/sys/net/altq/altq_cbq.c b/sys/net/altq/altq_cbq.c index a7b873f6c3..f0e52a3172 100644 --- a/sys/net/altq/altq_cbq.c +++ b/sys/net/altq/altq_cbq.c @@ -58,16 +58,25 @@ #include +#define CBQ_SUBQ_INDEX ALTQ_SUBQ_INDEX_DEFAULT +#define CBQ_LOCK(ifq) \ + ALTQ_SQ_LOCK(&(ifq)->altq_subq[CBQ_SUBQ_INDEX]) +#define CBQ_UNLOCK(ifq) \ + ALTQ_SQ_UNLOCK(&(ifq)->altq_subq[CBQ_SUBQ_INDEX]) +#define CBQ_ASSERT_LOCKED(ifq) \ + ALTQ_SQ_ASSERT_LOCKED(&(ifq)->altq_subq[CBQ_SUBQ_INDEX]) + /* * Forward Declarations. */ static int cbq_class_destroy(cbq_state_t *, struct rm_class *); static struct rm_class *clh_to_clp(cbq_state_t *, uint32_t); static int cbq_clear_interface(cbq_state_t *); -static int cbq_request(struct ifaltq *, int, void *); -static int cbq_enqueue(struct ifaltq *, struct mbuf *, +static int cbq_request(struct ifaltq_subque *, int, void *); +static int cbq_enqueue(struct ifaltq_subque *, struct mbuf *, struct altq_pktattr *); -static struct mbuf *cbq_dequeue(struct ifaltq *, struct mbuf *, int); +static struct mbuf *cbq_dequeue(struct ifaltq_subque *, struct mbuf *, + int); static void cbqrestart(struct ifaltq *); static void get_class_stats(class_stats_t *, struct rm_class *); static void cbq_purge(cbq_state_t *); @@ -153,14 +162,23 @@ cbq_clear_interface(cbq_state_t *cbqp) } static int -cbq_request(struct ifaltq *ifq, int req, void *arg) +cbq_request(struct ifaltq_subque *ifsq, int req, void *arg) { + struct ifaltq *ifq = ifsq->ifsq_altq; cbq_state_t *cbqp = (cbq_state_t *)ifq->altq_disc; crit_enter(); switch (req) { case ALTRQ_PURGE: - cbq_purge(cbqp); + if (ifsq_get_index(ifsq) == CBQ_SUBQ_INDEX) { + cbq_purge(cbqp); + } else { + /* + * Race happened, the unrelated subqueue was + * picked during the packet scheduler transition. + */ + ifsq_classic_request(ifsq, ALTRQ_PURGE, NULL); + } break; } crit_exit(); @@ -374,9 +392,9 @@ cbq_add_queue(struct pf_altq *a) return (EINVAL); ifq = cbqp->ifnp.ifq_; - ALTQ_LOCK(ifq); + CBQ_LOCK(ifq); error = cbq_add_queue_locked(a, cbqp); - ALTQ_UNLOCK(ifq); + CBQ_UNLOCK(ifq); return error; } @@ -425,9 +443,9 @@ cbq_remove_queue(struct pf_altq *a) return (EINVAL); ifq = cbqp->ifnp.ifq_; - ALTQ_LOCK(ifq); + CBQ_LOCK(ifq); error = cbq_remove_queue_locked(a, cbqp); - ALTQ_UNLOCK(ifq); + CBQ_UNLOCK(ifq); return error; } @@ -449,16 +467,16 @@ cbq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) return (EBADF); ifq = cbqp->ifnp.ifq_; - ALTQ_LOCK(ifq); + CBQ_LOCK(ifq); if ((cl = clh_to_clp(cbqp, a->qid)) == NULL) { - ALTQ_UNLOCK(ifq); + CBQ_UNLOCK(ifq); return (EINVAL); } get_class_stats(&stats, cl); - ALTQ_UNLOCK(ifq); + CBQ_UNLOCK(ifq); if ((error = copyout((caddr_t)&stats, ubuf, sizeof(stats))) != 0) return (error); @@ -468,7 +486,8 @@ cbq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) /* * int - * cbq_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pattr) + * cbq_enqueue(struct ifaltq_subqueue *ifq, struct mbuf *m, + * struct altq_pktattr *pattr) * - Queue data packets. * * cbq_enqueue is set to ifp->if_altqenqueue and called by an upper @@ -481,12 +500,24 @@ cbq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) */ static int -cbq_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pktattr) +cbq_enqueue(struct ifaltq_subque *ifsq, struct mbuf *m, + struct altq_pktattr *pktattr __unused) { + struct ifaltq *ifq = ifsq->ifsq_altq; cbq_state_t *cbqp = (cbq_state_t *)ifq->altq_disc; struct rm_class *cl; int len; + if (ifsq_get_index(ifsq) != CBQ_SUBQ_INDEX) { + /* + * Race happened, the unrelated subqueue was + * picked during the packet scheduler transition. + */ + ifsq_classic_request(ifsq, ALTRQ_PURGE, NULL); + m_freem(m); + return (ENOBUFS); + } + /* grab class set by classifier */ if ((m->m_flags & M_PKTHDR) == 0) { /* should not happen */ @@ -517,23 +548,33 @@ cbq_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pktattr) /* successfully queued. */ ++cbqp->cbq_qlen; - ++ifq->ifq_len; + ++ifsq->ifq_len; crit_exit(); return (0); } static struct mbuf * -cbq_dequeue(struct ifaltq *ifq, struct mbuf *mpolled, int op) +cbq_dequeue(struct ifaltq_subque *ifsq, struct mbuf *mpolled, int op) { + struct ifaltq *ifq = ifsq->ifsq_altq; cbq_state_t *cbqp = (cbq_state_t *)ifq->altq_disc; struct mbuf *m; + if (ifsq_get_index(ifsq) != CBQ_SUBQ_INDEX) { + /* + * Race happened, the unrelated subqueue was + * picked during the packet scheduler transition. + */ + ifsq_classic_request(ifsq, ALTRQ_PURGE, NULL); + return NULL; + } + crit_enter(); m = rmc_dequeue_next(&cbqp->ifnp, op); if (m && op == ALTDQ_REMOVE) { --cbqp->cbq_qlen; /* decrement # of packets in cbq */ - --ifq->ifq_len; + --ifsq->ifq_len; /* Update the class. */ rmc_update_class_util(&cbqp->ifnp); @@ -556,7 +597,7 @@ cbqrestart(struct ifaltq *ifq) { cbq_state_t *cbqp; - ALTQ_ASSERT_LOCKED(ifq); + CBQ_ASSERT_LOCKED(ifq); if (!ifq_is_enabled(ifq)) /* cbq must have been detached */ @@ -568,16 +609,17 @@ cbqrestart(struct ifaltq *ifq) if (cbqp->cbq_qlen > 0) { struct ifnet *ifp = ifq->altq_ifp; + struct ifaltq_subque *ifsq = &ifq->altq_subq[CBQ_SUBQ_INDEX]; /* Release the altq lock to avoid deadlock */ - ALTQ_UNLOCK(ifq); + CBQ_UNLOCK(ifq); ifnet_serialize_tx(ifp); - if (ifp->if_start && !ifq_is_oactive(&ifp->if_snd)) - (*ifp->if_start)(ifp); + if (ifp->if_start && !ifsq_is_oactive(ifsq)) + (*ifp->if_start)(ifp, ifsq); ifnet_deserialize_tx(ifp); - ALTQ_LOCK(ifq); + CBQ_LOCK(ifq); } } @@ -591,7 +633,7 @@ cbq_purge(cbq_state_t *cbqp) rmc_dropall(cl); } if (ifq_is_enabled(cbqp->ifnp.ifq_)) - cbqp->ifnp.ifq_->ifq_len = 0; + cbqp->ifnp.ifq_->altq_subq[CBQ_SUBQ_INDEX].ifq_len = 0; } #endif /* ALTQ_CBQ */ diff --git a/sys/net/altq/altq_fairq.c b/sys/net/altq/altq_fairq.c index 71ac80b6e5..c55d9dcf0b 100644 --- a/sys/net/altq/altq_fairq.c +++ b/sys/net/altq/altq_fairq.c @@ -116,18 +116,24 @@ #include +#define FAIRQ_SUBQ_INDEX ALTQ_SUBQ_INDEX_DEFAULT +#define FAIRQ_LOCK(ifq) \ + ALTQ_SQ_LOCK(&(ifq)->altq_subq[FAIRQ_SUBQ_INDEX]) +#define FAIRQ_UNLOCK(ifq) \ + ALTQ_SQ_UNLOCK(&(ifq)->altq_subq[FAIRQ_SUBQ_INDEX]) + /* * function prototypes */ static int fairq_clear_interface(struct fairq_if *); -static int fairq_request(struct ifaltq *, int, void *); +static int fairq_request(struct ifaltq_subque *, int, void *); static void fairq_purge(struct fairq_if *); static struct fairq_class *fairq_class_create(struct fairq_if *, int, int, u_int, struct fairq_opts *, int); static int fairq_class_destroy(struct fairq_class *); -static int fairq_enqueue(struct ifaltq *, struct mbuf *, +static int fairq_enqueue(struct ifaltq_subque *, struct mbuf *, struct altq_pktattr *); -static struct mbuf *fairq_dequeue(struct ifaltq *, struct mbuf *, int); +static struct mbuf *fairq_dequeue(struct ifaltq_subque *, struct mbuf *, int); static int fairq_addq(struct fairq_class *, struct mbuf *, int hash); static struct mbuf *fairq_getq(struct fairq_class *, uint64_t); @@ -224,9 +230,9 @@ fairq_add_queue(struct pf_altq *a) return (EINVAL); ifq = pif->pif_ifq; - ALTQ_LOCK(ifq); + FAIRQ_LOCK(ifq); error = fairq_add_queue_locked(a, pif); - ALTQ_UNLOCK(ifq); + FAIRQ_UNLOCK(ifq); return error; } @@ -254,9 +260,9 @@ fairq_remove_queue(struct pf_altq *a) return (EINVAL); ifq = pif->pif_ifq; - ALTQ_LOCK(ifq); + FAIRQ_LOCK(ifq); error = fairq_remove_queue_locked(a, pif); - ALTQ_UNLOCK(ifq); + FAIRQ_UNLOCK(ifq); return error; } @@ -278,16 +284,16 @@ fairq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) return (EBADF); ifq = pif->pif_ifq; - ALTQ_LOCK(ifq); + FAIRQ_LOCK(ifq); if ((cl = clh_to_clp(pif, a->qid)) == NULL) { - ALTQ_UNLOCK(ifq); + FAIRQ_UNLOCK(ifq); return (EINVAL); } get_class_stats(&stats, cl); - ALTQ_UNLOCK(ifq); + FAIRQ_UNLOCK(ifq); if ((error = copyout((caddr_t)&stats, ubuf, sizeof(stats))) != 0) return (error); @@ -315,14 +321,23 @@ fairq_clear_interface(struct fairq_if *pif) } static int -fairq_request(struct ifaltq *ifq, int req, void *arg) +fairq_request(struct ifaltq_subque *ifsq, int req, void *arg) { + struct ifaltq *ifq = ifsq->ifsq_altq; struct fairq_if *pif = (struct fairq_if *)ifq->altq_disc; crit_enter(); switch (req) { case ALTRQ_PURGE: - fairq_purge(pif); + if (ifsq_get_index(ifsq) == FAIRQ_SUBQ_INDEX) { + fairq_purge(pif); + } else { + /* + * Race happened, the unrelated subqueue was + * picked during the packet scheduler transition. + */ + ifsq_classic_request(ifsq, ALTRQ_PURGE, NULL); + } break; } crit_exit(); @@ -341,7 +356,7 @@ fairq_purge(struct fairq_if *pif) fairq_purgeq(cl); } if (ifq_is_enabled(pif->pif_ifq)) - pif->pif_ifq->ifq_len = 0; + pif->pif_ifq->altq_subq[FAIRQ_SUBQ_INDEX].ifq_len = 0; } static struct fairq_class * @@ -503,14 +518,26 @@ fairq_class_destroy(struct fairq_class *cl) * (*altq_enqueue) in struct ifaltq. */ static int -fairq_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pktattr) +fairq_enqueue(struct ifaltq_subque *ifsq, struct mbuf *m, + struct altq_pktattr *pktattr) { + struct ifaltq *ifq = ifsq->ifsq_altq; struct fairq_if *pif = (struct fairq_if *)ifq->altq_disc; struct fairq_class *cl; int error; int len; int hash; + if (ifsq_get_index(ifsq) != FAIRQ_SUBQ_INDEX) { + /* + * Race happened, the unrelated subqueue was + * picked during the packet scheduler transition. + */ + ifsq_classic_request(ifsq, ALTRQ_PURGE, NULL); + m_freem(m); + return ENOBUFS; + } + crit_enter(); /* grab class set by classifier */ @@ -549,7 +576,7 @@ fairq_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pktattr) error = ENOBUFS; goto done; } - ifq->ifq_len++; + ifsq->ifq_len++; error = 0; done: crit_exit(); @@ -566,8 +593,9 @@ done: * after ALTDQ_POLL. */ static struct mbuf * -fairq_dequeue(struct ifaltq *ifq, struct mbuf *mpolled, int op) +fairq_dequeue(struct ifaltq_subque *ifsq, struct mbuf *mpolled, int op) { + struct ifaltq *ifq = ifsq->ifsq_altq; struct fairq_if *pif = (struct fairq_if *)ifq->altq_disc; struct fairq_class *cl; struct fairq_class *best_cl; @@ -579,7 +607,16 @@ fairq_dequeue(struct ifaltq *ifq, struct mbuf *mpolled, int op) int pri; int hit_limit; - if (ifq_is_empty(ifq)) { + if (ifsq_get_index(ifsq) != FAIRQ_SUBQ_INDEX) { + /* + * Race happened, the unrelated subqueue was + * picked during the packet scheduler transition. + */ + ifsq_classic_request(ifsq, ALTRQ_PURGE, NULL); + return NULL; + } + + if (ifsq_is_empty(ifsq)) { /* no packet in the queue */ KKASSERT(mpolled == NULL); return (NULL); @@ -591,7 +628,7 @@ fairq_dequeue(struct ifaltq *ifq, struct mbuf *mpolled, int op) m = fairq_getq(best_cl, cur_time); pif->pif_poll_cache = NULL; if (m) { - ifq->ifq_len--; + ifsq->ifq_len--; PKTCNTR_ADD(&best_cl->cl_xmitcnt, m_pktlen(m)); } } else { @@ -640,7 +677,7 @@ fairq_dequeue(struct ifaltq *ifq, struct mbuf *mpolled, int op) } else if (best_cl) { m = fairq_getq(best_cl, cur_time); KKASSERT(best_m == m); - ifq->ifq_len--; + ifsq->ifq_len--; PKTCNTR_ADD(&best_cl->cl_xmitcnt, m_pktlen(m)); } else { m = NULL; diff --git a/sys/net/altq/altq_hfsc.c b/sys/net/altq/altq_hfsc.c index b0f32d550d..60f1badf78 100644 --- a/sys/net/altq/altq_hfsc.c +++ b/sys/net/altq/altq_hfsc.c @@ -66,11 +66,17 @@ #include +#define HFSC_SUBQ_INDEX ALTQ_SUBQ_INDEX_DEFAULT +#define HFSC_LOCK(ifq) \ + ALTQ_SQ_LOCK(&(ifq)->altq_subq[HFSC_SUBQ_INDEX]) +#define HFSC_UNLOCK(ifq) \ + ALTQ_SQ_UNLOCK(&(ifq)->altq_subq[HFSC_SUBQ_INDEX]) + /* * function prototypes */ static int hfsc_clear_interface(struct hfsc_if *); -static int hfsc_request(struct ifaltq *, int, void *); +static int hfsc_request(struct ifaltq_subque *, int, void *); static void hfsc_purge(struct hfsc_if *); static struct hfsc_class *hfsc_class_create(struct hfsc_if *, struct service_curve *, @@ -79,9 +85,9 @@ static struct hfsc_class *hfsc_class_create(struct hfsc_if *, struct hfsc_class *, int, int, int); static int hfsc_class_destroy(struct hfsc_class *); static struct hfsc_class *hfsc_nextclass(struct hfsc_class *); -static int hfsc_enqueue(struct ifaltq *, struct mbuf *, +static int hfsc_enqueue(struct ifaltq_subque *, struct mbuf *, struct altq_pktattr *); -static struct mbuf *hfsc_dequeue(struct ifaltq *, struct mbuf *, int); +static struct mbuf *hfsc_dequeue(struct ifaltq_subque *, struct mbuf *, int); static int hfsc_addq(struct hfsc_class *, struct mbuf *); static struct mbuf *hfsc_getq(struct hfsc_class *); @@ -238,9 +244,9 @@ hfsc_add_queue(struct pf_altq *a) return (EINVAL); ifq = hif->hif_ifq; - ALTQ_LOCK(ifq); + HFSC_LOCK(ifq); error = hfsc_add_queue_locked(a, hif); - ALTQ_UNLOCK(ifq); + HFSC_UNLOCK(ifq); return error; } @@ -268,9 +274,9 @@ hfsc_remove_queue(struct pf_altq *a) return (EINVAL); ifq = hif->hif_ifq; - ALTQ_LOCK(ifq); + HFSC_LOCK(ifq); error = hfsc_remove_queue_locked(a, hif); - ALTQ_UNLOCK(ifq); + HFSC_UNLOCK(ifq); return error; } @@ -292,16 +298,16 @@ hfsc_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) return (EBADF); ifq = hif->hif_ifq; - ALTQ_LOCK(ifq); + HFSC_LOCK(ifq); if ((cl = clh_to_clp(hif, a->qid)) == NULL) { - ALTQ_UNLOCK(ifq); + HFSC_UNLOCK(ifq); return (EINVAL); } get_class_stats(&stats, cl); - ALTQ_UNLOCK(ifq); + HFSC_UNLOCK(ifq); if ((error = copyout((caddr_t)&stats, ubuf, sizeof(stats))) != 0) return (error); @@ -340,14 +346,23 @@ hfsc_clear_interface(struct hfsc_if *hif) } static int -hfsc_request(struct ifaltq *ifq, int req, void *arg) +hfsc_request(struct ifaltq_subque *ifsq, int req, void *arg) { + struct ifaltq *ifq = ifsq->ifsq_altq; struct hfsc_if *hif = (struct hfsc_if *)ifq->altq_disc; crit_enter(); switch (req) { case ALTRQ_PURGE: - hfsc_purge(hif); + if (ifsq_get_index(ifsq) == HFSC_SUBQ_INDEX) { + hfsc_purge(hif); + } else { + /* + * Race happened, the unrelated subqueue was + * picked during the packet scheduler transition. + */ + ifsq_classic_request(ifsq, ALTRQ_PURGE, NULL); + } break; } crit_exit(); @@ -365,7 +380,7 @@ hfsc_purge(struct hfsc_if *hif) hfsc_purgeq(cl); } if (ifq_is_enabled(hif->hif_ifq)) - hif->hif_ifq->ifq_len = 0; + hif->hif_ifq->altq_subq[HFSC_SUBQ_INDEX].ifq_len = 0; } struct hfsc_class * @@ -640,12 +655,24 @@ hfsc_nextclass(struct hfsc_class *cl) * (*altq_enqueue) in struct ifaltq. */ static int -hfsc_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pktattr) +hfsc_enqueue(struct ifaltq_subque *ifsq, struct mbuf *m, + struct altq_pktattr *pktattr) { + struct ifaltq *ifq = ifsq->ifsq_altq; struct hfsc_if *hif = (struct hfsc_if *)ifq->altq_disc; struct hfsc_class *cl; int len; + if (ifsq_get_index(ifsq) != HFSC_SUBQ_INDEX) { + /* + * Race happened, the unrelated subqueue was + * picked during the packet scheduler transition. + */ + ifsq_classic_request(ifsq, ALTRQ_PURGE, NULL); + m_freem(m); + return ENOBUFS; + } + /* grab class set by classifier */ if ((m->m_flags & M_PKTHDR) == 0) { /* should not happen */ @@ -674,7 +701,7 @@ hfsc_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pktattr) crit_exit(); return (ENOBUFS); } - ifq->ifq_len++; + ifsq->ifq_len++; cl->cl_hif->hif_packets++; /* successfully queued. */ @@ -694,8 +721,9 @@ hfsc_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pktattr) * after ALTDQ_POLL. */ static struct mbuf * -hfsc_dequeue(struct ifaltq *ifq, struct mbuf *mpolled, int op) +hfsc_dequeue(struct ifaltq_subque *ifsq, struct mbuf *mpolled, int op) { + struct ifaltq *ifq = ifsq->ifsq_altq; struct hfsc_if *hif = (struct hfsc_if *)ifq->altq_disc; struct hfsc_class *cl; struct mbuf *m; @@ -703,6 +731,15 @@ hfsc_dequeue(struct ifaltq *ifq, struct mbuf *mpolled, int op) int realtime = 0; uint64_t cur_time; + if (ifsq_get_index(ifsq) != HFSC_SUBQ_INDEX) { + /* + * Race happened, the unrelated subqueue was + * picked during the packet scheduler transition. + */ + ifsq_classic_request(ifsq, ALTRQ_PURGE, NULL); + return NULL; + } + if (hif->hif_packets == 0) { /* no packet in the tree */ return (NULL); @@ -769,7 +806,7 @@ hfsc_dequeue(struct ifaltq *ifq, struct mbuf *mpolled, int op) panic("hfsc_dequeue:"); len = m_pktlen(m); cl->cl_hif->hif_packets--; - ifq->ifq_len--; + ifsq->ifq_len--; PKTCNTR_ADD(&cl->cl_stats.xmit_cnt, len); update_vf(cl, len, cur_time); @@ -854,7 +891,7 @@ hfsc_purgeq(struct hfsc_class *cl) PKTCNTR_ADD(&cl->cl_stats.drop_cnt, m_pktlen(m)); m_freem(m); cl->cl_hif->hif_packets--; - cl->cl_hif->hif_ifq->ifq_len--; + cl->cl_hif->hif_ifq->altq_subq[HFSC_SUBQ_INDEX].ifq_len--; } KKASSERT(qlen(cl->cl_q) == 0); diff --git a/sys/net/altq/altq_priq.c b/sys/net/altq/altq_priq.c index fd6d29a4d3..01335e5afb 100644 --- a/sys/net/altq/altq_priq.c +++ b/sys/net/altq/altq_priq.c @@ -58,16 +58,23 @@ #include +#define PRIQ_SUBQ_INDEX ALTQ_SUBQ_INDEX_DEFAULT +#define PRIQ_LOCK(ifq) \ + ALTQ_SQ_LOCK(&(ifq)->altq_subq[PRIQ_SUBQ_INDEX]) +#define PRIQ_UNLOCK(ifq) \ + ALTQ_SQ_UNLOCK(&(ifq)->altq_subq[PRIQ_SUBQ_INDEX]) + /* * function prototypes */ static int priq_clear_interface(struct priq_if *); -static int priq_request(struct ifaltq *, int, void *); +static int priq_request(struct ifaltq_subque *, int, void *); static void priq_purge(struct priq_if *); static struct priq_class *priq_class_create(struct priq_if *, int, int, int, int); static int priq_class_destroy(struct priq_class *); -static int priq_enqueue(struct ifaltq *, struct mbuf *, struct altq_pktattr *); -static struct mbuf *priq_dequeue(struct ifaltq *, struct mbuf *, int); +static int priq_enqueue(struct ifaltq_subque *, struct mbuf *, + struct altq_pktattr *); +static struct mbuf *priq_dequeue(struct ifaltq_subque *, struct mbuf *, int); static int priq_addq(struct priq_class *, struct mbuf *); static struct mbuf *priq_getq(struct priq_class *); @@ -161,9 +168,9 @@ priq_add_queue(struct pf_altq *a) return (EINVAL); ifq = pif->pif_ifq; - ALTQ_LOCK(ifq); + PRIQ_LOCK(ifq); error = priq_add_queue_locked(a, pif); - ALTQ_UNLOCK(ifq); + PRIQ_UNLOCK(ifq); return error; } @@ -191,9 +198,9 @@ priq_remove_queue(struct pf_altq *a) return (EINVAL); ifq = pif->pif_ifq; - ALTQ_LOCK(ifq); + PRIQ_LOCK(ifq); error = priq_remove_queue_locked(a, pif); - ALTQ_UNLOCK(ifq); + PRIQ_UNLOCK(ifq); return error; } @@ -215,16 +222,16 @@ priq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) return (EBADF); ifq = pif->pif_ifq; - ALTQ_LOCK(ifq); + PRIQ_LOCK(ifq); if ((cl = clh_to_clp(pif, a->qid)) == NULL) { - ALTQ_UNLOCK(ifq); + PRIQ_UNLOCK(ifq); return (EINVAL); } get_class_stats(&stats, cl); - ALTQ_UNLOCK(ifq); + PRIQ_UNLOCK(ifq); if ((error = copyout((caddr_t)&stats, ubuf, sizeof(stats))) != 0) return (error); @@ -252,14 +259,23 @@ priq_clear_interface(struct priq_if *pif) } static int -priq_request(struct ifaltq *ifq, int req, void *arg) +priq_request(struct ifaltq_subque *ifsq, int req, void *arg) { + struct ifaltq *ifq = ifsq->ifsq_altq; struct priq_if *pif = (struct priq_if *)ifq->altq_disc; crit_enter(); switch (req) { case ALTRQ_PURGE: - priq_purge(pif); + if (ifsq_get_index(ifsq) == PRIQ_SUBQ_INDEX) { + priq_purge(pif); + } else { + /* + * Race happened, the unrelated subqueue was + * picked during the packet scheduler transition. + */ + ifsq_classic_request(ifsq, ALTRQ_PURGE, NULL); + } break; } crit_exit(); @@ -278,7 +294,7 @@ priq_purge(struct priq_if *pif) priq_purgeq(cl); } if (ifq_is_enabled(pif->pif_ifq)) - pif->pif_ifq->ifq_len = 0; + pif->pif_ifq->altq_subq[PRIQ_SUBQ_INDEX].ifq_len = 0; } static struct priq_class * @@ -411,13 +427,25 @@ priq_class_destroy(struct priq_class *cl) * (*altq_enqueue) in struct ifaltq. */ static int -priq_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pktattr) +priq_enqueue(struct ifaltq_subque *ifsq, struct mbuf *m, + struct altq_pktattr *pktattr) { + struct ifaltq *ifq = ifsq->ifsq_altq; struct priq_if *pif = (struct priq_if *)ifq->altq_disc; struct priq_class *cl; int error; int len; + if (ifsq_get_index(ifsq) != PRIQ_SUBQ_INDEX) { + /* + * Race happened, the unrelated subqueue was + * picked during the packet scheduler transition. + */ + ifsq_classic_request(ifsq, ALTRQ_PURGE, NULL); + m_freem(m); + return ENOBUFS; + } + crit_enter(); /* grab class set by classifier */ @@ -449,7 +477,7 @@ priq_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pktattr) error = ENOBUFS; goto done; } - ifq->ifq_len++; + ifsq->ifq_len++; error = 0; done: crit_exit(); @@ -466,14 +494,24 @@ done: * after ALTDQ_POLL. */ static struct mbuf * -priq_dequeue(struct ifaltq *ifq, struct mbuf *mpolled, int op) +priq_dequeue(struct ifaltq_subque *ifsq, struct mbuf *mpolled, int op) { + struct ifaltq *ifq = ifsq->ifsq_altq; struct priq_if *pif = (struct priq_if *)ifq->altq_disc; struct priq_class *cl; struct mbuf *m; int pri; - if (ifq_is_empty(ifq)) { + if (ifsq_get_index(ifsq) != PRIQ_SUBQ_INDEX) { + /* + * Race happened, the unrelated subqueue was + * picked during the packet scheduler transition. + */ + ifsq_classic_request(ifsq, ALTRQ_PURGE, NULL); + return NULL; + } + + if (ifsq_is_empty(ifsq)) { /* no packet in the queue */ KKASSERT(mpolled == NULL); return (NULL); @@ -490,7 +528,7 @@ priq_dequeue(struct ifaltq *ifq, struct mbuf *mpolled, int op) m = priq_getq(cl); if (m != NULL) { - ifq->ifq_len--; + ifsq->ifq_len--; if (qempty(cl->cl_q)) cl->cl_period++; PKTCNTR_ADD(&cl->cl_xmitcnt, m_pktlen(m)); diff --git a/sys/net/altq/altq_rmclass.c b/sys/net/altq/altq_rmclass.c index d99368f4a0..221d46caad 100644 --- a/sys/net/altq/altq_rmclass.c +++ b/sys/net/altq/altq_rmclass.c @@ -1516,8 +1516,9 @@ rmc_restart(void *arg) { struct rm_class *cl = arg; struct rm_ifdat *ifd = cl->ifdat_; + struct ifaltq_subque *ifsq = &ifd->ifq_->altq_subq[0]; - ALTQ_LOCK(ifd->ifq_); + ALTQ_SQ_LOCK(ifsq); if (cl->sleeping_) { cl->sleeping_ = 0; cl->undertime_.tv_sec = 0; @@ -1527,7 +1528,7 @@ rmc_restart(void *arg) (ifd->restart)(ifd->ifq_); } } - ALTQ_UNLOCK(ifd->ifq_); + ALTQ_SQ_UNLOCK(ifsq); } /* diff --git a/sys/net/altq/altq_subr.c b/sys/net/altq/altq_subr.c index b6f36fb516..864f513b27 100644 --- a/sys/net/altq/altq_subr.c +++ b/sys/net/altq/altq_subr.c @@ -105,24 +105,19 @@ altq_lookup(const char *name, int type) int altq_attach(struct ifaltq *ifq, int type, void *discipline, - int (*enqueue)(struct ifaltq *, struct mbuf *, struct altq_pktattr *), - struct mbuf *(*dequeue)(struct ifaltq *, struct mbuf *, int), - int (*request)(struct ifaltq *, int, void *), - void *clfier, - void *(*classify)(struct ifaltq *, struct mbuf *, - struct altq_pktattr *)) + ifsq_enqueue_t enqueue, ifsq_dequeue_t dequeue, ifsq_request_t request, + void *clfier, + void *(*classify)(struct ifaltq *, struct mbuf *, struct altq_pktattr *)) { if (!ifq_is_ready(ifq)) return ENXIO; ifq->altq_type = type; ifq->altq_disc = discipline; - ifq->altq_enqueue = enqueue; - ifq->altq_dequeue = dequeue; - ifq->altq_request = request; ifq->altq_clfier = clfier; ifq->altq_classify = classify; ifq->altq_flags &= (ALTQF_CANTCHANGE|ALTQF_ENABLED); + ifq_set_methods(ifq, enqueue, dequeue, request); return 0; } @@ -150,9 +145,9 @@ altq_detach(struct ifaltq *ifq) { int error; - ALTQ_LOCK(ifq); + ifq_lock_all(ifq); error = altq_detach_locked(ifq); - ALTQ_UNLOCK(ifq); + ifq_unlock_all(ifq); return error; } @@ -165,7 +160,6 @@ altq_enable_locked(struct ifaltq *ifq) return 0; ifq_purge_all_locked(ifq); - KKASSERT(ifq->ifq_len == 0); ifq->altq_flags |= ALTQF_ENABLED; if (ifq->altq_clfier != NULL) @@ -178,9 +172,9 @@ altq_enable(struct ifaltq *ifq) { int error; - ALTQ_LOCK(ifq); + ifq_lock_all(ifq); error = altq_enable_locked(ifq); - ALTQ_UNLOCK(ifq); + ifq_unlock_all(ifq); return error; } @@ -191,7 +185,6 @@ altq_disable_locked(struct ifaltq *ifq) return 0; ifq_purge_all_locked(ifq); - KKASSERT(ifq->ifq_len == 0); ifq->altq_flags &= ~(ALTQF_ENABLED|ALTQF_CLASSIFY); return 0; } @@ -201,9 +194,9 @@ altq_disable(struct ifaltq *ifq) { int error; - ALTQ_LOCK(ifq); + ifq_lock_all(ifq); error = altq_disable_locked(ifq); - ALTQ_UNLOCK(ifq); + ifq_unlock_all(ifq); return error; } @@ -219,13 +212,23 @@ altq_disable(struct ifaltq *ifq) #define TBR_UNSCALE(x) ((x) >> TBR_SHIFT) struct mbuf * -tbr_dequeue(struct ifaltq *ifq, struct mbuf *mpolled, int op) +tbr_dequeue(struct ifaltq_subque *ifsq, struct mbuf *mpolled, int op) { + struct ifaltq *ifq = ifsq->ifsq_altq; struct tb_regulator *tbr; struct mbuf *m; int64_t interval; uint64_t now; + if (ifsq_get_index(ifsq) != ALTQ_SUBQ_INDEX_DEFAULT) { + /* + * Race happened, the unrelated subqueue was + * picked during the packet scheduler transition. + */ + ifsq_classic_request(ifsq, ALTRQ_PURGE, NULL); + return NULL; + } + crit_enter(); tbr = ifq->altq_tbr; if (op == ALTDQ_REMOVE && tbr->tbr_lastop == ALTDQ_POLL) { @@ -252,11 +255,11 @@ tbr_dequeue(struct ifaltq *ifq, struct mbuf *mpolled, int op) } if (ifq_is_enabled(ifq)) { - m = (*ifq->altq_dequeue)(ifq, mpolled, op); + m = (*ifsq->ifsq_dequeue)(ifsq, mpolled, op); } else if (op == ALTDQ_POLL) { - IF_POLL(ifq, m); + IF_POLL(ifsq, m); } else { - IF_DEQUEUE(ifq, m); + IF_DEQUEUE(ifsq, m); KKASSERT(mpolled == NULL || mpolled == m); } @@ -320,9 +323,9 @@ tbr_set(struct ifaltq *ifq, struct tb_profile *profile) { int error; - ALTQ_LOCK(ifq); + ifq_lock_all(ifq); error = tbr_set_locked(ifq, profile); - ALTQ_UNLOCK(ifq); + ifq_unlock_all(ifq); return error; } @@ -339,12 +342,15 @@ tbr_timeout(void *arg) active = 0; crit_enter(); for (ifp = TAILQ_FIRST(&ifnet); ifp; ifp = TAILQ_NEXT(ifp, if_list)) { + struct ifaltq_subque *ifsq = + &ifp->if_snd.altq_subq[ALTQ_SUBQ_INDEX_DEFAULT]; + if (ifp->if_snd.altq_tbr == NULL) continue; active++; - if (!ifq_is_empty(&ifp->if_snd) && ifp->if_start != NULL) { + if (!ifsq_is_empty(ifsq) && ifp->if_start != NULL) { ifnet_serialize_tx(ifp); - (*ifp->if_start)(ifp); + (*ifp->if_start)(ifp, ifsq); ifnet_deserialize_tx(ifp); } } @@ -396,7 +402,7 @@ altq_pfattach(struct pf_altq *a) return EINVAL; ifq = &ifp->if_snd; - ALTQ_LOCK(ifq); + ifq_lock_all(ifq); switch (a->scheduler) { #ifdef ALTQ_CBQ @@ -438,7 +444,7 @@ altq_pfattach(struct pf_altq *a) error = tbr_set_locked(ifq, &tb); } back: - ALTQ_UNLOCK(ifq); + ifq_unlock_all(ifq); return (error); } @@ -463,7 +469,7 @@ altq_pfdetach(struct pf_altq *a) if (a->altq_disc == NULL) return (0); - ALTQ_LOCK(ifq); + ifq_lock_all(ifq); if (a->altq_disc != ifq->altq_disc) goto back; @@ -474,7 +480,7 @@ altq_pfdetach(struct pf_altq *a) error = altq_detach_locked(ifq); back: - ALTQ_UNLOCK(ifq); + ifq_unlock_all(ifq); return (error); } diff --git a/sys/net/altq/if_altq.h b/sys/net/altq/if_altq.h index 9f0c08c8ce..195b9a77e2 100644 --- a/sys/net/altq/if_altq.h +++ b/sys/net/altq/if_altq.h @@ -32,65 +32,101 @@ #include #endif +/* Default subqueue */ +#define ALTQ_SUBQ_INDEX_DEFAULT 0 + +struct mbuf; struct altq_pktattr; +struct ifaltq_subque; struct ifaltq; -struct ifaltq_stage { - struct ifaltq *ifqs_altq; - int ifqs_cnt; - int ifqs_len; - uint32_t ifqs_flags; - TAILQ_ENTRY(ifaltq_stage) ifqs_link; +typedef int (*ifsq_enqueue_t)(struct ifaltq_subque *, struct mbuf *, + struct altq_pktattr *); +typedef struct mbuf *(*ifsq_dequeue_t)(struct ifaltq_subque *, + struct mbuf *, int); +typedef int (*ifsq_request_t)(struct ifaltq_subque *, int, void *); + +struct ifsubq_stage { + struct ifaltq_subque *stg_subq; + int stg_cnt; + int stg_len; + uint32_t stg_flags; + TAILQ_ENTRY(ifsubq_stage) stg_link; +} __cachealign; + +#define IFSQ_STAGE_FLAG_QUED 0x1 +#define IFSQ_STAGE_FLAG_SCHED 0x2 + +struct ifaltq_subque { + struct lwkt_serialize ifsq_lock; + int ifsq_index; + + struct ifaltq *ifsq_altq; + struct ifnet *ifsq_ifp; + void *ifsq_hw_priv; /* hw private data */ + + /* fields compatible with IFQ_ macros */ + struct mbuf *ifq_head; + struct mbuf *ifq_tail; + int ifq_len; + int ifq_maxlen; + + ifsq_enqueue_t ifsq_enqueue; + ifsq_dequeue_t ifsq_dequeue; + ifsq_request_t ifsq_request; + + struct mbuf *ifsq_prepended;/* mbuf dequeued, but not yet xmit */ + int ifsq_started; /* ifnet.if_start interlock */ + int ifsq_hw_oactive;/* hw too busy, protected by driver */ + int ifsq_cpuid; /* owner cpu */ + struct ifsubq_stage *ifsq_stage;/* packet staging information */ + struct netmsg_base *ifsq_ifstart_nmsg; + /* percpu msgs to sched if_start */ } __cachealign; -#define IFQ_STAGE_FLAG_QUED 0x1 -#define IFQ_STAGE_FLAG_SCHED 0x2 +#ifdef _KERNEL +#define ALTQ_SQ_ASSERT_LOCKED(ifsq) ASSERT_SERIALIZED(&(ifsq)->ifsq_lock) +#define ALTQ_SQ_LOCK_INIT(ifsq) lwkt_serialize_init(&(ifsq)->ifsq_lock) +#define ALTQ_SQ_LOCK(ifsq) \ + lwkt_serialize_adaptive_enter(&(ifsq)->ifsq_lock) +#define ALTQ_SQ_UNLOCK(ifsq) lwkt_serialize_exit(&(ifsq)->ifsq_lock) +#endif /* * Structure defining a queue for a network interface. */ struct ifaltq { - /* fields compatible with struct ifqueue */ - struct mbuf *ifq_head; - struct mbuf *ifq_tail; - int ifq_len; - int ifq_maxlen; - int ifq_drops; - /* alternate queueing related fields */ int altq_type; /* discipline type */ int altq_flags; /* flags (e.g. ready, in-use) */ void *altq_disc; /* for discipline-specific use */ struct ifnet *altq_ifp; /* back pointer to interface */ - int (*altq_enqueue)(struct ifaltq *, struct mbuf *, - struct altq_pktattr *); - struct mbuf *(*altq_dequeue)(struct ifaltq *, struct mbuf *, int); - int (*altq_request)(struct ifaltq *, int, void *); - /* classifier fields */ void *altq_clfier; /* classifier-specific use */ void *(*altq_classify)(struct ifaltq *, struct mbuf *, struct altq_pktattr *); + void *altq_unused; /* token bucket regulator */ struct tb_regulator *altq_tbr; - struct lwkt_serialize altq_lock; - struct mbuf *altq_prepended; /* mbuf dequeued, but not yet xmit */ - int altq_started; /* ifnet.if_start interlock */ - int altq_hw_oactive; /* hw too busy, protected by driver */ - int altq_cpuid; /* owner cpu */ - struct ifaltq_stage *altq_stage; - struct netmsg_base *altq_ifstart_nmsg; - /* percpu msgs to sched if_start */ + /* Sub-queues */ + int altq_subq_cnt; + struct ifaltq_subque *altq_subq; + + int altq_maxlen; }; -#define ALTQ_ASSERT_LOCKED(ifq) ASSERT_SERIALIZED(&(ifq)->altq_lock) -#define ALTQ_LOCK_INIT(ifq) lwkt_serialize_init(&(ifq)->altq_lock) -#define ALTQ_LOCK(ifq) lwkt_serialize_adaptive_enter(&(ifq)->altq_lock) -#define ALTQ_UNLOCK(ifq) lwkt_serialize_exit(&(ifq)->altq_lock) +#ifdef _KERNRL +/* COMPAT */ +#define ALTQ_LOCK(ifq) \ + ALTQ_SQ_LOCK(&(ifq)->altq_subq[ALTQ_SUBQ_INDEX_DEFAULT]) +/* COMPAT */ +#define ALTQ_UNLOCK(ifq) \ + ALTQ_SQ_UNLOCK(&(ifq)->altq_subq[ALTQ_SUBQ_INDEX_DEFAULT]) +#endif #ifdef _KERNEL @@ -149,15 +185,12 @@ struct tb_regulator { #define ALTRQ_PURGE 1 /* purge all packets */ int altq_attach(struct ifaltq *, int, void *, - int (*)(struct ifaltq *, struct mbuf *, struct altq_pktattr *), - struct mbuf *(*)(struct ifaltq *, struct mbuf *, int), - int (*)(struct ifaltq *, int, void *), - void *, void *(*)(struct ifaltq *, struct mbuf *, - struct altq_pktattr *)); + ifsq_enqueue_t, ifsq_dequeue_t, ifsq_request_t, void *, + void *(*)(struct ifaltq *, struct mbuf *, struct altq_pktattr *)); int altq_detach(struct ifaltq *); int altq_enable(struct ifaltq *); int altq_disable(struct ifaltq *); -struct mbuf *tbr_dequeue(struct ifaltq *, struct mbuf *, int); +struct mbuf *tbr_dequeue(struct ifaltq_subque *, struct mbuf *, int); extern int (*altq_input)(struct mbuf *, int); #endif /* _KERNEL */ diff --git a/sys/net/bridge/if_bridge.c b/sys/net/bridge/if_bridge.c index 955b85d9bd..1389b33bd3 100644 --- a/sys/net/bridge/if_bridge.c +++ b/sys/net/bridge/if_bridge.c @@ -368,7 +368,7 @@ static void bridge_ifdetach(void *, struct ifnet *); static void bridge_init(void *); static int bridge_from_us(struct bridge_softc *, struct ether_header *); static void bridge_stop(struct ifnet *); -static void bridge_start(struct ifnet *); +static void bridge_start(struct ifnet *, struct ifaltq_subque *); static struct mbuf *bridge_input(struct ifnet *, struct mbuf *); static int bridge_output(struct ifnet *, struct mbuf *); static struct ifnet *bridge_interface(void *if_bridge); @@ -2297,19 +2297,20 @@ bridge_interface(void *if_bridge) * Start output on a bridge. */ static void -bridge_start(struct ifnet *ifp) +bridge_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct bridge_softc *sc = ifp->if_softc; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_IFNET_SERIALIZED_TX(ifp); - ifq_set_oactive(&ifp->if_snd); + ifsq_set_oactive(ifsq); for (;;) { struct ifnet *dst_if = NULL; struct ether_header *eh; struct mbuf *m; - m = ifq_dequeue(&ifp->if_snd, NULL); + m = ifsq_dequeue(ifsq, NULL); if (m == NULL) break; mbuftrackid(m, 75); @@ -2347,7 +2348,7 @@ bridge_start(struct ifnet *ifp) else bridge_enqueue(dst_if, m); } - ifq_clr_oactive(&ifp->if_snd); + ifsq_clr_oactive(ifsq); } /* diff --git a/sys/net/ef/if_ef.c b/sys/net/ef/if_ef.c index fb7f319b6d..453176c03d 100644 --- a/sys/net/ef/if_ef.c +++ b/sys/net/ef/if_ef.c @@ -114,7 +114,7 @@ static int ef_attach(struct efnet *sc); static int ef_detach(struct efnet *sc); static void ef_init(void *); static int ef_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); -static void ef_start(struct ifnet *); +static void ef_start(struct ifnet *, struct ifaltq_subque *); static int ef_input(struct ifnet*, const struct ether_header *, struct mbuf *); static int ef_output(struct ifnet *ifp, struct mbuf **mp, struct sockaddr *dst, short *tp, int *hlen); @@ -201,30 +201,32 @@ ef_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cr) * place. */ static void -ef_start(struct ifnet *ifp) +ef_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct efnet *sc = (struct efnet*)ifp->if_softc; struct ifnet *p; struct mbuf *m; + struct ifaltq_subque *p_ifsq; - ifq_set_oactive(&ifp->if_snd); + ifsq_set_oactive(ifsq); p = sc->ef_ifp; + p_ifsq = ifq_get_subq_default(&p->if_snd); EFDEBUG("\n"); for (;;) { - m = ifq_dequeue(&ifp->if_snd, NULL); + m = ifsq_dequeue(ifsq, NULL); if (m == NULL) break; BPF_MTAP(ifp, m); - ifq_enqueue(&p->if_snd, m, NULL); - if (!ifq_is_oactive(&p->if_snd)) { - p->if_start(p); + ifsq_enqueue(p_ifsq, m, NULL); + if (!ifsq_is_oactive(p_ifsq)) { + p->if_start(p, p_ifsq); ifp->if_opackets++; } } - ifq_clr_oactive(&ifp->if_snd); + ifsq_clr_oactive(ifsq); return; } diff --git a/sys/net/if.c b/sys/net/if.c index c389dbcd89..be9da89850 100644 --- a/sys/net/if.c +++ b/sys/net/if.c @@ -103,8 +103,8 @@ struct netmsg_ifaddr { int tail; }; -struct ifaltq_stage_head { - TAILQ_HEAD(, ifaltq_stage) ifqs_head; +struct ifsubq_stage_head { + TAILQ_HEAD(, ifsubq_stage) stg_head; } __cachealign; /* @@ -130,10 +130,10 @@ extern void nd6_setmtu(struct ifnet *); SYSCTL_NODE(_net, PF_LINK, link, CTLFLAG_RW, 0, "Link layers"); SYSCTL_NODE(_net_link, 0, generic, CTLFLAG_RW, 0, "Generic link-management"); -static int ifq_stage_cntmax = 4; -TUNABLE_INT("net.link.stage_cntmax", &ifq_stage_cntmax); +static int ifsq_stage_cntmax = 4; +TUNABLE_INT("net.link.stage_cntmax", &ifsq_stage_cntmax); SYSCTL_INT(_net_link, OID_AUTO, stage_cntmax, CTLFLAG_RW, - &ifq_stage_cntmax, 0, "ifq staging packet count max"); + &ifsq_stage_cntmax, 0, "ifq staging packet count max"); SYSINIT(interfaces, SI_SUB_PROTO_IF, SI_ORDER_FIRST, ifinit, NULL) /* Must be after netisr_init */ @@ -155,8 +155,9 @@ int if_index = 0; struct ifnet **ifindex2ifnet = NULL; static struct thread ifnet_threads[MAXCPU]; -static struct ifaltq_stage_head ifq_stage_heads[MAXCPU]; +static struct ifsubq_stage_head ifsubq_stage_heads[MAXCPU]; +#ifdef notyet #define IFQ_KTR_STRING "ifq=%p" #define IFQ_KTR_ARGS struct ifaltq *ifq #ifndef KTR_IFQ @@ -184,6 +185,7 @@ KTR_INFO(KTR_IF_START, if_start, contend_sched, 3, KTR_INFO(KTR_IF_START, if_start, chase_sched, 4, IF_START_KTR_STRING, IF_START_KTR_ARGS); #define logifstart(name, arg) KTR_LOG(if_start_ ## name, arg) +#endif TAILQ_HEAD(, ifg_group) ifg_head = TAILQ_HEAD_INITIALIZER(ifg_head); @@ -203,7 +205,7 @@ ifinit(void *dummy) crit_enter(); TAILQ_FOREACH(ifp, &ifnet, if_link) { - if (ifp->if_snd.ifq_maxlen == 0) { + if (ifp->if_snd.altq_maxlen == 0) { if_printf(ifp, "XXX: driver didn't set ifq_maxlen\n"); ifq_set_maxlen(&ifp->if_snd, ifqmaxlen); } @@ -214,10 +216,10 @@ ifinit(void *dummy) } static void -ifq_ifstart_ipifunc(void *arg) +ifsq_ifstart_ipifunc(void *arg) { - struct ifaltq *ifq = arg; - struct lwkt_msg *lmsg = ifq_get_ifstart_lmsg(ifq, mycpuid); + struct ifaltq_subque *ifsq = arg; + struct lwkt_msg *lmsg = ifsq_get_ifstart_lmsg(ifsq, mycpuid); crit_enter(); if (lmsg->ms_flags & MSGF_DONE) @@ -226,49 +228,49 @@ ifq_ifstart_ipifunc(void *arg) } static __inline void -ifq_stage_remove(struct ifaltq_stage_head *head, struct ifaltq_stage *stage) +ifsq_stage_remove(struct ifsubq_stage_head *head, struct ifsubq_stage *stage) { - KKASSERT(stage->ifqs_flags & IFQ_STAGE_FLAG_QUED); - TAILQ_REMOVE(&head->ifqs_head, stage, ifqs_link); - stage->ifqs_flags &= ~(IFQ_STAGE_FLAG_QUED | IFQ_STAGE_FLAG_SCHED); - stage->ifqs_cnt = 0; - stage->ifqs_len = 0; + KKASSERT(stage->stg_flags & IFSQ_STAGE_FLAG_QUED); + TAILQ_REMOVE(&head->stg_head, stage, stg_link); + stage->stg_flags &= ~(IFSQ_STAGE_FLAG_QUED | IFSQ_STAGE_FLAG_SCHED); + stage->stg_cnt = 0; + stage->stg_len = 0; } static __inline void -ifq_stage_insert(struct ifaltq_stage_head *head, struct ifaltq_stage *stage) +ifsq_stage_insert(struct ifsubq_stage_head *head, struct ifsubq_stage *stage) { - KKASSERT((stage->ifqs_flags & - (IFQ_STAGE_FLAG_QUED | IFQ_STAGE_FLAG_SCHED)) == 0); - stage->ifqs_flags |= IFQ_STAGE_FLAG_QUED; - TAILQ_INSERT_TAIL(&head->ifqs_head, stage, ifqs_link); + KKASSERT((stage->stg_flags & + (IFSQ_STAGE_FLAG_QUED | IFSQ_STAGE_FLAG_SCHED)) == 0); + stage->stg_flags |= IFSQ_STAGE_FLAG_QUED; + TAILQ_INSERT_TAIL(&head->stg_head, stage, stg_link); } /* * Schedule ifnet.if_start on ifnet's CPU */ static void -ifq_ifstart_schedule(struct ifaltq *ifq, int force) +ifsq_ifstart_schedule(struct ifaltq_subque *ifsq, int force) { int cpu; if (!force && curthread->td_type == TD_TYPE_NETISR && - ifq_stage_cntmax > 0) { - struct ifaltq_stage *stage = ifq_get_stage(ifq, mycpuid); - - stage->ifqs_cnt = 0; - stage->ifqs_len = 0; - if ((stage->ifqs_flags & IFQ_STAGE_FLAG_QUED) == 0) - ifq_stage_insert(&ifq_stage_heads[mycpuid], stage); - stage->ifqs_flags |= IFQ_STAGE_FLAG_SCHED; + ifsq_stage_cntmax > 0) { + struct ifsubq_stage *stage = ifsq_get_stage(ifsq, mycpuid); + + stage->stg_cnt = 0; + stage->stg_len = 0; + if ((stage->stg_flags & IFSQ_STAGE_FLAG_QUED) == 0) + ifsq_stage_insert(&ifsubq_stage_heads[mycpuid], stage); + stage->stg_flags |= IFSQ_STAGE_FLAG_SCHED; return; } - cpu = ifq_get_cpuid(ifq); + cpu = ifsq_get_cpuid(ifsq); if (cpu != mycpuid) - lwkt_send_ipiq(globaldata_find(cpu), ifq_ifstart_ipifunc, ifq); + lwkt_send_ipiq(globaldata_find(cpu), ifsq_ifstart_ipifunc, ifsq); else - ifq_ifstart_ipifunc(ifq); + ifsq_ifstart_ipifunc(ifsq); } /* @@ -277,14 +279,14 @@ ifq_ifstart_schedule(struct ifaltq *ifq, int force) * if ifnet.if_start does not need to be scheduled */ static __inline int -ifq_ifstart_need_schedule(struct ifaltq *ifq, int running) +ifsq_ifstart_need_schedule(struct ifaltq_subque *ifsq, int running) { - if (!running || ifq_is_empty(ifq) + if (!running || ifsq_is_empty(ifsq) #ifdef ALTQ - || ifq->altq_tbr != NULL + || ifsq->ifsq_altq->altq_tbr != NULL #endif ) { - ALTQ_LOCK(ifq); + ALTQ_SQ_LOCK(ifsq); /* * ifnet.if_start interlock is released, if: * 1) Hardware can not take any packets, due to @@ -300,45 +302,43 @@ ifq_ifstart_need_schedule(struct ifaltq *ifq, int running) * dequeueing. * TBR callout will call ifnet.if_start */ - if (!running || !ifq_data_ready(ifq)) { - ifq_clr_started(ifq); - ALTQ_UNLOCK(ifq); + if (!running || !ifsq_data_ready(ifsq)) { + ifsq_clr_started(ifsq); + ALTQ_SQ_UNLOCK(ifsq); return 0; } - ALTQ_UNLOCK(ifq); + ALTQ_SQ_UNLOCK(ifsq); } return 1; } static void -ifq_ifstart_dispatch(netmsg_t msg) +ifsq_ifstart_dispatch(netmsg_t msg) { struct lwkt_msg *lmsg = &msg->base.lmsg; - struct ifaltq *ifq = lmsg->u.ms_resultp; - struct ifnet *ifp = ifq->altq_ifp; + struct ifaltq_subque *ifsq = lmsg->u.ms_resultp; + struct ifnet *ifp = ifsq_get_ifp(ifsq); int running = 0, need_sched; crit_enter(); lwkt_replymsg(lmsg, 0); /* reply ASAP */ crit_exit(); - if (mycpuid != ifq_get_cpuid(ifq)) { + if (mycpuid != ifsq_get_cpuid(ifsq)) { /* * We need to chase the ifnet CPU change. */ - logifstart(chase_sched, ifp); - ifq_ifstart_schedule(ifq, 1); + ifsq_ifstart_schedule(ifsq, 1); return; } ifnet_serialize_tx(ifp); - if ((ifp->if_flags & IFF_RUNNING) && !ifq_is_oactive(ifq)) { - logifstart(run, ifp); - ifp->if_start(ifp); - if ((ifp->if_flags & IFF_RUNNING) && !ifq_is_oactive(ifq)) + if ((ifp->if_flags & IFF_RUNNING) && !ifsq_is_oactive(ifsq)) { + ifp->if_start(ifp, ifsq); + if ((ifp->if_flags & IFF_RUNNING) && !ifsq_is_oactive(ifsq)) running = 1; } - need_sched = ifq_ifstart_need_schedule(ifq, running); + need_sched = ifsq_ifstart_need_schedule(ifsq, running); ifnet_deserialize_tx(ifp); if (need_sched) { @@ -347,51 +347,59 @@ ifq_ifstart_dispatch(netmsg_t msg) * scheduled on ifnet's CPU, and we keep going. * NOTE: ifnet.if_start interlock is not released. */ - logifstart(sched, ifp); - ifq_ifstart_schedule(ifq, 0); + ifsq_ifstart_schedule(ifsq, 0); } } /* Device driver ifnet.if_start helper function */ void -if_devstart(struct ifnet *ifp) +ifsq_devstart(struct ifaltq_subque *ifsq) { - struct ifaltq *ifq = &ifp->if_snd; + struct ifnet *ifp = ifsq_get_ifp(ifsq); int running = 0; ASSERT_IFNET_SERIALIZED_TX(ifp); - ALTQ_LOCK(ifq); - if (ifq_is_started(ifq) || !ifq_data_ready(ifq)) { - logifstart(avoid, ifp); - ALTQ_UNLOCK(ifq); + ALTQ_SQ_LOCK(ifsq); + if (ifsq_is_started(ifsq) || !ifsq_data_ready(ifsq)) { + ALTQ_SQ_UNLOCK(ifsq); return; } - ifq_set_started(ifq); - ALTQ_UNLOCK(ifq); + ifsq_set_started(ifsq); + ALTQ_SQ_UNLOCK(ifsq); - logifstart(run, ifp); - ifp->if_start(ifp); + ifp->if_start(ifp, ifsq); - if ((ifp->if_flags & IFF_RUNNING) && !ifq_is_oactive(ifq)) + if ((ifp->if_flags & IFF_RUNNING) && !ifsq_is_oactive(ifsq)) running = 1; - if (ifq_ifstart_need_schedule(ifq, running)) { + if (ifsq_ifstart_need_schedule(ifsq, running)) { /* * More data need to be transmitted, ifnet.if_start is * scheduled on ifnet's CPU, and we keep going. * NOTE: ifnet.if_start interlock is not released. */ - logifstart(sched, ifp); - ifq_ifstart_schedule(ifq, 0); + ifsq_ifstart_schedule(ifsq, 0); } } +void +if_devstart(struct ifnet *ifp) +{ + ifsq_devstart(ifq_get_subq_default(&ifp->if_snd)); +} + /* Device driver ifnet.if_start schedule helper function */ +void +ifsq_devstart_sched(struct ifaltq_subque *ifsq) +{ + ifsq_ifstart_schedule(ifsq, 1); +} + void if_devstart_sched(struct ifnet *ifp) { - ifq_ifstart_schedule(&ifp->if_snd, 1); + ifsq_devstart_sched(ifq_get_subq_default(&ifp->if_snd)); } static void @@ -439,7 +447,7 @@ if_attach(struct ifnet *ifp, lwkt_serialize_t serializer) struct sockaddr_dl *sdl; struct ifaddr *ifa; struct ifaltq *ifq; - int i; + int i, q; static int if_indexlim = 8; @@ -564,27 +572,50 @@ if_attach(struct ifnet *ifp, lwkt_serialize_t serializer) ifq->altq_flags &= ALTQF_CANTCHANGE; ifq->altq_tbr = NULL; ifq->altq_ifp = ifp; - ifq->altq_started = 0; - ifq->altq_prepended = NULL; - ALTQ_LOCK_INIT(ifq); - ifq_set_classic(ifq); - ifq_set_cpuid(ifq, 0); - ifq->altq_stage = - kmalloc_cachealign(ncpus * sizeof(struct ifaltq_stage), + if (ifq->altq_subq_cnt <= 0) + ifq->altq_subq_cnt = 1; + ifq->altq_subq = kmalloc_cachealign( + ifq->altq_subq_cnt * sizeof(struct ifaltq_subque), M_DEVBUF, M_WAITOK | M_ZERO); - for (i = 0; i < ncpus; ++i) - ifq->altq_stage[i].ifqs_altq = ifq; - ifq->altq_ifstart_nmsg = - kmalloc(ncpus * sizeof(*ifq->altq_ifstart_nmsg), - M_LWKTMSG, M_WAITOK); - for (i = 0; i < ncpus; ++i) { - netmsg_init(&ifq->altq_ifstart_nmsg[i], NULL, - &netisr_adone_rport, 0, ifq_ifstart_dispatch); - ifq->altq_ifstart_nmsg[i].lmsg.u.ms_resultp = ifq; + if (ifq->altq_maxlen == 0) { + if_printf(ifp, "driver didn't set ifq_maxlen\n"); + ifq_set_maxlen(ifq, ifqmaxlen); } + for (q = 0; q < ifq->altq_subq_cnt; ++q) { + struct ifaltq_subque *ifsq = &ifq->altq_subq[q]; + + ALTQ_SQ_LOCK_INIT(ifsq); + ifsq->ifsq_index = q; + + ifsq->ifsq_altq = ifq; + ifsq->ifsq_ifp = ifp; + + ifsq->ifq_maxlen = ifq->altq_maxlen; + ifsq->ifsq_prepended = NULL; + ifsq->ifsq_started = 0; + ifsq->ifsq_hw_oactive = 0; + ifsq_set_cpuid(ifsq, 0); + + ifsq->ifsq_stage = + kmalloc_cachealign(ncpus * sizeof(struct ifsubq_stage), + M_DEVBUF, M_WAITOK | M_ZERO); + for (i = 0; i < ncpus; ++i) + ifsq->ifsq_stage[i].stg_subq = ifsq; + + ifsq->ifsq_ifstart_nmsg = + kmalloc(ncpus * sizeof(struct netmsg_base), + M_LWKTMSG, M_WAITOK); + for (i = 0; i < ncpus; ++i) { + netmsg_init(&ifsq->ifsq_ifstart_nmsg[i], NULL, + &netisr_adone_rport, 0, ifsq_ifstart_dispatch); + ifsq->ifsq_ifstart_nmsg[i].lmsg.u.ms_resultp = ifsq; + } + } + ifq_set_classic(ifq); + if (!SLIST_EMPTY(&domains)) if_attachdomain1(ifp); @@ -685,10 +716,15 @@ static void ifq_stage_detach_handler(netmsg_t nmsg) { struct ifaltq *ifq = nmsg->lmsg.u.ms_resultp; - struct ifaltq_stage *stage = ifq_get_stage(ifq, mycpuid); + int q; - if (stage->ifqs_flags & IFQ_STAGE_FLAG_QUED) - ifq_stage_remove(&ifq_stage_heads[mycpuid], stage); + for (q = 0; q < ifq->altq_subq_cnt; ++q) { + struct ifaltq_subque *ifsq = &ifq->altq_subq[q]; + struct ifsubq_stage *stage = ifsq_get_stage(ifsq, mycpuid); + + if (stage->stg_flags & IFSQ_STAGE_FLAG_QUED) + ifsq_stage_remove(&ifsubq_stage_heads[mycpuid], stage); + } lwkt_replymsg(&nmsg->lmsg, 0); } @@ -714,7 +750,7 @@ void if_detach(struct ifnet *ifp) { struct radix_node_head *rnh; - int i; + int i, q; int cpu, origcpu; struct domain *dp; @@ -812,8 +848,12 @@ if_detach(struct ifnet *ifp) lwkt_synchronize_ipiqs("if_detach"); ifq_stage_detach(&ifp->if_snd); - kfree(ifp->if_snd.altq_ifstart_nmsg, M_LWKTMSG); - kfree(ifp->if_snd.altq_stage, M_DEVBUF); + for (q = 0; q < ifp->if_snd.altq_subq_cnt; ++q) { + struct ifaltq_subque *ifsq = &ifp->if_snd.altq_subq[q]; + + kfree(ifsq->ifsq_ifstart_nmsg, M_LWKTMSG); + kfree(ifsq->ifsq_stage, M_DEVBUF); + } crit_exit(); } @@ -2391,37 +2431,49 @@ if_free(struct ifnet *ifp) void ifq_set_classic(struct ifaltq *ifq) { - ifq->altq_enqueue = ifq_classic_enqueue; - ifq->altq_dequeue = ifq_classic_dequeue; - ifq->altq_request = ifq_classic_request; + ifq_set_methods(ifq, ifsq_classic_enqueue, ifsq_classic_dequeue, + ifsq_classic_request); +} + +void +ifq_set_methods(struct ifaltq *ifq, ifsq_enqueue_t enqueue, + ifsq_dequeue_t dequeue, ifsq_request_t request) +{ + int q; + + for (q = 0; q < ifq->altq_subq_cnt; ++q) { + struct ifaltq_subque *ifsq = &ifq->altq_subq[q]; + + ifsq->ifsq_enqueue = enqueue; + ifsq->ifsq_dequeue = dequeue; + ifsq->ifsq_request = request; + } } int -ifq_classic_enqueue(struct ifaltq *ifq, struct mbuf *m, - struct altq_pktattr *pa __unused) +ifsq_classic_enqueue(struct ifaltq_subque *ifsq, struct mbuf *m, + struct altq_pktattr *pa __unused) { - logifq(enqueue, ifq); - if (IF_QFULL(ifq)) { + if (IF_QFULL(ifsq)) { m_freem(m); return(ENOBUFS); } else { - IF_ENQUEUE(ifq, m); + IF_ENQUEUE(ifsq, m); return(0); } } struct mbuf * -ifq_classic_dequeue(struct ifaltq *ifq, struct mbuf *mpolled, int op) +ifsq_classic_dequeue(struct ifaltq_subque *ifsq, struct mbuf *mpolled, int op) { struct mbuf *m; switch (op) { case ALTDQ_POLL: - IF_POLL(ifq, m); + IF_POLL(ifsq, m); break; case ALTDQ_REMOVE: - logifq(dequeue, ifq); - IF_DEQUEUE(ifq, m); + IF_DEQUEUE(ifsq, m); break; default: panic("unsupported ALTQ dequeue op: %d", op); @@ -2431,11 +2483,11 @@ ifq_classic_dequeue(struct ifaltq *ifq, struct mbuf *mpolled, int op) } int -ifq_classic_request(struct ifaltq *ifq, int req, void *arg) +ifsq_classic_request(struct ifaltq_subque *ifsq, int req, void *arg) { switch (req) { case ALTRQ_PURGE: - IF_DRAIN(ifq); + IF_DRAIN(ifsq); break; default: panic("unsupported ALTQ request: %d", req); @@ -2444,9 +2496,9 @@ ifq_classic_request(struct ifaltq *ifq, int req, void *arg) } static void -ifq_ifstart_try(struct ifaltq *ifq, int force_sched) +ifsq_ifstart_try(struct ifaltq_subque *ifsq, int force_sched) { - struct ifnet *ifp = ifq->altq_ifp; + struct ifnet *ifp = ifsq_get_ifp(ifsq); int running = 0, need_sched; /* @@ -2460,18 +2512,16 @@ ifq_ifstart_try(struct ifaltq *ifq, int force_sched) * ifnet.if_start is scheduled on ifnet's * CPU, and we keep going. */ - logifstart(contend_sched, ifp); - ifq_ifstart_schedule(ifq, 1); + ifsq_ifstart_schedule(ifsq, 1); return; } - if ((ifp->if_flags & IFF_RUNNING) && !ifq_is_oactive(ifq)) { - logifstart(run, ifp); - ifp->if_start(ifp); - if ((ifp->if_flags & IFF_RUNNING) && !ifq_is_oactive(ifq)) + if ((ifp->if_flags & IFF_RUNNING) && !ifsq_is_oactive(ifsq)) { + ifp->if_start(ifp, ifsq); + if ((ifp->if_flags & IFF_RUNNING) && !ifsq_is_oactive(ifsq)) running = 1; } - need_sched = ifq_ifstart_need_schedule(ifq, running); + need_sched = ifsq_ifstart_need_schedule(ifsq, running); ifnet_deserialize_tx(ifp); @@ -2481,15 +2531,14 @@ ifq_ifstart_try(struct ifaltq *ifq, int force_sched) * scheduled on ifnet's CPU, and we keep going. * NOTE: ifnet.if_start interlock is not released. */ - logifstart(sched, ifp); - ifq_ifstart_schedule(ifq, force_sched); + ifsq_ifstart_schedule(ifsq, force_sched); } } /* - * IFQ packets staging mechanism: + * IFSUBQ packets staging mechanism: * - * The packets enqueued into IFQ are staged to a certain amount before the + * The packets enqueued into IFSUBQ are staged to a certain amount before the * ifnet's if_start is called. In this way, the driver could avoid writing * to hardware registers upon every packet, instead, hardware registers * could be written when certain amount of packets are put onto hardware @@ -2500,34 +2549,39 @@ ifq_ifstart_try(struct ifaltq *ifq, int force_sched) * aggeregation is also mentioned by Luigi Rizzo's netmap paper * (http://info.iet.unipi.it/~luigi/netmap/). * - * IFQ packets staging is performed for two entry points into drivers's + * IFSUBQ packets staging is performed for two entry points into drivers's * transmission function: - * - Direct ifnet's if_start calling, i.e. ifq_ifstart_try() - * - ifnet's if_start scheduling, i.e. ifq_ifstart_schedule() + * - Direct ifnet's if_start calling, i.e. ifsq_ifstart_try() + * - ifnet's if_start scheduling, i.e. ifsq_ifstart_schedule() * - * IFQ packets staging will be stopped upon any of the following conditions: + * IFSUBQ packets staging will be stopped upon any of the following conditions: * - If the count of packets enqueued on the current CPU is great than or - * equal to ifq_stage_cntmax. (XXX this should be per-interface) + * equal to ifsq_stage_cntmax. (XXX this should be per-interface) * - If the total length of packets enqueued on the current CPU is great * than or equal to the hardware's MTU - max_protohdr. max_protohdr is * cut from the hardware's MTU mainly bacause a full TCP segment's size * is usually less than hardware's MTU. - * - ifq_ifstart_schedule() is not pending on the current CPU and if_start + * - ifsq_ifstart_schedule() is not pending on the current CPU and if_start * interlock (if_snd.altq_started) is not released. * - The if_start_rollup(), which is registered as low priority netisr * rollup function, is called; probably because no more work is pending * for netisr. * * NOTE: - * Currently IFQ packet staging is only performed in netisr threads. + * Currently IFSUBQ packet staging is only performed in netisr threads. */ int ifq_dispatch(struct ifnet *ifp, struct mbuf *m, struct altq_pktattr *pa) { struct ifaltq *ifq = &ifp->if_snd; + struct ifaltq_subque *ifsq; int error, start = 0, len, mcast = 0, avoid_start = 0; - struct ifaltq_stage_head *head = NULL; - struct ifaltq_stage *stage = NULL; + struct ifsubq_stage_head *head = NULL; + struct ifsubq_stage *stage = NULL; + int qid = 0; /* XXX */ + + /* TODO find qid here */ + ifsq = &ifq->altq_subq[qid]; ASSERT_IFNET_NOT_SERIALIZED_TX(ifp); @@ -2536,32 +2590,32 @@ ifq_dispatch(struct ifnet *ifp, struct mbuf *m, struct altq_pktattr *pa) mcast = 1; if (curthread->td_type == TD_TYPE_NETISR) { - head = &ifq_stage_heads[mycpuid]; - stage = ifq_get_stage(ifq, mycpuid); + head = &ifsubq_stage_heads[mycpuid]; + stage = ifsq_get_stage(ifsq, mycpuid); - stage->ifqs_cnt++; - stage->ifqs_len += len; - if (stage->ifqs_cnt < ifq_stage_cntmax && - stage->ifqs_len < (ifp->if_mtu - max_protohdr)) + stage->stg_cnt++; + stage->stg_len += len; + if (stage->stg_cnt < ifsq_stage_cntmax && + stage->stg_len < (ifp->if_mtu - max_protohdr)) avoid_start = 1; } - ALTQ_LOCK(ifq); - error = ifq_enqueue_locked(ifq, m, pa); + ALTQ_SQ_LOCK(ifsq); + error = ifsq_enqueue_locked(ifsq, m, pa); if (error) { - if (!ifq_data_ready(ifq)) { - ALTQ_UNLOCK(ifq); + if (!ifsq_data_ready(ifsq)) { + ALTQ_SQ_UNLOCK(ifsq); return error; } avoid_start = 0; } - if (!ifq_is_started(ifq)) { + if (!ifsq_is_started(ifsq)) { if (avoid_start) { - ALTQ_UNLOCK(ifq); + ALTQ_SQ_UNLOCK(ifsq); KKASSERT(!error); - if ((stage->ifqs_flags & IFQ_STAGE_FLAG_QUED) == 0) - ifq_stage_insert(head, stage); + if ((stage->stg_flags & IFSQ_STAGE_FLAG_QUED) == 0) + ifsq_stage_insert(head, stage); ifp->if_obytes += len; if (mcast) @@ -2572,10 +2626,10 @@ ifq_dispatch(struct ifnet *ifp, struct mbuf *m, struct altq_pktattr *pa) /* * Hold the interlock of ifnet.if_start */ - ifq_set_started(ifq); + ifsq_set_started(ifsq); start = 1; } - ALTQ_UNLOCK(ifq); + ALTQ_SQ_UNLOCK(ifsq); if (!error) { ifp->if_obytes += len; @@ -2584,29 +2638,27 @@ ifq_dispatch(struct ifnet *ifp, struct mbuf *m, struct altq_pktattr *pa) } if (stage != NULL) { - if (!start && (stage->ifqs_flags & IFQ_STAGE_FLAG_SCHED)) { - KKASSERT(stage->ifqs_flags & IFQ_STAGE_FLAG_QUED); + if (!start && (stage->stg_flags & IFSQ_STAGE_FLAG_SCHED)) { + KKASSERT(stage->stg_flags & IFSQ_STAGE_FLAG_QUED); if (!avoid_start) { - ifq_stage_remove(head, stage); - ifq_ifstart_schedule(ifq, 1); + ifsq_stage_remove(head, stage); + ifsq_ifstart_schedule(ifsq, 1); } return error; } - if (stage->ifqs_flags & IFQ_STAGE_FLAG_QUED) { - ifq_stage_remove(head, stage); + if (stage->stg_flags & IFSQ_STAGE_FLAG_QUED) { + ifsq_stage_remove(head, stage); } else { - stage->ifqs_cnt = 0; - stage->ifqs_len = 0; + stage->stg_cnt = 0; + stage->stg_len = 0; } } - if (!start) { - logifstart(avoid, ifp); + if (!start) return error; - } - ifq_ifstart_try(ifq, 0); + ifsq_ifstart_try(ifsq, 0); return error; } @@ -2813,37 +2865,37 @@ ifnet_service_loop(void *arg __unused) static void if_start_rollup(void) { - struct ifaltq_stage_head *head = &ifq_stage_heads[mycpuid]; - struct ifaltq_stage *stage; + struct ifsubq_stage_head *head = &ifsubq_stage_heads[mycpuid]; + struct ifsubq_stage *stage; - while ((stage = TAILQ_FIRST(&head->ifqs_head)) != NULL) { - struct ifaltq *ifq = stage->ifqs_altq; + while ((stage = TAILQ_FIRST(&head->stg_head)) != NULL) { + struct ifaltq_subque *ifsq = stage->stg_subq; int is_sched = 0; - if (stage->ifqs_flags & IFQ_STAGE_FLAG_SCHED) + if (stage->stg_flags & IFSQ_STAGE_FLAG_SCHED) is_sched = 1; - ifq_stage_remove(head, stage); + ifsq_stage_remove(head, stage); if (is_sched) { - ifq_ifstart_schedule(ifq, 1); + ifsq_ifstart_schedule(ifsq, 1); } else { int start = 0; - ALTQ_LOCK(ifq); - if (!ifq_is_started(ifq)) { + ALTQ_SQ_LOCK(ifsq); + if (!ifsq_is_started(ifsq)) { /* * Hold the interlock of ifnet.if_start */ - ifq_set_started(ifq); + ifsq_set_started(ifsq); start = 1; } - ALTQ_UNLOCK(ifq); + ALTQ_SQ_UNLOCK(ifsq); if (start) - ifq_ifstart_try(ifq, 1); + ifsq_ifstart_try(ifsq, 1); } - KKASSERT((stage->ifqs_flags & - (IFQ_STAGE_FLAG_QUED | IFQ_STAGE_FLAG_SCHED)) == 0); + KKASSERT((stage->stg_flags & + (IFSQ_STAGE_FLAG_QUED | IFSQ_STAGE_FLAG_SCHED)) == 0); } } @@ -2863,7 +2915,7 @@ ifnetinit(void *dummy __unused) } for (i = 0; i < ncpus; ++i) - TAILQ_INIT(&ifq_stage_heads[i].ifqs_head); + TAILQ_INIT(&ifsubq_stage_heads[i].stg_head); netisr_register_rollup(if_start_rollup, NETISR_ROLLUP_PRIO_IFSTART); } @@ -2940,5 +2992,5 @@ if_ring_count2(int cnt, int cnt_max) void ifq_set_maxlen(struct ifaltq *ifq, int len) { - ifq->ifq_maxlen = len + (ncpus * ifq_stage_cntmax); + ifq->altq_maxlen = len + (ncpus * ifsq_stage_cntmax); } diff --git a/sys/net/if_loop.c b/sys/net/if_loop.c index 7659f2755d..b90bbce1ec 100644 --- a/sys/net/if_loop.c +++ b/sys/net/if_loop.c @@ -85,7 +85,7 @@ static int looutput(struct ifnet *, struct mbuf *, struct sockaddr *, static int loioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void lortrequest(int, struct rtentry *, struct rt_addrinfo *); #ifdef ALTQ -static void lo_altqstart(struct ifnet *); +static void lo_altqstart(struct ifnet *, struct ifaltq_subque *); #endif PSEUDO_SET(loopattach, if_loop); @@ -226,6 +226,7 @@ rel: * a simplex interface). */ if (ifq_is_enabled(&ifp->if_snd) && ifp->if_start == lo_altqstart) { + struct ifaltq_subque *ifsq = ifq_get_subq_default(&ifp->if_snd); struct altq_pktattr pktattr; int32_t *afp; int error; @@ -248,9 +249,9 @@ rel: * be held for MPSAFE subsystems. */ crit_enter(); - error = ifq_enqueue(&ifp->if_snd, m, &pktattr); + error = ifsq_enqueue(ifsq, m, &pktattr); ifnet_serialize_tx(ifp); - ifp->if_start(ifp); + ifp->if_start(ifp, ifsq); ifnet_deserialize_tx(ifp); crit_exit(); return (error); @@ -289,7 +290,7 @@ rel: #ifdef ALTQ static void -lo_altqstart(struct ifnet *ifp) +lo_altqstart(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct mbuf *m; int32_t af, *afp; @@ -297,7 +298,7 @@ lo_altqstart(struct ifnet *ifp) while (1) { crit_enter(); - m = ifq_dequeue(&ifp->if_snd, NULL); + m = ifsq_dequeue(ifsq, NULL); crit_exit(); if (m == NULL) return; diff --git a/sys/net/if_mib.c b/sys/net/if_mib.c index 10d082b181..6c0365f777 100644 --- a/sys/net/if_mib.c +++ b/sys/net/if_mib.c @@ -103,9 +103,11 @@ sysctl_ifdata(SYSCTL_HANDLER_ARGS) /* XXX bad syntax! */ COPY(flags); COPY(data); #undef COPY + ifmd.ifmd_snd_maxlen = ifp->if_snd.altq_maxlen; +#ifdef notyet ifmd.ifmd_snd_len = ifp->if_snd.ifq_len; - ifmd.ifmd_snd_maxlen = ifp->if_snd.ifq_maxlen; ifmd.ifmd_snd_drops = ifp->if_snd.ifq_drops; +#endif error = SYSCTL_OUT(req, &ifmd, sizeof ifmd); if (error || !req->newptr) @@ -126,8 +128,11 @@ sysctl_ifdata(SYSCTL_HANDLER_ARGS) /* XXX bad syntax! */ #undef DONTCOPY #define COPY(fld) ifp->if_##fld = ifmd.ifmd_##fld COPY(data); + +#ifdef notyet ifp->if_snd.ifq_maxlen = ifmd.ifmd_snd_maxlen; ifp->if_snd.ifq_drops = ifmd.ifmd_snd_drops; +#endif #undef COPY break; diff --git a/sys/net/if_var.h b/sys/net/if_var.h index 474dac5eaa..5ec5b512bf 100644 --- a/sys/net/if_var.h +++ b/sys/net/if_var.h @@ -220,7 +220,7 @@ struct ifnet { void (*if_input) /* input routine from hardware driver */ (struct ifnet *, struct mbuf *); void (*if_start) /* initiate output routine */ - (struct ifnet *); + (struct ifnet *, struct ifaltq_subque *); int (*if_ioctl) /* ioctl routine */ (struct ifnet *, u_long, caddr_t, struct ucred *); void (*if_watchdog) /* timer routine */ @@ -847,8 +847,8 @@ struct ifaddr *ifaddr_byindex(unsigned short); struct ifmultiaddr *ifmaof_ifpforaddr(struct sockaddr *, struct ifnet *); int if_simloop(struct ifnet *ifp, struct mbuf *m, int af, int hlen); -void if_devstart(struct ifnet *ifp); -void if_devstart_sched(struct ifnet *ifp); +void if_devstart(struct ifnet *ifp); /* COMPAT */ +void if_devstart_sched(struct ifnet *ifp); /* COMPAT */ int if_ring_count2(int cnt, int cnt_max); #define IF_LLSOCKADDR(ifp) \ diff --git a/sys/net/ifq_var.h b/sys/net/ifq_var.h index 28be3e5dc8..d6c1e55991 100644 --- a/sys/net/ifq_var.h +++ b/sys/net/ifq_var.h @@ -57,18 +57,29 @@ #include #endif +#define ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq) \ + KASSERT(ifsq_get_ifp((ifsq)) == (ifp) && \ + ifsq_get_index((ifsq)) == ALTQ_SUBQ_INDEX_DEFAULT, \ + ("not ifp's default subqueue")); + struct ifaltq; /* * Support for "classic" ALTQ interfaces. */ -int ifq_classic_enqueue(struct ifaltq *, struct mbuf *, +int ifsq_classic_enqueue(struct ifaltq_subque *, struct mbuf *, struct altq_pktattr *); -struct mbuf *ifq_classic_dequeue(struct ifaltq *, struct mbuf *, int); -int ifq_classic_request(struct ifaltq *, int, void *); +struct mbuf *ifsq_classic_dequeue(struct ifaltq_subque *, struct mbuf *, + int); +int ifsq_classic_request(struct ifaltq_subque *, int, void *); void ifq_set_classic(struct ifaltq *); void ifq_set_maxlen(struct ifaltq *, int); +void ifq_set_methods(struct ifaltq *, ifsq_enqueue_t, + ifsq_dequeue_t, ifsq_request_t); + +void ifsq_devstart(struct ifaltq_subque *ifsq); +void ifsq_devstart_sched(struct ifaltq_subque *ifsq); /* * Dispatch a packet to an interface. @@ -122,52 +133,53 @@ ifq_set_ready(struct ifaltq *_ifq) * ALTQ lock must be held */ static __inline int -ifq_enqueue_locked(struct ifaltq *_ifq, struct mbuf *_m, - struct altq_pktattr *_pa) +ifsq_enqueue_locked(struct ifaltq_subque *_ifsq, struct mbuf *_m, + struct altq_pktattr *_pa) { #ifdef ALTQ - if (!ifq_is_enabled(_ifq)) - return ifq_classic_enqueue(_ifq, _m, _pa); + if (!ifq_is_enabled(_ifsq->ifsq_altq)) + return ifsq_classic_enqueue(_ifsq, _m, _pa); else #endif - return _ifq->altq_enqueue(_ifq, _m, _pa); + return _ifsq->ifsq_enqueue(_ifsq, _m, _pa); } static __inline int -ifq_enqueue(struct ifaltq *_ifq, struct mbuf *_m, struct altq_pktattr *_pa) +ifsq_enqueue(struct ifaltq_subque *_ifsq, struct mbuf *_m, + struct altq_pktattr *_pa) { int _error; - ALTQ_LOCK(_ifq); - _error = ifq_enqueue_locked(_ifq, _m, _pa); - ALTQ_UNLOCK(_ifq); + ALTQ_SQ_LOCK(_ifsq); + _error = ifsq_enqueue_locked(_ifsq, _m, _pa); + ALTQ_SQ_UNLOCK(_ifsq); return _error; } static __inline struct mbuf * -ifq_dequeue(struct ifaltq *_ifq, struct mbuf *_mpolled) +ifsq_dequeue(struct ifaltq_subque *_ifsq, struct mbuf *_mpolled) { struct mbuf *_m; - ALTQ_LOCK(_ifq); - if (_ifq->altq_prepended != NULL) { - _m = _ifq->altq_prepended; - _ifq->altq_prepended = NULL; - KKASSERT(_ifq->ifq_len > 0); - _ifq->ifq_len--; - ALTQ_UNLOCK(_ifq); + ALTQ_SQ_LOCK(_ifsq); + if (_ifsq->ifsq_prepended != NULL) { + _m = _ifsq->ifsq_prepended; + _ifsq->ifsq_prepended = NULL; + KKASSERT(_ifsq->ifq_len > 0); + _ifsq->ifq_len--; + ALTQ_SQ_UNLOCK(_ifsq); return _m; } #ifdef ALTQ - if (_ifq->altq_tbr != NULL) - _m = tbr_dequeue(_ifq, _mpolled, ALTDQ_REMOVE); - else if (!ifq_is_enabled(_ifq)) - _m = ifq_classic_dequeue(_ifq, _mpolled, ALTDQ_REMOVE); + if (_ifsq->ifsq_altq->altq_tbr != NULL) + _m = tbr_dequeue(_ifsq, _mpolled, ALTDQ_REMOVE); + else if (!ifq_is_enabled(_ifsq->ifsq_altq)) + _m = ifsq_classic_dequeue(_ifsq, _mpolled, ALTDQ_REMOVE); else #endif - _m = _ifq->altq_dequeue(_ifq, _mpolled, ALTDQ_REMOVE); - ALTQ_UNLOCK(_ifq); + _m = _ifsq->ifsq_dequeue(_ifsq, _mpolled, ALTDQ_REMOVE); + ALTQ_SQ_UNLOCK(_ifsq); return _m; } @@ -175,29 +187,29 @@ ifq_dequeue(struct ifaltq *_ifq, struct mbuf *_mpolled) * ALTQ lock must be held */ static __inline struct mbuf * -ifq_poll_locked(struct ifaltq *_ifq) +ifsq_poll_locked(struct ifaltq_subque *_ifsq) { - if (_ifq->altq_prepended != NULL) - return _ifq->altq_prepended; + if (_ifsq->ifsq_prepended != NULL) + return _ifsq->ifsq_prepended; #ifdef ALTQ - if (_ifq->altq_tbr != NULL) - return tbr_dequeue(_ifq, NULL, ALTDQ_POLL); - else if (!ifq_is_enabled(_ifq)) - return ifq_classic_dequeue(_ifq, NULL, ALTDQ_POLL); + if (_ifsq->ifsq_altq->altq_tbr != NULL) + return tbr_dequeue(_ifsq, NULL, ALTDQ_POLL); + else if (!ifq_is_enabled(_ifsq->ifsq_altq)) + return ifsq_classic_dequeue(_ifsq, NULL, ALTDQ_POLL); else #endif - return _ifq->altq_dequeue(_ifq, NULL, ALTDQ_POLL); + return _ifsq->ifsq_dequeue(_ifsq, NULL, ALTDQ_POLL); } static __inline struct mbuf * -ifq_poll(struct ifaltq *_ifq) +ifsq_poll(struct ifaltq_subque *_ifsq) { struct mbuf *_m; - ALTQ_LOCK(_ifq); - _m = ifq_poll_locked(_ifq); - ALTQ_UNLOCK(_ifq); + ALTQ_SQ_LOCK(_ifsq); + _m = ifsq_poll_locked(_ifsq); + ALTQ_SQ_UNLOCK(_ifsq); return _m; } @@ -205,29 +217,47 @@ ifq_poll(struct ifaltq *_ifq) * ALTQ lock must be held */ static __inline void -ifq_purge_locked(struct ifaltq *_ifq) +ifsq_purge_locked(struct ifaltq_subque *_ifsq) { - if (_ifq->altq_prepended != NULL) { - m_freem(_ifq->altq_prepended); - _ifq->altq_prepended = NULL; - KKASSERT(_ifq->ifq_len > 0); - _ifq->ifq_len--; + if (_ifsq->ifsq_prepended != NULL) { + m_freem(_ifsq->ifsq_prepended); + _ifsq->ifsq_prepended = NULL; + KKASSERT(_ifsq->ifq_len > 0); + _ifsq->ifq_len--; } #ifdef ALTQ - if (!ifq_is_enabled(_ifq)) - ifq_classic_request(_ifq, ALTRQ_PURGE, NULL); + if (!ifq_is_enabled(_ifsq->ifsq_altq)) + ifsq_classic_request(_ifsq, ALTRQ_PURGE, NULL); else #endif - _ifq->altq_request(_ifq, ALTRQ_PURGE, NULL); + _ifsq->ifsq_request(_ifsq, ALTRQ_PURGE, NULL); } static __inline void -ifq_purge(struct ifaltq *_ifq) +ifsq_purge(struct ifaltq_subque *_ifsq) +{ + ALTQ_SQ_LOCK(_ifsq); + ifsq_purge_locked(_ifsq); + ALTQ_SQ_UNLOCK(_ifsq); +} + +static __inline void +ifq_lock_all(struct ifaltq *_ifq) +{ + int _q; + + for (_q = 0; _q < _ifq->altq_subq_cnt; ++_q) + ALTQ_SQ_LOCK(&_ifq->altq_subq[_q]); +} + +static __inline void +ifq_unlock_all(struct ifaltq *_ifq) { - ALTQ_LOCK(_ifq); - ifq_purge_locked(_ifq); - ALTQ_UNLOCK(_ifq); + int _q; + + for (_q = _ifq->altq_subq_cnt - 1; _q >= 0; --_q) + ALTQ_SQ_UNLOCK(&_ifq->altq_subq[_q]); } /* @@ -236,69 +266,79 @@ ifq_purge(struct ifaltq *_ifq) static __inline void ifq_purge_all_locked(struct ifaltq *_ifq) { - /* XXX temporary */ - ifq_purge_locked(_ifq); + int _q; + + for (_q = 0; _q < _ifq->altq_subq_cnt; ++_q) + ifsq_purge_locked(&_ifq->altq_subq[_q]); } static __inline void ifq_purge_all(struct ifaltq *_ifq) { - ALTQ_LOCK(_ifq); + ifq_lock_all(_ifq); ifq_purge_all_locked(_ifq); - ALTQ_UNLOCK(_ifq); + ifq_unlock_all(_ifq); } static __inline void ifq_classify(struct ifaltq *_ifq, struct mbuf *_m, uint8_t _af, - struct altq_pktattr *_pa) + struct altq_pktattr *_pa) { #ifdef ALTQ - ALTQ_LOCK(_ifq); if (ifq_is_enabled(_ifq)) { _pa->pattr_af = _af; _pa->pattr_hdr = mtod(_m, caddr_t); - if (_ifq->altq_flags & ALTQF_CLASSIFY) - _ifq->altq_classify(_ifq, _m, _pa); + if (ifq_is_enabled(_ifq) && + (_ifq->altq_flags & ALTQF_CLASSIFY)) { + /* XXX default subqueue */ + struct ifaltq_subque *_ifsq = + &_ifq->altq_subq[ALTQ_SUBQ_INDEX_DEFAULT]; + + ALTQ_SQ_LOCK(_ifsq); + if (ifq_is_enabled(_ifq) && + (_ifq->altq_flags & ALTQF_CLASSIFY)) + _ifq->altq_classify(_ifq, _m, _pa); + ALTQ_SQ_UNLOCK(_ifsq); + } } - ALTQ_UNLOCK(_ifq); #endif } static __inline void -ifq_prepend(struct ifaltq *_ifq, struct mbuf *_m) +ifsq_prepend(struct ifaltq_subque *_ifsq, struct mbuf *_m) { - ALTQ_LOCK(_ifq); - KASSERT(_ifq->altq_prepended == NULL, ("pending prepended mbuf")); - _ifq->altq_prepended = _m; - _ifq->ifq_len++; - ALTQ_UNLOCK(_ifq); + ALTQ_SQ_LOCK(_ifsq); + KASSERT(_ifsq->ifsq_prepended == NULL, ("pending prepended mbuf")); + _ifsq->ifsq_prepended = _m; + _ifsq->ifq_len++; + ALTQ_SQ_UNLOCK(_ifsq); } /* * Interface TX serializer must be held */ static __inline void -ifq_set_oactive(struct ifaltq *_ifq) +ifsq_set_oactive(struct ifaltq_subque *_ifsq) { - _ifq->altq_hw_oactive = 1; + _ifsq->ifsq_hw_oactive = 1; } /* * Interface TX serializer must be held */ static __inline void -ifq_clr_oactive(struct ifaltq *_ifq) +ifsq_clr_oactive(struct ifaltq_subque *_ifsq) { - _ifq->altq_hw_oactive = 0; + _ifsq->ifsq_hw_oactive = 0; } /* * Interface TX serializer must be held */ static __inline int -ifq_is_oactive(const struct ifaltq *_ifq) +ifsq_is_oactive(const struct ifaltq_subque *_ifsq) { - return _ifq->altq_hw_oactive; + return _ifsq->ifsq_hw_oactive; } /* @@ -311,90 +351,191 @@ ifq_is_oactive(const struct ifaltq *_ifq) static __inline int ifq_handoff(struct ifnet *_ifp, struct mbuf *_m, struct altq_pktattr *_pa) { + struct ifaltq_subque *_ifsq; int _error; + int _qid = ALTQ_SUBQ_INDEX_DEFAULT; /* XXX default subqueue */ + + _ifsq = &_ifp->if_snd.altq_subq[_qid]; ASSERT_IFNET_SERIALIZED_TX(_ifp); - _error = ifq_enqueue(&_ifp->if_snd, _m, _pa); + _error = ifsq_enqueue(_ifsq, _m, _pa); if (_error == 0) { _ifp->if_obytes += _m->m_pkthdr.len; if (_m->m_flags & M_MCAST) _ifp->if_omcasts++; - if (!ifq_is_oactive(&_ifp->if_snd)) - (*_ifp->if_start)(_ifp); + if (!ifsq_is_oactive(_ifsq)) + (*_ifp->if_start)(_ifp, _ifsq); } return(_error); } static __inline int -ifq_is_empty(struct ifaltq *_ifq) +ifsq_is_empty(const struct ifaltq_subque *_ifsq) { - return(_ifq->ifq_len == 0); + return(_ifsq->ifq_len == 0); } /* * ALTQ lock must be held */ static __inline int -ifq_data_ready(struct ifaltq *_ifq) +ifsq_data_ready(struct ifaltq_subque *_ifsq) { #ifdef ALTQ - if (_ifq->altq_tbr != NULL) - return (ifq_poll_locked(_ifq) != NULL); + if (_ifsq->ifsq_altq->altq_tbr != NULL) + return (ifsq_poll_locked(_ifsq) != NULL); else #endif - return !ifq_is_empty(_ifq); + return !ifsq_is_empty(_ifsq); } /* * ALTQ lock must be held */ static __inline int -ifq_is_started(const struct ifaltq *_ifq) +ifsq_is_started(const struct ifaltq_subque *_ifsq) { - return _ifq->altq_started; + return _ifsq->ifsq_started; } /* * ALTQ lock must be held */ static __inline void -ifq_set_started(struct ifaltq *_ifq) +ifsq_set_started(struct ifaltq_subque *_ifsq) { - _ifq->altq_started = 1; + _ifsq->ifsq_started = 1; } /* * ALTQ lock must be held */ static __inline void -ifq_clr_started(struct ifaltq *_ifq) +ifsq_clr_started(struct ifaltq_subque *_ifsq) { - _ifq->altq_started = 0; + _ifsq->ifsq_started = 0; } -static __inline struct ifaltq_stage * -ifq_get_stage(struct ifaltq *_ifq, int cpuid) +static __inline struct ifsubq_stage * +ifsq_get_stage(struct ifaltq_subque *_ifsq, int _cpuid) { - return &_ifq->altq_stage[cpuid]; + return &_ifsq->ifsq_stage[_cpuid]; } static __inline int -ifq_get_cpuid(const struct ifaltq *_ifq) +ifsq_get_cpuid(const struct ifaltq_subque *_ifsq) { - return _ifq->altq_cpuid; + return _ifsq->ifsq_cpuid; } static __inline void -ifq_set_cpuid(struct ifaltq *_ifq, int cpuid) +ifsq_set_cpuid(struct ifaltq_subque *_ifsq, int _cpuid) { - KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid altq_cpuid %d", cpuid)); - _ifq->altq_cpuid = cpuid; + KASSERT(_cpuid >= 0 && _cpuid < ncpus, + ("invalid ifsq_cpuid %d", _cpuid)); + _ifsq->ifsq_cpuid = _cpuid; } static __inline struct lwkt_msg * -ifq_get_ifstart_lmsg(struct ifaltq *_ifq, int cpuid) +ifsq_get_ifstart_lmsg(struct ifaltq_subque *_ifsq, int _cpuid) +{ + return &_ifsq->ifsq_ifstart_nmsg[_cpuid].lmsg; +} + +static __inline int +ifsq_get_index(const struct ifaltq_subque *_ifsq) +{ + return _ifsq->ifsq_index; +} + +static __inline void +ifsq_set_priv(struct ifaltq_subque *_ifsq, void *_priv) +{ + _ifsq->ifsq_hw_priv = _priv; +} + +static __inline void * +ifsq_get_priv(const struct ifaltq_subque *_ifsq) +{ + return _ifsq->ifsq_hw_priv; +} + +static __inline struct ifnet * +ifsq_get_ifp(const struct ifaltq_subque *_ifsq) +{ + return _ifsq->ifsq_ifp; +} + +static __inline struct ifaltq_subque * +ifq_get_subq_default(const struct ifaltq *_ifq) +{ + return &_ifq->altq_subq[ALTQ_SUBQ_INDEX_DEFAULT]; +} + +static __inline struct ifaltq_subque * +ifq_get_subq(const struct ifaltq *_ifq, int _idx) +{ + KASSERT(_idx >= 0 && _idx < _ifq->altq_subq_cnt, + ("invalid qid %d", _idx)); + return &_ifq->altq_subq[_idx]; +} + +/* COMPAT */ +static __inline int +ifq_is_oactive(const struct ifaltq *_ifq) +{ + return ifsq_is_oactive(ifq_get_subq_default(_ifq)); +} + +/* COMPAT */ +static __inline void +ifq_set_oactive(struct ifaltq *_ifq) +{ + ifsq_set_oactive(ifq_get_subq_default(_ifq)); +} + +/* COMPAT */ +static __inline void +ifq_clr_oactive(struct ifaltq *_ifq) +{ + ifsq_clr_oactive(ifq_get_subq_default(_ifq)); +} + +/* COMPAT */ +static __inline int +ifq_is_empty(struct ifaltq *_ifq) +{ + return ifsq_is_empty(ifq_get_subq_default(_ifq)); +} + +/* COMPAT */ +static __inline void +ifq_purge(struct ifaltq *_ifq) +{ + ifsq_purge(ifq_get_subq_default(_ifq)); +} + +/* COMPAT */ +static __inline struct mbuf * +ifq_dequeue(struct ifaltq *_ifq, struct mbuf *_mpolled) +{ + return ifsq_dequeue(ifq_get_subq_default(_ifq), _mpolled); +} + +/* COMPAT */ +static __inline void +ifq_prepend(struct ifaltq *_ifq, struct mbuf *_m) +{ + ifsq_prepend(ifq_get_subq_default(_ifq), _m); +} + +/* COMPAT */ +static __inline void +ifq_set_cpuid(struct ifaltq *_ifq, int _cpuid) { - return &_ifq->altq_ifstart_nmsg[cpuid].lmsg; + KASSERT(_ifq->altq_subq_cnt == 1, + ("invalid subqueue count %d", _ifq->altq_subq_cnt)); + ifsq_set_cpuid(ifq_get_subq_default(_ifq), _cpuid); } #endif /* _KERNEL */ diff --git a/sys/net/pf/if_pflog.c b/sys/net/pf/if_pflog.c index 6f68fd5749..ab317d9f8c 100644 --- a/sys/net/pf/if_pflog.c +++ b/sys/net/pf/if_pflog.c @@ -88,7 +88,7 @@ int pflogoutput(struct ifnet *, struct mbuf *, struct sockaddr *, struct rtentry *); int pflogioctl(struct ifnet *, u_long, caddr_t, struct ucred *); void pflogrtrequest(int, struct rtentry *, struct sockaddr *); -void pflogstart(struct ifnet *); +void pflogstart(struct ifnet *, struct ifaltq_subque *); int pflog_clone_create(struct if_clone *, int, caddr_t); int pflog_clone_destroy(struct ifnet *); @@ -181,14 +181,15 @@ pflog_clone_destroy(struct ifnet *ifp) * Start output on the pflog interface. */ void -pflogstart(struct ifnet *ifp) +pflogstart(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct mbuf *m; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_LWKT_TOKEN_HELD(&pf_token); for (;;) { - m = ifq_dequeue(&ifp->if_snd, NULL); + m = ifsq_dequeue(ifsq, NULL); if (m == NULL) return; else diff --git a/sys/net/pf/if_pfsync.c b/sys/net/pf/if_pfsync.c index 229d6b2a3e..47c622aa39 100644 --- a/sys/net/pf/if_pfsync.c +++ b/sys/net/pf/if_pfsync.c @@ -95,7 +95,7 @@ int pfsync_alloc_scrub_memory(struct pfsync_state_peer *, int pfsyncoutput(struct ifnet *, struct mbuf *, struct sockaddr *, struct rtentry *); int pfsyncioctl(struct ifnet *, u_long, caddr_t, struct ucred *); -void pfsyncstart(struct ifnet *); +void pfsyncstart(struct ifnet *, struct ifaltq_subque *); struct mbuf *pfsync_get_mbuf(struct pfsync_softc *, u_int8_t, void **); int pfsync_request_update(struct pfsync_state_upd *, struct in_addr *); @@ -210,9 +210,10 @@ pfsync_clone_destroy(struct ifnet *ifp) * Start output on the pfsync interface. */ void -pfsyncstart(struct ifnet *ifp) +pfsyncstart(struct ifnet *ifp, struct ifaltq_subque *ifsq) { - ifq_purge(&ifp->if_snd); + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + ifsq_purge(ifsq); } int diff --git a/sys/net/ppp/if_ppp.c b/sys/net/ppp/if_ppp.c index 674d2a3c34..c9119b2fff 100644 --- a/sys/net/ppp/if_ppp.c +++ b/sys/net/ppp/if_ppp.c @@ -150,7 +150,7 @@ static void ppp_ccp (struct ppp_softc *, struct mbuf *m, int rcvd); static void ppp_ccp_closed (struct ppp_softc *); static void ppp_inproc (struct ppp_softc *, struct mbuf *); static void pppdumpm (struct mbuf *m0); -static void ppp_ifstart(struct ifnet *ifp); +static void ppp_ifstart(struct ifnet *ifp, struct ifaltq_subque *ifsq); /* * Some useful mbuf macros not in mbuf.h. @@ -202,6 +202,7 @@ pppintr(netmsg_t msg) { struct mbuf *m; struct ppp_softc *sc; + struct ifaltq_subque *ifsq; int i; /* @@ -214,10 +215,12 @@ pppintr(netmsg_t msg) get_mplock(); sc = ppp_softc; + ifsq = ifq_get_subq_default(&sc->sc_if.if_snd); + for (i = 0; i < NPPP; ++i, ++sc) { ifnet_serialize_all(&sc->sc_if); if (!(sc->sc_flags & SC_TBUSY) - && (!ifq_is_empty(&sc->sc_if.if_snd) || !IF_QEMPTY(&sc->sc_fastq))) { + && (!ifsq_is_empty(ifsq) || !IF_QEMPTY(&sc->sc_fastq))) { sc->sc_flags |= SC_TBUSY; (*sc->sc_start)(sc); } @@ -875,7 +878,8 @@ pppoutput_serialized(struct ifnet *ifp, struct mbuf *m0, struct sockaddr *dst, } } else { ASSERT_IFNET_SERIALIZED_TX(&sc->sc_if); - error = ifq_enqueue(&sc->sc_if.if_snd, m0, &pktattr); + error = ifsq_enqueue(ifq_get_subq_default(&sc->sc_if.if_snd), m0, + &pktattr); } if (error) { crit_exit(); @@ -920,9 +924,11 @@ ppp_requeue(struct ppp_softc *sc) { struct mbuf *m, **mpp; struct ifqueue *ifq; + struct ifaltq_subque *ifsq; enum NPmode mode; int error; + ifsq = ifq_get_subq_default(&sc->sc_if.if_snd); for (mpp = &sc->sc_npqueue; (m = *mpp) != NULL; ) { switch (PPP_PROTOCOL(mtod(m, u_char *))) { case PPP_IP: @@ -949,7 +955,7 @@ ppp_requeue(struct ppp_softc *sc) error = 0; } } else { - error = ifq_enqueue(&sc->sc_if.if_snd, m, NULL); + error = ifsq_enqueue(ifsq, m, NULL); } if (error) { sc->sc_if.if_oerrors++; @@ -1004,7 +1010,7 @@ ppp_dequeue(struct ppp_softc *sc) */ IF_DEQUEUE(&sc->sc_fastq, m); if (m == NULL) - m = ifq_dequeue(&sc->sc_if.if_snd, NULL); + m = ifsq_dequeue(ifq_get_subq_default(&sc->sc_if.if_snd), NULL); if (m == NULL) return NULL; @@ -1605,7 +1611,7 @@ done: * if_start to send a packet. */ static void -ppp_ifstart(struct ifnet *ifp) +ppp_ifstart(struct ifnet *ifp, struct ifaltq_subque *ifsq __unused) { struct ppp_softc *sc; diff --git a/sys/net/sl/if_sl.c b/sys/net/sl/if_sl.c index d78e8d1e26..1e97b95a83 100644 --- a/sys/net/sl/if_sl.c +++ b/sys/net/sl/if_sl.c @@ -507,7 +507,8 @@ sloutput_serialized(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, error = 0; } } else { - error = ifq_enqueue(&sc->sc_if.if_snd, m, &pktattr); + error = ifsq_enqueue(ifq_get_subq_default(&sc->sc_if.if_snd), + m, &pktattr); } if (error) { sc->sc_if.if_oerrors++; @@ -542,6 +543,7 @@ static int slstart(struct tty *tp) { struct sl_softc *sc = (struct sl_softc *)tp->t_sc; + struct ifaltq_subque *ifsq = ifq_get_subq_default(&sc->sc_if.if_snd); struct mbuf *m; u_char *cp; struct ip *ip; @@ -582,7 +584,7 @@ slstart(struct tty *tp) if (m) sc->sc_if.if_omcasts++; /* XXX */ else - m = ifq_dequeue(&sc->sc_if.if_snd, NULL); + m = ifsq_dequeue(ifsq, NULL); crit_exit(); if (m == NULL) { lwkt_reltoken(&tty_token); diff --git a/sys/net/sppp/if_spppsubr.c b/sys/net/sppp/if_spppsubr.c index 15c1be5fef..aa0ee3f45a 100644 --- a/sys/net/sppp/if_spppsubr.c +++ b/sys/net/sppp/if_spppsubr.c @@ -726,6 +726,7 @@ sppp_output_serialized(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, struct rtentry *rt) { struct sppp *sp = (struct sppp*) ifp; + struct ifaltq_subque *ifsq = ifq_get_subq_default(&ifp->if_snd); struct ppp_header *h; struct ifqueue *ifq = NULL; int rv = 0; @@ -937,21 +938,20 @@ sppp_output_serialized(struct ifnet *ifp, struct mbuf *m, IF_DROP(ifq); m_freem(m); rv = ENOBUFS; - ifq->ifq_drops++; } else { IF_ENQUEUE(ifq, m); rv = 0; } } else { - rv = ifq_enqueue(&ifp->if_snd, m, &pktattr); + rv = ifsq_enqueue(ifsq, m, &pktattr); } if (rv) { ++ifp->if_oerrors; crit_exit(); return(rv); } - if (!ifq_is_oactive(&ifp->if_snd)) - (*ifp->if_start) (ifp); + if (!ifsq_is_oactive(ifsq)) + (*ifp->if_start) (ifp, ifsq); /* * Count output packets and bytes. @@ -1079,7 +1079,7 @@ sppp_isempty(struct ifnet *ifp) crit_enter(); empty = IF_QEMPTY(&sp->pp_fastq) && IF_QEMPTY(&sp->pp_cpq) && - ifq_is_empty(&sp->pp_if.if_snd); + ifsq_is_empty(ifq_get_subq_default(&sp->pp_if.if_snd)); crit_exit(); return (empty); } @@ -1105,8 +1105,10 @@ sppp_dequeue(struct ifnet *ifp) if (m == NULL && (sppp_ncp_check(sp) || sp->pp_mode == IFF_CISCO)) { IF_DEQUEUE(&sp->pp_fastq, m); - if (m == NULL) - m = ifq_dequeue(&sp->pp_if.if_snd, NULL); + if (m == NULL) { + m = ifsq_dequeue( + ifq_get_subq_default(&sp->pp_if.if_snd), NULL); + } } crit_exit(); @@ -1128,7 +1130,7 @@ sppp_pick(struct ifnet *ifp) if (m == NULL && (sp->pp_phase == PHASE_NETWORK || sp->pp_mode == IFF_CISCO)) { if ((m = sp->pp_fastq.ifq_head) == NULL) - m = ifq_poll(&sp->pp_if.if_snd); + m = ifsq_poll(ifq_get_subq_default(&sp->pp_if.if_snd)); } crit_exit(); @@ -1342,6 +1344,7 @@ sppp_cisco_send(struct sppp *sp, int type, long par1, long par2) #else u_long t = (time.tv_sec - boottime.tv_sec) * 1000; #endif + struct ifaltq_subque *ifsq; #if defined(__DragonFly__) getmicrouptime(&tv); @@ -1383,8 +1386,9 @@ sppp_cisco_send(struct sppp *sp, int type, long par1, long par2) m_freem (m); } else IF_ENQUEUE (&sp->pp_cpq, m); - if (!ifq_is_oactive(&ifp->if_snd)) - (*ifp->if_start) (ifp); + ifsq = ifq_get_subq_default(&ifp->if_snd); + if (!ifsq_is_oactive(ifsq)) + (*ifp->if_start) (ifp, ifsq); ifp->if_obytes += m->m_pkthdr.len + 3; } @@ -1403,6 +1407,7 @@ sppp_cp_send(struct sppp *sp, u_short proto, u_char type, struct ppp_header *h; struct lcp_header *lh; struct mbuf *m; + struct ifaltq_subque *ifsq; if (len > MHLEN - PPP_HEADER_LEN - LCP_HEADER_LEN) len = MHLEN - PPP_HEADER_LEN - LCP_HEADER_LEN; @@ -1439,8 +1444,9 @@ sppp_cp_send(struct sppp *sp, u_short proto, u_char type, ++ifp->if_oerrors; } else IF_ENQUEUE (&sp->pp_cpq, m); - if (!ifq_is_oactive(&ifp->if_snd)) - (*ifp->if_start) (ifp); + ifsq = ifq_get_subq_default(&ifp->if_snd); + if (!ifsq_is_oactive(ifsq)) + (*ifp->if_start) (ifp, ifsq); ifp->if_obytes += m->m_pkthdr.len + 3; } @@ -4726,6 +4732,7 @@ sppp_auth_send(const struct cp *cp, struct sppp *sp, int len; unsigned int mlen; const char *msg; + struct ifaltq_subque *ifsq; __va_list ap; MGETHDR (m, MB_DONTWAIT, MT_DATA); @@ -4777,8 +4784,9 @@ sppp_auth_send(const struct cp *cp, struct sppp *sp, ++ifp->if_oerrors; } else IF_ENQUEUE (&sp->pp_cpq, m); - if (!ifq_is_oactive(&ifp->if_snd)) - (*ifp->if_start) (ifp); + ifsq = ifq_get_subq_default(&ifp->if_snd); + if (!ifsq_is_oactive(ifsq)) + (*ifp->if_start) (ifp, ifsq); ifp->if_obytes += m->m_pkthdr.len + 3; } diff --git a/sys/net/tap/if_tap.c b/sys/net/tap/if_tap.c index afb8a2c8f2..99d84ff151 100644 --- a/sys/net/tap/if_tap.c +++ b/sys/net/tap/if_tap.c @@ -106,7 +106,8 @@ static int tap_clone_destroy(struct ifnet *); /* network interface */ -static void tapifstart (struct ifnet *); +static void tapifstart (struct ifnet *, + struct ifaltq_subque *); static int tapifioctl (struct ifnet *, u_long, caddr_t, struct ucred *); static void tapifinit (void *); @@ -528,6 +529,7 @@ tapifinit(void *xtp) { struct tap_softc *tp = xtp; struct ifnet *ifp = &tp->tap_if; + struct ifaltq_subque *ifsq = ifq_get_subq_default(&ifp->if_snd); TAPDEBUG(ifp, "initializing, minor = %#x tap_flags = 0x%x\n", minor(tp->tap_dev), tp->tap_flags); @@ -537,10 +539,10 @@ tapifinit(void *xtp) tapifstop(tp, 1); ifp->if_flags |= IFF_RUNNING; - ifq_clr_oactive(&ifp->if_snd); + ifsq_clr_oactive(ifsq); /* attempt to start output */ - tapifstart(ifp); + tapifstart(ifp, ifsq); } @@ -633,13 +635,14 @@ tapifioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cr) * MPSAFE */ static void -tapifstart(struct ifnet *ifp) +tapifstart(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct tap_softc *tp = ifp->if_softc; struct ifqueue *ifq; struct mbuf *m; int has_data = 0; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); TAPDEBUG(ifp, "starting, minor = %#x\n", minor(tp->tap_dev)); /* @@ -651,14 +654,14 @@ tapifstart(struct ifnet *ifp) ((tp->tap_flags & TAP_READY) != TAP_READY)) { TAPDEBUG(ifp, "not ready. minor = %#x, tap_flags = 0x%x\n", minor(tp->tap_dev), tp->tap_flags); - ifq_purge(&ifp->if_snd); + ifsq_purge(ifsq); return; } - ifq_set_oactive(&ifp->if_snd); + ifsq_set_oactive(ifsq); ifq = &tp->tap_devq; - while ((m = ifq_dequeue(&ifp->if_snd, NULL)) != NULL) { + while ((m = ifsq_dequeue(ifsq, NULL)) != NULL) { if (IF_QFULL(ifq)) { IF_DROP(ifq); ifp->if_oerrors++; @@ -685,7 +688,7 @@ tapifstart(struct ifnet *ifp) } } - ifq_clr_oactive(&ifp->if_snd); + ifsq_clr_oactive(ifsq); } @@ -748,7 +751,7 @@ tapioctl(struct dev_ioctl_args *ap) /* Take a look at devq first */ IF_POLL(&tp->tap_devq, mb); if (mb == NULL) - mb = ifq_poll(&ifp->if_snd); + mb = ifsq_poll(ifq_get_subq_default(&ifp->if_snd)); if (mb != NULL) { for(; mb != NULL; mb = mb->m_next) @@ -1043,7 +1046,7 @@ tapifstop(struct tap_softc *tp, int clear_flags) tp->tap_flags &= ~TAP_CLOSEDOWN; if (clear_flags) { ifp->if_flags &= ~IFF_RUNNING; - ifq_clr_oactive(&ifp->if_snd); + ifsq_clr_oactive(ifq_get_subq_default(&ifp->if_snd)); } } diff --git a/sys/net/tun/if_tun.c b/sys/net/tun/if_tun.c index 73d6e2ab49..94f686145a 100644 --- a/sys/net/tun/if_tun.c +++ b/sys/net/tun/if_tun.c @@ -75,7 +75,7 @@ static int tunoutput (struct ifnet *, struct mbuf *, struct sockaddr *, struct rtentry *rt); static int tunifioctl (struct ifnet *, u_long, caddr_t, struct ucred *); static int tuninit (struct ifnet *); -static void tunstart(struct ifnet *); +static void tunstart(struct ifnet *, struct ifaltq_subque *); static void tun_filter_detach(struct knote *); static int tun_filter_read(struct knote *, long); static int tun_filter_write(struct knote *, long); @@ -493,10 +493,11 @@ tunioctl(struct dev_ioctl_args *ap) tp->tun_flags &= ~TUN_ASYNC; break; case FIONREAD: - if (!ifq_is_empty(&tp->tun_if.if_snd)) { + if (!ifsq_is_empty(ifq_get_subq_default(&tp->tun_if.if_snd))) { struct mbuf *mb; - mb = ifq_poll(&tp->tun_if.if_snd); + mb = ifsq_poll( + ifq_get_subq_default(&tp->tun_if.if_snd)); for( *(int *)ap->a_data = 0; mb != NULL; mb = mb->m_next) *(int *)ap->a_data += mb->m_len; } else { @@ -536,6 +537,7 @@ tunread(struct dev_read_args *ap) struct uio *uio = ap->a_uio; struct tun_softc *tp = dev->si_drv1; struct ifnet *ifp = &tp->tun_if; + struct ifaltq_subque *ifsq = ifq_get_subq_default(&ifp->if_snd); struct mbuf *m0; int error=0, len; @@ -549,7 +551,7 @@ tunread(struct dev_read_args *ap) ifnet_serialize_all(ifp); - while ((m0 = ifq_dequeue(&ifp->if_snd, NULL)) == NULL) { + while ((m0 = ifsq_dequeue(ifsq, NULL)) == NULL) { if (ap->a_ioflag & IO_NDELAY) { ifnet_deserialize_all(ifp); return EWOULDBLOCK; @@ -769,7 +771,7 @@ tun_filter_read(struct knote *kn, long hint) int ready = 0; ifnet_serialize_all(&tp->tun_if); - if (!ifq_is_empty(&tp->tun_if.if_snd)) + if (!ifsq_is_empty(ifq_get_subq_default(&tp->tun_if.if_snd))) ready = 1; ifnet_deserialize_all(&tp->tun_if); @@ -783,15 +785,17 @@ tun_filter_read(struct knote *kn, long hint) * to notify readers when outgoing packets become ready. */ static void -tunstart(struct ifnet *ifp) +tunstart(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct tun_softc *tp = ifp->if_softc; struct mbuf *m; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + if (!ifq_is_enabled(&ifp->if_snd)) return; - m = ifq_poll(&ifp->if_snd); + m = ifsq_poll(ifsq); if (m != NULL) { if (tp->tun_flags & TUN_RWAIT) { tp->tun_flags &= ~TUN_RWAIT; diff --git a/sys/net/vlan/if_vlan.c b/sys/net/vlan/if_vlan.c index 32b6fc8700..ba2e2ec710 100644 --- a/sys/net/vlan/if_vlan.c +++ b/sys/net/vlan/if_vlan.c @@ -182,7 +182,7 @@ static int vlan_clone_destroy(struct ifnet *); static void vlan_ifdetach(void *, struct ifnet *); static void vlan_init(void *); -static void vlan_start(struct ifnet *); +static void vlan_start(struct ifnet *, struct ifaltq_subque *); static int vlan_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static void vlan_input(struct mbuf *); @@ -497,26 +497,30 @@ vlan_init(void *xsc) } static void -vlan_start(struct ifnet *ifp) +vlan_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct ifvlan *ifv = ifp->if_softc; struct ifnet *ifp_p = ifv->ifv_p; struct mbuf *m; + lwkt_port_t p_port; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); ASSERT_IFNET_SERIALIZED_TX(ifp); if (ifp_p == NULL) { - ifq_purge(&ifp->if_snd); + ifsq_purge(ifsq); return; } if ((ifp->if_flags & IFF_RUNNING) == 0) return; + p_port = netisr_portfn( + ifsq_get_cpuid(ifq_get_subq_default(&ifp_p->if_snd))); for (;;) { struct netmsg_packet *nmp; - m = ifq_dequeue(&ifp->if_snd, NULL); + m = ifsq_dequeue(ifsq, NULL); if (m == NULL) break; BPF_MTAP(ifp, m); @@ -550,8 +554,7 @@ vlan_start(struct ifnet *ifp) nmp->nm_packet = m; nmp->base.lmsg.u.ms_resultp = ifp_p; - lwkt_sendmsg(netisr_portfn(ifq_get_cpuid(&ifp_p->if_snd)), - &nmp->base.lmsg); + lwkt_sendmsg(p_port, &nmp->base.lmsg); ifp->if_opackets++; } } diff --git a/sys/netgraph/eiface/ng_eiface.c b/sys/netgraph/eiface/ng_eiface.c index c944cb1c33..f5f9257da5 100644 --- a/sys/netgraph/eiface/ng_eiface.c +++ b/sys/netgraph/eiface/ng_eiface.c @@ -89,7 +89,7 @@ typedef struct ng_eiface_private *priv_p; /* Interface methods */ static void ng_eiface_init(void *xsc); -static void ng_eiface_start(struct ifnet *ifp); +static void ng_eiface_start(struct ifnet *ifp, struct ifaltq_subque *); static int ng_eiface_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cr); #ifdef DEBUG @@ -219,7 +219,7 @@ ng_eiface_init(void *xsc) */ static void -ng_eiface_start(struct ifnet *ifp) +ng_eiface_start(struct ifnet *ifp, struct ifaltq_subque *ifsq __unused) { const priv_p priv = (priv_p) ifp->if_softc; meta_p meta = NULL; diff --git a/sys/netgraph/fec/ng_fec.c b/sys/netgraph/fec/ng_fec.c index 813f2c8692..2c6d1c1132 100644 --- a/sys/netgraph/fec/ng_fec.c +++ b/sys/netgraph/fec/ng_fec.c @@ -171,7 +171,7 @@ typedef struct ng_fec_private *priv_p; /* Interface methods */ static void ng_fec_input(struct ifnet *, struct mbuf **); -static void ng_fec_start(struct ifnet *ifp); +static void ng_fec_start(struct ifnet *ifp, struct ifaltq_subque *); static int ng_fec_choose_port(struct ng_fec_bundle *b, struct mbuf *m, struct ifnet **ifp); static int ng_fec_setport(struct ifnet *ifp, u_long cmd, caddr_t data); @@ -977,7 +977,7 @@ ng_fec_choose_port(struct ng_fec_bundle *b, * transmission. */ static void -ng_fec_start(struct ifnet *ifp) +ng_fec_start(struct ifnet *ifp, struct ifaltq_subque *ifsq __unused) { struct ng_fec_private *priv; struct ng_fec_bundle *b; diff --git a/sys/netgraph/iface/ng_iface.c b/sys/netgraph/iface/ng_iface.c index a87a91014d..0040cfa0a7 100644 --- a/sys/netgraph/iface/ng_iface.c +++ b/sys/netgraph/iface/ng_iface.c @@ -110,7 +110,7 @@ struct ng_iface_private { typedef struct ng_iface_private *priv_p; /* Interface methods */ -static void ng_iface_start(struct ifnet *ifp); +static void ng_iface_start(struct ifnet *ifp, struct ifaltq_subque *); static int ng_iface_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cr); static int ng_iface_output(struct ifnet *ifp, struct mbuf *m0, @@ -475,7 +475,7 @@ ng_iface_output(struct ifnet *ifp, struct mbuf *m, */ static void -ng_iface_start(struct ifnet *ifp) +ng_iface_start(struct ifnet *ifp, struct ifaltq_subque *ifsq __unused) { kprintf("%s: %s called?", ifp->if_xname, __func__); } diff --git a/sys/netgraph7/dragonfly.h b/sys/netgraph7/dragonfly.h index 50574e7c3e..ecf872727d 100644 --- a/sys/netgraph7/dragonfly.h +++ b/sys/netgraph7/dragonfly.h @@ -56,8 +56,8 @@ typedef __va_list va_list; #define IFNET_RLOCK() crit_enter() #define IFNET_RUNLOCK() crit_exit() -#define IFQ_LOCK(ifp) lwkt_serialize_enter(&(ifp)->altq_lock) -#define IFQ_UNLOCK(ifp) lwkt_serialize_exit(&(ifp)->altq_lock) +#define IFQ_LOCK(ifq) ALTQ_LOCK((ifq)) +#define IFQ_UNLOCK(ifq) ALTQ_unLOCK((ifq)) #define printf kprintf #define sprintf ksprintf diff --git a/sys/netgraph7/iface/ng_iface.c b/sys/netgraph7/iface/ng_iface.c index c32fa979ea..cfb05ee496 100644 --- a/sys/netgraph7/iface/ng_iface.c +++ b/sys/netgraph7/iface/ng_iface.c @@ -116,7 +116,7 @@ struct ng_iface_private { typedef struct ng_iface_private *priv_p; /* Interface methods */ -static void ng_iface_start(struct ifnet *ifp); +static void ng_iface_start(struct ifnet *ifp, struct ifaltq_subque *); static int ng_iface_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cr); static int ng_iface_output(struct ifnet *ifp, struct mbuf *m0, @@ -454,7 +454,7 @@ ng_iface_output(struct ifnet *ifp, struct mbuf *m, * Start method is used only when ALTQ is enabled. */ static void -ng_iface_start(struct ifnet *ifp) +ng_iface_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { struct mbuf *m = NULL; sa_family_t sa; diff --git a/sys/netgraph7/ng_eiface.c b/sys/netgraph7/ng_eiface.c index 3f3a46b6c8..dbc2a4ab73 100644 --- a/sys/netgraph7/ng_eiface.c +++ b/sys/netgraph7/ng_eiface.c @@ -82,7 +82,7 @@ typedef struct ng_eiface_private *priv_p; /* Interface methods */ static void ng_eiface_init(void *xsc); -static void ng_eiface_start(struct ifnet *ifp); +static void ng_eiface_start(struct ifnet *ifp, struct ifaltq_subque *); static int ng_eiface_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data); #ifdef DEBUG static void ng_eiface_print_ioctl(struct ifnet *ifp, int cmd, caddr_t data); @@ -273,7 +273,7 @@ ng_eiface_start2(node_p node, hook_p hook, void *arg1, int arg2) * somehow, but we can't and if we did would we solve anything? */ static void -ng_eiface_start(struct ifnet *ifp) +ng_eiface_start(struct ifnet *ifp, struct ifaltq_subque *ifsq __unused) { const priv_p priv = (priv_p)ifp->if_softc; diff --git a/sys/netgraph7/ng_fec.c b/sys/netgraph7/ng_fec.c index d54a33175f..90cee84c23 100644 --- a/sys/netgraph7/ng_fec.c +++ b/sys/netgraph7/ng_fec.c @@ -188,7 +188,7 @@ typedef struct ng_fec_private *priv_p; /* Interface methods */ static void ng_fec_input(struct ifnet *, struct mbuf *); -static void ng_fec_start(struct ifnet *ifp); +static void ng_fec_start(struct ifnet *ifp, struct ifaltq_subque *); static int ng_fec_choose_port(struct ng_fec_bundle *b, struct mbuf *m, struct ifnet **ifp); static int ng_fec_setport(struct ifnet *ifp, u_long cmd, caddr_t data); @@ -1112,7 +1112,7 @@ ng_fec_choose_port(struct ng_fec_bundle *b, * transmission. */ static void -ng_fec_start(struct ifnet *ifp) +ng_fec_start(struct ifnet *ifp, struct ifaltq_subque *ifsq __unused) { struct ng_fec_private *priv; struct ng_fec_bundle *b; diff --git a/sys/netgraph7/ng_sppp.c b/sys/netgraph7/ng_sppp.c index 9dac9dadda..c9bf9b9843 100644 --- a/sys/netgraph7/ng_sppp.c +++ b/sys/netgraph7/ng_sppp.c @@ -59,7 +59,7 @@ struct ng_sppp_private { typedef struct ng_sppp_private *priv_p; /* Interface methods */ -static void ng_sppp_start (struct ifnet *ifp); +static void ng_sppp_start (struct ifnet *ifp, struct ifaltq_subque *); static int ng_sppp_ioctl (struct ifnet *ifp, u_long cmd, caddr_t data); /* Netgraph methods */ @@ -187,7 +187,7 @@ ng_sppp_ioctl (struct ifnet *ifp, u_long command, caddr_t data) */ static void -ng_sppp_start (struct ifnet *ifp) +ng_sppp_start (struct ifnet *ifp, struct ifaltq_subque *ifsq __unused) { struct mbuf *m; int len, error = 0; diff --git a/sys/netinet/ip_carp.c b/sys/netinet/ip_carp.c index 71c549bd3c..2a5e10da1f 100644 --- a/sys/netinet/ip_carp.c +++ b/sys/netinet/ip_carp.c @@ -383,7 +383,7 @@ static void carp_init(void *); static int carp_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); static int carp_output(struct ifnet *, struct mbuf *, struct sockaddr *, struct rtentry *); -static void carp_start(struct ifnet *); +static void carp_start(struct ifnet *, struct ifaltq_subque *); static void carp_multicast_cleanup(struct carp_softc *); static void carp_add_addr(struct carp_softc *, struct ifaddr *); @@ -2623,7 +2623,7 @@ carp_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, * Start output on carp interface. This function should never be called. */ static void -carp_start(struct ifnet *ifp) +carp_start(struct ifnet *ifp, struct ifaltq_subque *ifsq __unused) { panic("%s: start called", ifp->if_xname); } diff --git a/sys/netproto/802_11/ieee80211_proto.h b/sys/netproto/802_11/ieee80211_proto.h index 58950798e8..59aa700be4 100644 --- a/sys/netproto/802_11/ieee80211_proto.h +++ b/sys/netproto/802_11/ieee80211_proto.h @@ -74,7 +74,7 @@ int ieee80211_output(struct ifnet *, struct mbuf *, void ieee80211_send_setup(struct ieee80211_node *, struct mbuf *, int, int, const uint8_t [IEEE80211_ADDR_LEN], const uint8_t [IEEE80211_ADDR_LEN], const uint8_t [IEEE80211_ADDR_LEN]); -void ieee80211_start(struct ifnet *); +void ieee80211_start(struct ifnet *, struct ifaltq_subque *); int ieee80211_send_nulldata(struct ieee80211_node *); int ieee80211_classify(struct ieee80211_node *, struct mbuf *m); struct mbuf *ieee80211_mbuf_adjust(struct ieee80211vap *, int, diff --git a/sys/netproto/802_11/wlan/ieee80211_hostap.c b/sys/netproto/802_11/wlan/ieee80211_hostap.c index 24751633ad..bc629d3fc7 100644 --- a/sys/netproto/802_11/wlan/ieee80211_hostap.c +++ b/sys/netproto/802_11/wlan/ieee80211_hostap.c @@ -2221,6 +2221,7 @@ hostap_recv_pspoll(struct ieee80211_node *ni, struct mbuf *m0) struct ieee80211vap *vap = ni->ni_vap; struct ieee80211_frame_min *wh; struct ifnet *ifp; + struct ifaltq_subque *ifsq; struct mbuf *m; uint16_t aid; int qlen; @@ -2288,6 +2289,8 @@ hostap_recv_pspoll(struct ieee80211_node *ni, struct mbuf *m0) ifp = vap->iv_ic->ic_ifp; else ifp = vap->iv_ifp; - ifq_enqueue(&ifp->if_snd, m, NULL); - ifp->if_start(ifp); + + ifsq = ifq_get_subq_default(&ifp->if_snd); + ifsq_enqueue(ifsq, m, NULL); + ifp->if_start(ifp, ifsq); } diff --git a/sys/netproto/802_11/wlan/ieee80211_output.c b/sys/netproto/802_11/wlan/ieee80211_output.c index d519cc3ea8..2cba9f8444 100644 --- a/sys/netproto/802_11/wlan/ieee80211_output.c +++ b/sys/netproto/802_11/wlan/ieee80211_output.c @@ -110,7 +110,7 @@ doprint(struct ieee80211vap *vap, int subtype) * before dispatching them to the underlying device. */ void -ieee80211_start(struct ifnet *ifp) +ieee80211_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) { #define IS_DWDS(vap) \ (vap->iv_opmode == IEEE80211_M_WDS && \ @@ -123,13 +123,15 @@ ieee80211_start(struct ifnet *ifp) struct ether_header *eh; int error; + ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); + /* NB: parent must be up and running */ if (!IFNET_IS_UP_RUNNING(parent)) { IEEE80211_DPRINTF(vap, IEEE80211_MSG_OUTPUT, "%s: ignore queue, parent %s not up+running\n", __func__, parent->if_xname); /* XXX stat */ - ifq_purge(&ifp->if_snd); + ifsq_purge(ifsq); return; } if (vap->iv_state == IEEE80211_S_SLEEP) { @@ -137,7 +139,7 @@ ieee80211_start(struct ifnet *ifp) * In power save, wakeup device for transmit. */ ieee80211_new_state(vap, IEEE80211_S_RUN, 0); - ifq_purge(&ifp->if_snd); + ifsq_purge(ifsq); return; } /* @@ -153,12 +155,12 @@ ieee80211_start(struct ifnet *ifp) "%s: ignore queue, in %s state\n", __func__, ieee80211_state_name[vap->iv_state]); vap->iv_stats.is_tx_badstate++; - ifq_set_oactive(&ifp->if_snd); + ifsq_set_oactive(ifsq); return; } } for (;;) { - m = ifq_dequeue(&ifp->if_snd, NULL); + m = ifsq_dequeue(ifsq, NULL); if (m == NULL) break; /* @@ -382,9 +384,11 @@ ieee80211_output(struct ifnet *ifp, struct mbuf *m, struct ieee80211_node *ni = NULL; struct ieee80211vap *vap; struct ieee80211_frame *wh; + struct ifaltq_subque *ifsq; int error; - if (ifq_is_oactive(&ifp->if_snd)) { + ifsq = ifq_get_subq_default(&ifp->if_snd); + if (ifsq_is_oactive(ifsq)) { /* * Short-circuit requests if the vap is marked OACTIVE * as this can happen because a packet came down through diff --git a/sys/netproto/802_11/wlan/ieee80211_power.c b/sys/netproto/802_11/wlan/ieee80211_power.c index 1137bc2450..1cdcbcde0f 100644 --- a/sys/netproto/802_11/wlan/ieee80211_power.c +++ b/sys/netproto/802_11/wlan/ieee80211_power.c @@ -38,6 +38,7 @@ #include #include +#include #include #include @@ -400,6 +401,7 @@ pwrsave_flushq(struct ieee80211_node *ni) struct ieee80211vap *vap = ni->ni_vap; struct ieee80211_psq_head *qhead; struct ifnet *parent, *ifp; + struct ifaltq_subque *ifp_ifsq, *parent_ifsq; IEEE80211_NOTE(vap, IEEE80211_MSG_POWER, ni, "flush ps queue, %u packets queued", psq->psq_len); @@ -408,34 +410,49 @@ pwrsave_flushq(struct ieee80211_node *ni) if (qhead->head != NULL) { /* XXX could dispatch through vap and check M_ENCAP */ parent = vap->iv_ic->ic_ifp; + parent_ifsq = ifq_get_subq_default(&parent->if_snd); + + /* XXX this breaks ALTQ's packet scheduler */ + ALTQ_SQ_LOCK(parent_ifsq); /* XXX need different driver interface */ /* XXX bypasses q max and OACTIVE */ - IF_PREPEND_LIST(&parent->if_snd, qhead->head, qhead->tail, + IF_PREPEND_LIST(parent_ifsq, qhead->head, qhead->tail, qhead->len); + ALTQ_SQ_UNLOCK(parent_ifsq); + qhead->head = qhead->tail = NULL; qhead->len = 0; - } else + } else { parent = NULL; + parent_ifsq = NULL; + } qhead = &psq->psq_head[1]; /* 802.3 frames */ if (qhead->head != NULL) { ifp = vap->iv_ifp; + ifp_ifsq = ifq_get_subq_default(&ifp->if_snd); + + /* XXX this breaks ALTQ's packet scheduler */ + ALTQ_SQ_LOCK(ifp_ifsq); /* XXX need different driver interface */ /* XXX bypasses q max and OACTIVE */ - IF_PREPEND_LIST(&ifp->if_snd, qhead->head, qhead->tail, - qhead->len); + IF_PREPEND_LIST(ifp_ifsq, qhead->head, qhead->tail, qhead->len); + ALTQ_SQ_UNLOCK(ifp_ifsq); + qhead->head = qhead->tail = NULL; qhead->len = 0; - } else + } else { ifp = NULL; + ifp_ifsq = NULL; + } psq->psq_len = 0; /* NB: do this outside the psq lock */ /* XXX packets might get reordered if parent is OACTIVE */ - if (parent != NULL) - parent->if_start(parent); - if (ifp != NULL) - ifp->if_start(ifp); + if (parent != NULL && parent_ifsq != NULL) + parent->if_start(parent, parent_ifsq); + if (ifp != NULL && ifp_ifsq != NULL) + ifp->if_start(ifp, ifp_ifsq); } /* diff --git a/sys/netproto/802_11/wlan/ieee80211_proto.c b/sys/netproto/802_11/wlan/ieee80211_proto.c index 4ba3c4d203..3d7163b3cc 100644 --- a/sys/netproto/802_11/wlan/ieee80211_proto.c +++ b/sys/netproto/802_11/wlan/ieee80211_proto.c @@ -1646,6 +1646,9 @@ ieee80211_newstate_task(void *xvap, int npending) goto done; if (nstate == IEEE80211_S_RUN) { + struct ifaltq_subque *ifsq = + ifq_get_subq_default(&vap->iv_ifp->if_snd); + /* * OACTIVE may be set on the vap if the upper layer * tried to transmit (e.g. IPv6 NDP) before we reach @@ -1654,8 +1657,8 @@ ieee80211_newstate_task(void *xvap, int npending) * Note this can also happen as a result of SLEEP->RUN * (i.e. coming out of power save mode). */ - ifq_clr_oactive(&vap->iv_ifp->if_snd); - vap->iv_ifp->if_start(vap->iv_ifp); + ifsq_clr_oactive(ifsq); + vap->iv_ifp->if_start(vap->iv_ifp, ifsq); /* bring up any vaps waiting on us */ wakeupwaiting(vap); diff --git a/sys/netproto/ipx/ipx_ip.c b/sys/netproto/ipx/ipx_ip.c index 00216ea357..d73686cadb 100644 --- a/sys/netproto/ipx/ipx_ip.c +++ b/sys/netproto/ipx/ipx_ip.c @@ -87,7 +87,7 @@ static int ipxipioctl(struct ifnet *ifp, u_long cmd, caddr_t data, static int ipxipoutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, struct rtentry *rt); static void ipxip_rtchange(struct in_addr *dst); -static void ipxipstart(struct ifnet *ifp); +static void ipxipstart(struct ifnet *ifp, struct ifaltq_subque *); static struct ifnet_en * ipxipattach(void) @@ -302,7 +302,7 @@ ipxipoutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, } static void -ipxipstart(struct ifnet *ifp) +ipxipstart(struct ifnet *ifp, struct ifaltq_subque *ifsq __unused) { panic("ipxip_start called"); } diff --git a/usr.bin/netstat/if.c b/usr.bin/netstat/if.c index 05228796e9..33fa970ab9 100644 --- a/usr.bin/netstat/if.c +++ b/usr.bin/netstat/if.c @@ -222,7 +222,7 @@ intpr(int interval1, u_long ifnetaddr, void (*pfunc)(char *)) ierrors = ifnet.if_ierrors; collisions = ifnet.if_collisions; timer = ifnet.if_timer; - drops = ifnet.if_snd.ifq_drops; + drops = 0; if (ifaddraddr == 0) { printf("%-7.7s %-5lu ", name, ifnet.if_mtu); @@ -557,7 +557,7 @@ loop: ifnet.if_obytes - ip->ift_ob, ifnet.if_collisions - ip->ift_co); if (dflag) - printf(" %5u", ifnet.if_snd.ifq_drops - ip->ift_dr); + printf(" %5u", 0 - ip->ift_dr); } ip->ift_ip = ifnet.if_ipackets; ip->ift_ie = ifnet.if_ierrors; @@ -566,7 +566,7 @@ loop: ip->ift_oe = ifnet.if_oerrors; ip->ift_ob = ifnet.if_obytes; ip->ift_co = ifnet.if_collisions; - ip->ift_dr = ifnet.if_snd.ifq_drops; + ip->ift_dr = 0; } else { sum->ift_ip = 0; sum->ift_ie = 0; @@ -590,7 +590,7 @@ loop: sum->ift_oe += ifnet.if_oerrors; sum->ift_ob += ifnet.if_obytes; sum->ift_co += ifnet.if_collisions; - sum->ift_dr += ifnet.if_snd.ifq_drops; + sum->ift_dr += 0; off = (u_long)TAILQ_NEXT(&ifnet, if_link); } if (!first) { -- 2.41.0