From 9db4b353edaae5cb8914ed55a1403419e6d3f724 Mon Sep 17 00:00:00 2001 From: Sepherosa Ziehau Date: Wed, 14 May 2008 11:59:24 +0000 Subject: [PATCH] Reduce ifnet.if_serializer contention on output path: - Push ifnet.if_serializer holding down into each ifnet.if_output implementation - Add a serializer into ifaltq, which is used to protect send queue instead of its parent's if_serializer. This change has following implication: o On output path, enqueueing packets and calling ifnet.if_start are decoupled o In device drivers, poll->dev_encap_ok->dequeue operation sequence is no longer safe, instead dequeue->dev_encap_fail->prepend should be used This serializer will be held by using lwkt_serialize_adaptive_enter() - Add altq_started field into ifaltq, which is used to interlock the calling of its parent's if_start, to reduce ifnet.if_serializer contention. if_devstart(), a helper function which utilizes ifaltq.altq_started, is added to reduce code duplication in ethernet device drivers. - Add if_cpuid into ifnet. This field indicates on which CPU device driver's interrupt will happen. - Add ifq_dispatch(). This function will try to hold ifnet.if_serializer in order to call ifnet.if_start. If this attempt fails, this function will schedule ifnet.if_start to be called on CPU located by ifnet.if_start_cpuid if_start_nmsg, which is per-CPU netmsg, is added to ifnet to facilitate ifnet.if_start scheduling. ifq_dispatch() is called by ether_output_frame() currently - Use ifq_classic_ functions, if altq is not enabled - Fix various device drivers bugs in their if_start implementation - Add ktr for ifq classic enqueue and dequeue - Add ktr for ifnet.if_start --- sys/conf/options | 4 +- sys/config/LINT | 4 +- sys/contrib/ipfilter/netinet/ip_fil.c | 18 +- sys/dev/netif/acx/if_acx.c | 21 +- sys/dev/netif/an/if_an.c | 23 +- sys/dev/netif/an/if_an_isa.c | 8 +- sys/dev/netif/an/if_an_pccard.c | 9 +- sys/dev/netif/an/if_an_pci.c | 9 +- sys/dev/netif/ath/ath/if_ath.c | 25 +- sys/dev/netif/aue/if_aue.c | 16 +- sys/dev/netif/axe/if_axe.c | 17 +- sys/dev/netif/bce/if_bce.c | 22 +- sys/dev/netif/bfe/if_bfe.c | 14 +- sys/dev/netif/bge/if_bge.c | 29 +- sys/dev/netif/bwi/if_bwi.c | 14 +- sys/dev/netif/cs/if_cs.c | 13 +- sys/dev/netif/cue/if_cue.c | 10 +- sys/dev/netif/dc/if_dc.c | 60 ++--- sys/dev/netif/de/if_de.c | 17 +- sys/dev/netif/ed/if_ed.c | 7 +- sys/dev/netif/ed/if_ed_isa.c | 15 +- sys/dev/netif/ed/if_ed_pccard.c | 9 +- sys/dev/netif/ed/if_ed_pci.c | 16 +- sys/dev/netif/em/if_em.c | 29 +- sys/dev/netif/ep/if_ep.c | 18 +- sys/dev/netif/ep/if_ep_eisa.c | 9 +- sys/dev/netif/ep/if_ep_isa.c | 10 +- sys/dev/netif/ep/if_ep_pccard.c | 7 +- sys/dev/netif/et/if_et.c | 17 +- sys/dev/netif/ex/if_ex.c | 14 +- sys/dev/netif/ex/if_ex_isa.c | 9 +- sys/dev/netif/ex/if_ex_pccard.c | 9 +- sys/dev/netif/fe/if_fe.c | 8 +- sys/dev/netif/fwe/if_fwe.c | 4 +- sys/dev/netif/fxp/if_fxp.c | 42 ++- sys/dev/netif/ie/if_ie.c | 4 +- sys/dev/netif/iwi/if_iwi.c | 19 +- sys/dev/netif/iwl/iwl2100.c | 25 +- sys/dev/netif/kue/if_kue.c | 12 +- sys/dev/netif/le/if_le.c | 16 +- sys/dev/netif/lge/if_lge.c | 66 +++-- sys/dev/netif/lge/if_lgereg.h | 6 +- sys/dev/netif/lnc/am7990.c | 10 +- sys/dev/netif/lnc/am79900.c | 10 +- sys/dev/netif/lnc/if_lnc_isa.c | 6 +- sys/dev/netif/lnc/if_lnc_pci.c | 6 +- sys/dev/netif/lnc/lance.c | 4 +- sys/dev/netif/msk/if_msk.c | 29 +- sys/dev/netif/my/if_my.c | 13 +- sys/dev/netif/ndis/if_ndis.c | 8 +- sys/dev/netif/nfe/if_nfe.c | 15 +- sys/dev/netif/nge/if_nge.c | 72 +++-- sys/dev/netif/pcn/if_pcn.c | 72 +++-- sys/dev/netif/ral/rt2560.c | 23 +- sys/dev/netif/ral/rt2661.c | 12 +- sys/dev/netif/re/if_re.c | 40 +-- sys/dev/netif/rl/if_rl.c | 17 +- sys/dev/netif/rtw/rtw.c | 16 +- sys/dev/netif/rue/if_rue.c | 13 +- sys/dev/netif/rum/if_rum.c | 18 +- sys/dev/netif/sbni/if_sbni.c | 4 +- sys/dev/netif/sbni/if_sbni_pci.c | 14 +- sys/dev/netif/sbsh/if_sbsh.c | 6 +- sys/dev/netif/sf/if_sf.c | 126 ++++----- sys/dev/netif/sf/if_sfreg.h | 4 +- sys/dev/netif/sis/if_sis.c | 84 ++++-- sys/dev/netif/sk/if_sk.c | 32 ++- sys/dev/netif/sln/if_sln.c | 19 +- sys/dev/netif/sn/if_sn.c | 58 ++-- sys/dev/netif/ste/if_ste.c | 24 +- sys/dev/netif/stge/if_stge.c | 17 +- sys/dev/netif/ti/if_ti.c | 17 +- sys/dev/netif/tl/if_tl.c | 10 +- sys/dev/netif/tx/if_tx.c | 10 +- sys/dev/netif/txp/if_txp.c | 66 +++-- sys/dev/netif/ural/if_ural.c | 18 +- sys/dev/netif/vge/if_vge.c | 26 +- sys/dev/netif/vr/if_vr.c | 20 +- sys/dev/netif/vx/if_vx.c | 22 +- sys/dev/netif/vx/if_vx_eisa.c | 14 +- sys/dev/netif/vx/if_vx_pci.c | 13 +- sys/dev/netif/wb/if_wb.c | 10 +- sys/dev/netif/wi/if_wi.c | 36 +-- sys/dev/netif/wl/if_wl.c | 10 +- sys/dev/netif/xe/if_xe.c | 30 ++- sys/dev/netif/xl/if_xl.c | 33 ++- sys/kern/kern_intr.c | 15 +- sys/kern/lwkt_serialize.c | 58 ++-- sys/net/altq/altq_cbq.c | 84 ++++-- sys/net/altq/altq_fairq.c | 87 ++++-- sys/net/altq/altq_hfsc.c | 81 ++++-- sys/net/altq/altq_priq.c | 87 ++++-- sys/net/altq/altq_subr.c | 145 +++++++--- sys/net/altq/altq_var.h | 10 +- sys/net/altq/if_altq.h | 21 +- sys/net/bpf.c | 6 +- sys/net/gif/if_gif.c | 20 +- sys/net/gre/if_gre.c | 19 +- sys/net/i4b/driver/i4b_ipr.c | 19 +- sys/net/if.c | 369 ++++++++++++++++++++++++-- sys/net/if_atmsubr.c | 7 +- sys/net/if_ethersubr.c | 12 +- sys/net/if_loop.c | 6 +- sys/net/if_var.h | 21 +- sys/net/ifq_var.h | 139 ++++++++-- sys/net/pf/pf.c | 11 +- sys/net/ppp/if_ppp.c | 21 +- sys/net/ppp_layer/ppp_tty.c | 4 +- sys/net/sl/if_sl.c | 19 +- sys/net/sppp/if_spppsubr.c | 19 +- sys/net/stf/if_stf.c | 19 +- sys/net/tun/if_tun.c | 19 +- sys/netgraph/fec/ng_fec.c | 19 +- sys/netgraph/iface/ng_iface.c | 19 +- sys/netinet/if_ether.c | 16 +- sys/netinet/ip_flow.c | 4 +- sys/netinet/ip_output.c | 15 +- sys/netinet6/ip6_mroute.c | 6 +- sys/netinet6/nd6.c | 11 +- sys/netproto/atalk/aarp.c | 16 +- sys/netproto/atalk/ddp_output.c | 6 +- sys/netproto/atm/atm_if.c | 20 +- sys/netproto/ipx/ipx_ip.c | 19 +- sys/netproto/ipx/ipx_outputfl.c | 12 +- sys/netproto/natm/natm.c | 6 +- sys/netproto/ns/ns_ip.c | 21 +- sys/netproto/ns/ns_output.c | 8 +- sys/sys/interrupt.h | 3 +- sys/sys/serialize.h | 13 +- 129 files changed, 2299 insertions(+), 1044 deletions(-) diff --git a/sys/conf/options b/sys/conf/options index bcc6de8aab..3806ed692d 100644 --- a/sys/conf/options +++ b/sys/conf/options @@ -1,5 +1,5 @@ # $FreeBSD: src/sys/conf/options,v 1.191.2.53 2003/06/04 17:56:58 sam Exp $ -# $DragonFly: src/sys/conf/options,v 1.84 2008/05/02 07:40:32 sephe Exp $ +# $DragonFly: src/sys/conf/options,v 1.85 2008/05/14 11:59:18 sephe Exp $ # # On the handling of kernel options # @@ -527,6 +527,8 @@ KTR_IF_EM opt_ktr.h KTR_TESTLOG opt_ktr.h KTR_SERIALIZER opt_ktr.h KTR_POLLING opt_ktr.h +KTR_IFQ opt_ktr.h +KTR_IF_START opt_ktr.h # bce driver BCE_DEBUG opt_bce.h diff --git a/sys/config/LINT b/sys/config/LINT index c373351769..75d3ddb9fc 100644 --- a/sys/config/LINT +++ b/sys/config/LINT @@ -3,7 +3,7 @@ # as much of the source tree as it can. # # $FreeBSD: src/sys/i386/conf/LINT,v 1.749.2.144 2003/06/04 17:56:59 sam Exp $ -# $DragonFly: src/sys/config/LINT,v 1.160 2008/05/02 07:40:32 sephe Exp $ +# $DragonFly: src/sys/config/LINT,v 1.161 2008/05/14 11:59:18 sephe Exp $ # # NB: You probably don't want to try running a kernel built from this # file. Instead, you should start from GENERIC, and add options from @@ -2747,6 +2747,8 @@ options KTR #options KTR_GIANT_CONTENTION #options KTR_IF_BGE #options KTR_IF_EM +#options KTR_IF_START +#options KTR_IFQ #options KTR_IPIQ #options KTR_MEMORY #options KTR_SERIALIZER diff --git a/sys/contrib/ipfilter/netinet/ip_fil.c b/sys/contrib/ipfilter/netinet/ip_fil.c index 88154bcd22..2be449de65 100644 --- a/sys/contrib/ipfilter/netinet/ip_fil.c +++ b/sys/contrib/ipfilter/netinet/ip_fil.c @@ -6,7 +6,7 @@ * @(#)ip_fil.c 2.41 6/5/96 (C) 1993-2000 Darren Reed * @(#)$Id: ip_fil.c,v 2.42.2.60 2002/08/28 12:40:39 darrenr Exp $ * $FreeBSD: src/sys/contrib/ipfilter/netinet/ip_fil.c,v 1.25.2.7 2004/07/04 09:24:38 darrenr Exp $ - * $DragonFly: src/sys/contrib/ipfilter/netinet/ip_fil.c,v 1.26 2006/12/23 00:27:02 swildner Exp $ + * $DragonFly: src/sys/contrib/ipfilter/netinet/ip_fil.c,v 1.27 2008/05/14 11:59:18 sephe Exp $ */ #ifndef SOLARIS #define SOLARIS (defined(sun) && (defined(__svr4__) || defined(__SVR4))) @@ -1874,17 +1874,13 @@ frdest_t *fdp; # ifdef IRIX IFNET_UPPERLOCK(ifp); # endif - lwkt_serialize_enter(ifp->if_serializer); - error = (*ifp->if_output)(ifp, m, (struct sockaddr *)dst, - ro->ro_rt); - lwkt_serialize_exit(ifp->if_serializer); + error = ifp->if_output(ifp, m, (struct sockaddr *)dst, + ro->ro_rt); # ifdef IRIX IFNET_UPPERUNLOCK(ifp); # endif # else - lwkt_serialize_enter(ifp->if_serializer); - error = (*ifp->if_output)(ifp, m, (struct sockaddr *)dst); - lwkt_serialize_exit(ifp->if_serializer); + error = ifp->if_output(ifp, m, (struct sockaddr *)dst); # endif goto done; } @@ -1974,15 +1970,13 @@ sendorfree: m0 = m->m_act; m->m_act = 0; if (error == 0) { - lwkt_serialize_enter(ifp->if_serializer); # if (BSD >= 199306) || (defined(IRIX) && (IRIX >= 605)) - error = (*ifp->if_output)(ifp, m, + error = ifp->if_output(ifp, m, (struct sockaddr *)dst, ro->ro_rt); # else - error = (*ifp->if_output)(ifp, m, + error = ifp->if_output(ifp, m, (struct sockaddr *)dst); # endif - lwkt_serialize_exit(ifp->if_serializer); } else { m_freem(m); } diff --git a/sys/dev/netif/acx/if_acx.c b/sys/dev/netif/acx/if_acx.c index c4c0a6f3c0..34c839b3e4 100644 --- a/sys/dev/netif/acx/if_acx.c +++ b/sys/dev/netif/acx/if_acx.c @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sys/dev/netif/acx/if_acx.c,v 1.26 2008/02/14 12:53:52 sephe Exp $ + * $DragonFly: src/sys/dev/netif/acx/if_acx.c,v 1.27 2008/05/14 11:59:18 sephe Exp $ */ /* @@ -77,6 +77,7 @@ #include #include #include +#include #include #include #include @@ -532,6 +533,9 @@ acx_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->sc_irq_res)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + if (bootverbose) ieee80211_announce(ic); @@ -1085,8 +1089,13 @@ acx_start(struct ifnet *ifp) ASSERT_SERIALIZED(ifp->if_serializer); - if ((sc->sc_flags & ACX_FLAG_FW_LOADED) == 0 || - (ifp->if_flags & IFF_RUNNING) == 0 || + if ((sc->sc_flags & ACX_FLAG_FW_LOADED) == 0) { + ifq_purge(&ifp->if_snd); + ieee80211_drain_mgtq(&ic->ic_mgtq); + return; + } + + if ((ifp->if_flags & IFF_RUNNING) == 0 || (ifp->if_flags & IFF_OACTIVE)) return; @@ -1130,8 +1139,10 @@ acx_start(struct ifnet *ifp) } else if (!ifq_is_empty(&ifp->if_snd)) { struct ether_header *eh; - if (ic->ic_state != IEEE80211_S_RUN) + if (ic->ic_state != IEEE80211_S_RUN) { + ifq_purge(&ifp->if_snd); break; + } m = ifq_dequeue(&ifp->if_snd, NULL); if (m == NULL) @@ -1345,7 +1356,7 @@ acx_txeof(struct acx_softc *sc) if (bd->tx_used_count != ACX_TX_DESC_CNT) { ifp->if_flags &= ~IFF_OACTIVE; - acx_start(ifp); + ifp->if_start(ifp); } } diff --git a/sys/dev/netif/an/if_an.c b/sys/dev/netif/an/if_an.c index 08b1e1101e..1ab8055ccf 100644 --- a/sys/dev/netif/an/if_an.c +++ b/sys/dev/netif/an/if_an.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/an/if_an.c,v 1.2.2.13 2003/02/11 03:32:48 ambrisko Exp $ - * $DragonFly: src/sys/dev/netif/an/if_an.c,v 1.42 2007/04/07 11:01:06 swildner Exp $ + * $DragonFly: src/sys/dev/netif/an/if_an.c,v 1.43 2008/05/14 11:59:18 sephe Exp $ */ /* @@ -1181,9 +1181,7 @@ an_intr(void *xsc) CSR_WRITE_2(sc, AN_INT_EN(sc->mpi350), AN_INTRS(sc->mpi350)); if ((ifp->if_flags & IFF_UP) && !ifq_is_empty(&ifp->if_snd)) - an_start(ifp); - - return; + if_devstart(ifp); } static int @@ -2492,8 +2490,10 @@ an_start(struct ifnet *ifp) if (ifp->if_flags & IFF_OACTIVE) return; - if (!sc->an_associated) + if (!sc->an_associated) { + ifq_purge(&ifp->if_snd); return; + } /* We can't send in monitor mode so toss any attempts. */ if (sc->an_monitor && (ifp->if_flags & IFF_PROMISC)) { @@ -2699,12 +2699,11 @@ an_shutdown(device_t dev) void an_resume(device_t dev) { - struct an_softc *sc; - struct ifnet *ifp; - int i; + struct an_softc *sc = device_get_softc(dev); + struct ifnet *ifp = &sc->arpcom.ac_if; + int i; - sc = device_get_softc(dev); - ifp = &sc->arpcom.ac_if; + lwkt_serialize_enter(ifp->if_serializer); an_reset(sc); if (sc->mpi350) @@ -2721,9 +2720,9 @@ an_resume(device_t dev) } if (ifp->if_flags & IFF_UP) - an_start(ifp); + if_devstart(ifp); - return; + lwkt_serialize_exit(ifp->if_serializer); } #ifdef ANCACHE diff --git a/sys/dev/netif/an/if_an_isa.c b/sys/dev/netif/an/if_an_isa.c index 95b8ae20f7..006b71d2b0 100644 --- a/sys/dev/netif/an/if_an_isa.c +++ b/sys/dev/netif/an/if_an_isa.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/an/if_an_isa.c,v 1.1.2.5 2003/02/01 03:25:12 ambrisko Exp $ - * $DragonFly: src/sys/dev/netif/an/if_an_isa.c,v 1.16 2006/10/25 20:55:55 dillon Exp $ + * $DragonFly: src/sys/dev/netif/an/if_an_isa.c,v 1.17 2008/05/14 11:59:18 sephe Exp $ */ /* @@ -52,7 +52,7 @@ #include #include #include - +#include #include #include #include @@ -103,6 +103,7 @@ static int an_attach_isa(device_t dev) { struct an_softc *sc = device_get_softc(dev); + struct ifnet *ifp = &sc->arpcom.ac_if; int flags = device_get_flags(dev); int error; @@ -125,6 +126,9 @@ an_attach_isa(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->irq_res)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return (0); fail: diff --git a/sys/dev/netif/an/if_an_pccard.c b/sys/dev/netif/an/if_an_pccard.c index 194bbf9ae6..7902fbe6e6 100644 --- a/sys/dev/netif/an/if_an_pccard.c +++ b/sys/dev/netif/an/if_an_pccard.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/an/if_an_pccard.c,v 1.1.2.6 2003/02/01 03:25:12 ambrisko Exp $ - * $DragonFly: src/sys/dev/netif/an/if_an_pccard.c,v 1.15 2006/10/25 20:55:55 dillon Exp $ + * $DragonFly: src/sys/dev/netif/an/if_an_pccard.c,v 1.16 2008/05/14 11:59:18 sephe Exp $ */ /* @@ -51,7 +51,7 @@ #include #include #include - +#include #include #include #include @@ -143,6 +143,7 @@ static int an_pccard_attach(device_t dev) { struct an_softc *sc = device_get_softc(dev); + struct ifnet *ifp = &sc->arpcom.ac_if; int flags = device_get_flags(dev); int error; @@ -167,6 +168,10 @@ an_pccard_attach(device_t dev) ifmedia_removeall(&sc->an_ifmedia); goto fail; } + + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->irq_res)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return 0; fail: diff --git a/sys/dev/netif/an/if_an_pci.c b/sys/dev/netif/an/if_an_pci.c index 07d77db18f..e2f95ce04f 100644 --- a/sys/dev/netif/an/if_an_pci.c +++ b/sys/dev/netif/an/if_an_pci.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/an/if_an_pci.c,v 1.2.2.8 2003/02/11 03:32:48 ambrisko Exp $ - * $DragonFly: src/sys/dev/netif/an/if_an_pci.c,v 1.20 2006/10/25 20:55:55 dillon Exp $ + * $DragonFly: src/sys/dev/netif/an/if_an_pci.c,v 1.21 2008/05/14 11:59:18 sephe Exp $ */ /* @@ -66,7 +66,7 @@ #include #include #include - +#include #include #include #include @@ -139,9 +139,11 @@ static int an_attach_pci(device_t dev) { struct an_softc *sc; + struct ifnet *ifp; int flags, error; sc = device_get_softc(dev); + ifp = &sc->arpcom.ac_if; flags = device_get_flags(dev); error = an_alloc_port(dev, sc->port_rid, 1); @@ -211,6 +213,9 @@ an_attach_pci(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->irq_res)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return(0); fail: diff --git a/sys/dev/netif/ath/ath/if_ath.c b/sys/dev/netif/ath/ath/if_ath.c index 8e6b379c47..905c54e995 100644 --- a/sys/dev/netif/ath/ath/if_ath.c +++ b/sys/dev/netif/ath/ath/if_ath.c @@ -34,7 +34,7 @@ * THE POSSIBILITY OF SUCH DAMAGES. * * $FreeBSD: src/sys/dev/ath/if_ath.c,v 1.94.2.23 2006/07/10 01:15:24 sam Exp $ - * $DragonFly: src/sys/dev/netif/ath/ath/if_ath.c,v 1.8 2007/09/15 21:24:59 swildner Exp $ + * $DragonFly: src/sys/dev/netif/ath/ath/if_ath.c,v 1.9 2008/05/14 11:59:18 sephe Exp $ */ /* @@ -62,6 +62,7 @@ #include #include #include +#include #include #include @@ -646,6 +647,9 @@ ath_attach(uint16_t devid, struct ath_softc *sc) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->sc_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + if (bootverbose) ieee80211_announce(ic); ath_announce(sc); @@ -748,7 +752,7 @@ ath_resume(struct ath_softc *sc) if (ifp->if_flags & IFF_UP) { ath_init(sc); if (ifp->if_flags & IFF_RUNNING) - ath_start(ifp); + ifp->if_start(ifp); } if (sc->sc_softled) { ath_hal_gpioCfgOutput(sc->sc_ah, sc->sc_ledpin); @@ -1179,7 +1183,7 @@ ath_reset(struct ifnet *ifp) ath_beacon_config(sc); /* restart beacons */ ath_hal_intrset(ah, sc->sc_imask); - ath_start(ifp); /* restart xmit */ + ifp->if_start(ifp); /* restart xmit */ return 0; } @@ -1195,7 +1199,13 @@ ath_start(struct ifnet *ifp) struct ieee80211_frame *wh; struct ether_header *eh; - if ((ifp->if_flags & IFF_RUNNING) == 0 || sc->sc_invalid) + if (sc->sc_invalid) { + ifq_purge(&ifp->if_snd); + ieee80211_drain_mgtq(&ic->ic_mgtq); + return; + } + + if ((ifp->if_flags & IFF_RUNNING) == 0) return; for (;;) { @@ -1228,6 +1238,7 @@ ath_start(struct ifnet *ifp) ieee80211_state_name[ic->ic_state]); sc->sc_stats.ast_tx_discard++; STAILQ_INSERT_TAIL(&sc->sc_txbuf, bf, bf_list); + ifq_purge(&ifp->if_snd); break; } m = ifq_dequeue(&ifp->if_snd, NULL); @@ -4043,7 +4054,7 @@ ath_tx_proc_q0(struct ath_softc *sc) if (sc->sc_softled) ath_led_event(sc, ATH_LED_TX); - ath_start(ifp); + ifp->if_start(ifp); } /* @@ -4079,7 +4090,7 @@ ath_tx_proc_q0123(struct ath_softc *sc) if (sc->sc_softled) ath_led_event(sc, ATH_LED_TX); - ath_start(ifp); + ifp->if_start(ifp); } /* @@ -4107,7 +4118,7 @@ ath_tx_proc(struct ath_softc *sc) if (sc->sc_softled) ath_led_event(sc, ATH_LED_TX); - ath_start(ifp); + ifp->if_start(ifp); } static void diff --git a/sys/dev/netif/aue/if_aue.c b/sys/dev/netif/aue/if_aue.c index f1816b8743..acc141082c 100644 --- a/sys/dev/netif/aue/if_aue.c +++ b/sys/dev/netif/aue/if_aue.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/usb/if_aue.c,v 1.78 2003/12/17 14:23:07 sanpei Exp $ - * $DragonFly: src/sys/dev/netif/aue/if_aue.c,v 1.37 2008/01/06 16:55:50 swildner Exp $ + * $DragonFly: src/sys/dev/netif/aue/if_aue.c,v 1.38 2008/05/14 11:59:18 sephe Exp $ */ /* @@ -962,7 +962,8 @@ aue_rxeof(usbd_xfer_handle xfer, usbd_private_handle priv, usbd_status status) usb_ether_input(m); aue_rxstart(ifp); if (!ifq_is_empty(&ifp->if_snd)) - (*ifp->if_start)(ifp); + if_devstart(ifp); + AUE_UNLOCK(sc); return; done: @@ -1020,7 +1021,7 @@ aue_txeof(usbd_xfer_handle xfer, usbd_private_handle priv, usbd_status status) ifp->if_opackets++; if (!ifq_is_empty(&ifp->if_snd)) - (*ifp->if_start)(ifp); + if_devstart(ifp); AUE_UNLOCK(sc); @@ -1051,7 +1052,7 @@ aue_tick(void *xsc) IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) { sc->aue_link++; if (!ifq_is_empty(&ifp->if_snd)) - aue_start(ifp); + if_devstart(ifp); } callout_reset(&sc->aue_stat_timer, hz, aue_tick, sc); @@ -1113,6 +1114,7 @@ aue_start(struct ifnet *ifp) AUE_LOCK(sc); if (!sc->aue_link) { + ifq_purge(&ifp->if_snd); AUE_UNLOCK(sc); return; } @@ -1122,18 +1124,18 @@ aue_start(struct ifnet *ifp) return; } - m_head = ifq_poll(&ifp->if_snd); + m_head = ifq_dequeue(&ifp->if_snd, NULL); if (m_head == NULL) { AUE_UNLOCK(sc); return; } if (aue_encap(sc, m_head, 0)) { + /* aue_encap() will free m_head, if we reach here */ ifp->if_flags |= IFF_OACTIVE; AUE_UNLOCK(sc); return; } - ifq_dequeue(&ifp->if_snd, m_head); /* * If there's a BPF listener, bounce a copy of this frame @@ -1365,7 +1367,7 @@ aue_watchdog(struct ifnet *ifp) aue_txeof(c->aue_xfer, c, stat); if (!ifq_is_empty(&ifp->if_snd)) - aue_start(ifp); + if_devstart(ifp); AUE_UNLOCK(sc); return; } diff --git a/sys/dev/netif/axe/if_axe.c b/sys/dev/netif/axe/if_axe.c index 8f81f274ab..91fcc7f5e2 100644 --- a/sys/dev/netif/axe/if_axe.c +++ b/sys/dev/netif/axe/if_axe.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/usb/if_axe.c,v 1.10 2003/12/08 07:54:14 obrien Exp $ - * $DragonFly: src/sys/dev/netif/axe/if_axe.c,v 1.27 2008/01/06 16:55:50 swildner Exp $ + * $DragonFly: src/sys/dev/netif/axe/if_axe.c,v 1.28 2008/05/14 11:59:18 sephe Exp $ */ /* * ASIX Electronics AX88172 USB 2.0 ethernet driver. Used in the @@ -696,7 +696,8 @@ axe_txeof(usbd_xfer_handle xfer, usbd_private_handle priv, usbd_status status) ifp->if_opackets++; if (!ifq_is_empty(&ifp->if_snd)) - (*ifp->if_start)(ifp); + if_devstart(ifp); + lwkt_serialize_exit(ifp->if_serializer); } @@ -717,7 +718,7 @@ axe_tick(void *xsc) IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) { sc->axe_link++; if (!ifq_is_empty(&ifp->if_snd)) - axe_start(ifp); + if_devstart(ifp); } callout_reset(&sc->axe_stat_timer, hz, axe_tick, sc); lwkt_serialize_exit(ifp->if_serializer); @@ -761,6 +762,7 @@ axe_start(struct ifnet *ifp) struct mbuf *m_head = NULL; if (!sc->axe_link) { + ifq_purge(&ifp->if_snd); return; } @@ -768,16 +770,15 @@ axe_start(struct ifnet *ifp) return; } - m_head = ifq_poll(&ifp->if_snd); - if (m_head == NULL) { + m_head = ifq_dequeue(&ifp->if_snd, NULL); + if (m_head == NULL) return; - } if (axe_encap(sc, m_head, 0)) { + /* axe_encap() will free m_head, if we reach here */ ifp->if_flags |= IFF_OACTIVE; return; } - ifq_dequeue(&ifp->if_snd, m_head); /* * If there's a BPF listener, bounce a copy of this frame @@ -953,7 +954,7 @@ axe_watchdog(struct ifnet *ifp) axe_txeof(c->axe_xfer, c, stat); if (!ifq_is_empty(&ifp->if_snd)) - axe_start(ifp); + if_devstart(ifp); } /* diff --git a/sys/dev/netif/bce/if_bce.c b/sys/dev/netif/bce/if_bce.c index 3299cd763d..81f34538c6 100644 --- a/sys/dev/netif/bce/if_bce.c +++ b/sys/dev/netif/bce/if_bce.c @@ -28,7 +28,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/bce/if_bce.c,v 1.31 2007/05/16 23:34:11 davidch Exp $ - * $DragonFly: src/sys/dev/netif/bce/if_bce.c,v 1.3 2008/03/10 12:59:51 sephe Exp $ + * $DragonFly: src/sys/dev/netif/bce/if_bce.c,v 1.4 2008/05/14 11:59:18 sephe Exp $ */ /* @@ -50,6 +50,7 @@ #include #include #include +#include #include #include #include @@ -744,6 +745,9 @@ bce_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->bce_res_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + /* Print some important debugging info. */ DBRUN(BCE_INFO, bce_dump_driver_state(sc)); @@ -4471,8 +4475,12 @@ bce_start(struct ifnet *ifp) ASSERT_SERIALIZED(ifp->if_serializer); /* If there's no link or the transmit queue is empty then just exit. */ - if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING || - !sc->bce_link) + if (!sc->bce_link) { + ifq_purge(&ifp->if_snd); + return; + } + + if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING) return; DBPRINT(sc, BCE_INFO_SEND, @@ -4666,7 +4674,7 @@ bce_watchdog(struct ifnet *ifp) ifp->if_oerrors++; if (!ifq_is_empty(&ifp->if_snd)) - ifp->if_start(ifp); + if_devstart(ifp); } @@ -4753,7 +4761,7 @@ bce_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) /* Check for new frames to transmit. */ if (!ifq_is_empty(&ifp->if_snd)) - ifp->if_start(ifp); + if_devstart(ifp); } #endif /* DEVICE_POLLING */ @@ -4896,7 +4904,7 @@ bce_intr(void *xsc) /* Handle any frames that arrived while handling the interrupt. */ if (!ifq_is_empty(&ifp->if_snd)) - ifp->if_start(ifp); + if_devstart(ifp); } @@ -5252,7 +5260,7 @@ bce_tick_serialized(struct bce_softc *sc) sc->bce_link++; /* Now that link is up, handle any outstanding TX traffic. */ if (!ifq_is_empty(&ifp->if_snd)) - ifp->if_start(ifp); + if_devstart(ifp); } } diff --git a/sys/dev/netif/bfe/if_bfe.c b/sys/dev/netif/bfe/if_bfe.c index a1920f647e..b66013cfc7 100644 --- a/sys/dev/netif/bfe/if_bfe.c +++ b/sys/dev/netif/bfe/if_bfe.c @@ -29,7 +29,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/dev/bfe/if_bfe.c 1.4.4.7 2004/03/02 08:41:33 julian Exp v - * $DragonFly: src/sys/dev/netif/bfe/if_bfe.c,v 1.32 2008/03/23 09:44:36 sephe Exp $ + * $DragonFly: src/sys/dev/netif/bfe/if_bfe.c,v 1.33 2008/05/14 11:59:18 sephe Exp $ */ #include @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -331,6 +332,7 @@ bfe_attach(device_t dev) sc->bfe_dev = dev; callout_init(&sc->bfe_stat_timer); +#ifndef BURN_BRIDGES /* * Handle power management nonsense. */ @@ -351,6 +353,7 @@ bfe_attach(device_t dev) pci_write_config(dev, BFE_PCI_MEMLO, membase, 4); pci_write_config(dev, BFE_PCI_INTLINE, irq, 4); } +#endif /* !BURN_BRIDGE */ /* * Map control/status registers. @@ -428,6 +431,9 @@ bfe_attach(device_t dev) device_printf(dev, "couldn't set up irq\n"); goto fail; } + + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->bfe_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); return 0; fail: bfe_detach(dev); @@ -1194,7 +1200,7 @@ bfe_intr(void *xsc) /* We have packets pending, fire them out */ if ((ifp->if_flags & IFF_RUNNING) && !ifq_is_empty(&ifp->if_snd)) - bfe_start(ifp); + if_devstart(ifp); } static int @@ -1301,8 +1307,10 @@ bfe_start(struct ifnet *ifp) * Not much point trying to send if the link is down * or we have nothing to send. */ - if (!sc->bfe_link) + if (!sc->bfe_link) { + ifq_purge(&ifp->if_snd); return; + } if (ifp->if_flags & IFF_OACTIVE) return; diff --git a/sys/dev/netif/bge/if_bge.c b/sys/dev/netif/bge/if_bge.c index d7fb71217b..a577138ebe 100644 --- a/sys/dev/netif/bge/if_bge.c +++ b/sys/dev/netif/bge/if_bge.c @@ -31,7 +31,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/bge/if_bge.c,v 1.3.2.39 2005/07/03 03:41:18 silby Exp $ - * $DragonFly: src/sys/dev/netif/bge/if_bge.c,v 1.90 2008/03/10 12:59:51 sephe Exp $ + * $DragonFly: src/sys/dev/netif/bge/if_bge.c,v 1.91 2008/05/14 11:59:18 sephe Exp $ * */ @@ -78,6 +78,7 @@ #include #include #include +#include #include #include #include @@ -1955,6 +1956,10 @@ bge_attach(device_t dev) device_printf(dev, "couldn't set up irq\n"); goto fail; } + + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->bge_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return(0); fail: bge_detach(dev); @@ -2375,7 +2380,7 @@ bge_txeof(struct bge_softc *sc) ifp->if_timer = 0; if (!ifq_is_empty(&ifp->if_snd)) - ifp->if_start(ifp); + if_devstart(ifp); } #ifdef DEVICE_POLLING @@ -2727,7 +2732,7 @@ bge_start(struct ifnet *ifp) need_trans = 0; while (sc->bge_cdata.bge_tx_chain[prodidx] == NULL) { - m_head = ifq_poll(&ifp->if_snd); + m_head = ifq_dequeue(&ifp->if_snd, NULL); if (m_head == NULL) break; @@ -2744,11 +2749,12 @@ bge_start(struct ifnet *ifp) * chain at once. * (paranoia -- may not actually be needed) */ - if (m_head->m_flags & M_FIRSTFRAG && - m_head->m_pkthdr.csum_flags & (CSUM_DELAY_DATA)) { + if ((m_head->m_flags & M_FIRSTFRAG) && + (m_head->m_pkthdr.csum_flags & CSUM_DELAY_DATA)) { if ((BGE_TX_RING_CNT - sc->bge_txcnt) < - m_head->m_pkthdr.csum_data + 16) { + m_head->m_pkthdr.csum_data + BGE_NSEG_RSVD) { ifp->if_flags |= IFF_OACTIVE; + ifq_prepend(&ifp->if_snd, m_head); break; } } @@ -2762,15 +2768,10 @@ bge_start(struct ifnet *ifp) if ((BGE_TX_RING_CNT - sc->bge_txcnt) < (BGE_NSEG_RSVD + BGE_NSEG_SPARE)) { ifp->if_flags |= IFF_OACTIVE; + ifq_prepend(&ifp->if_snd, m_head); break; } - /* - * Dequeue the packet before encapsulation, since - * bge_encap() may free the packet if error happens. - */ - ifq_dequeue(&ifp->if_snd, m_head); - /* * Pack the data into the transmit ring. If we * don't have room, set the OACTIVE flag and wait @@ -3093,7 +3094,7 @@ bge_watchdog(struct ifnet *ifp) ifp->if_oerrors++; if (!ifq_is_empty(&ifp->if_snd)) - ifp->if_start(ifp); + if_devstart(ifp); } /* @@ -3242,7 +3243,7 @@ bge_resume(device_t dev) bge_init(sc); if (!ifq_is_empty(&ifp->if_snd)) - ifp->if_start(ifp); + if_devstart(ifp); } lwkt_serialize_exit(ifp->if_serializer); diff --git a/sys/dev/netif/bwi/if_bwi.c b/sys/dev/netif/bwi/if_bwi.c index ed08a81ebb..90f4b59510 100644 --- a/sys/dev/netif/bwi/if_bwi.c +++ b/sys/dev/netif/bwi/if_bwi.c @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sys/dev/netif/bwi/if_bwi.c,v 1.20 2008/02/15 11:48:15 sephe Exp $ + * $DragonFly: src/sys/dev/netif/bwi/if_bwi.c,v 1.21 2008/05/14 11:59:19 sephe Exp $ */ #include @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -814,6 +815,9 @@ bwi_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->sc_irq_res)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + if (bootverbose) ieee80211_announce(ic); @@ -1510,7 +1514,7 @@ back: if (error) bwi_stop(sc, 1); else - bwi_start(ifp); + ifp->if_start(ifp); } static int @@ -1601,8 +1605,10 @@ bwi_start(struct ifnet *ifp) } else if (!ifq_is_empty(&ifp->if_snd)) { struct ether_header *eh; - if (ic->ic_state != IEEE80211_S_RUN) + if (ic->ic_state != IEEE80211_S_RUN) { + ifq_purge(&ifp->if_snd); break; + } m = ifq_dequeue(&ifp->if_snd, NULL); if (m == NULL) @@ -2706,7 +2712,9 @@ static int bwi_set_chan(struct bwi_softc *sc, struct ieee80211_channel *c) { struct ieee80211com *ic = &sc->sc_ic; +#ifdef INVARIANTS struct ifnet *ifp = &ic->ic_if; +#endif struct bwi_mac *mac; uint16_t flags; u_int chan; diff --git a/sys/dev/netif/cs/if_cs.c b/sys/dev/netif/cs/if_cs.c index 7a4c5ef61c..47f7064779 100644 --- a/sys/dev/netif/cs/if_cs.c +++ b/sys/dev/netif/cs/if_cs.c @@ -25,7 +25,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/dev/cs/if_cs.c,v 1.19.2.1 2001/01/25 20:13:48 imp Exp $ - * $DragonFly: src/sys/dev/netif/cs/if_cs.c,v 1.28 2006/12/22 23:26:19 swildner Exp $ + * $DragonFly: src/sys/dev/netif/cs/if_cs.c,v 1.29 2008/05/14 11:59:19 sephe Exp $ */ /* @@ -39,6 +39,7 @@ #include #include +#include #include #include #include @@ -684,6 +685,9 @@ cs_attach(device_t dev) goto bad; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->irq_res)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return 0; bad: @@ -797,7 +801,7 @@ cs_init(void *xsc) /* * Start sending process */ - cs_start(ifp); + if_devstart(&sc->arpcom.ac_if); } /* @@ -928,9 +932,8 @@ csintr(void *arg) } } - if (!(ifp->if_flags & IFF_OACTIVE)) { - cs_start(ifp); - } + if (!(ifp->if_flags & IFF_OACTIVE)) + if_devstart(ifp); } /* diff --git a/sys/dev/netif/cue/if_cue.c b/sys/dev/netif/cue/if_cue.c index 7c7a890a87..5a16abfb51 100644 --- a/sys/dev/netif/cue/if_cue.c +++ b/sys/dev/netif/cue/if_cue.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/usb/if_cue.c,v 1.45 2003/12/08 07:54:14 obrien Exp $ - * $DragonFly: src/sys/dev/netif/cue/if_cue.c,v 1.31 2007/11/06 07:37:00 hasso Exp $ + * $DragonFly: src/sys/dev/netif/cue/if_cue.c,v 1.32 2008/05/14 11:59:19 sephe Exp $ */ /* @@ -739,7 +739,7 @@ cue_txeof(usbd_xfer_handle xfer, usbd_private_handle priv, usbd_status status) ifp->if_opackets++; if (!ifq_is_empty(&ifp->if_snd)) - (*ifp->if_start)(ifp); + if_devstart(ifp); CUE_UNLOCK(sc); @@ -826,18 +826,18 @@ cue_start(struct ifnet *ifp) return; } - m_head = ifq_poll(&ifp->if_snd); + m_head = ifq_dequeue(&ifp->if_snd, NULL); if (m_head == NULL) { CUE_UNLOCK(sc); return; } if (cue_encap(sc, m_head, 0)) { + /* cue_encap() will free m_head, if we reach here */ ifp->if_flags |= IFF_OACTIVE; CUE_UNLOCK(sc); return; } - ifq_dequeue(&ifp->if_snd, m_head); /* * If there's a BPF listener, bounce a copy of this frame @@ -1018,7 +1018,7 @@ cue_watchdog(struct ifnet *ifp) cue_txeof(c->cue_xfer, c, stat); if (!ifq_is_empty(&ifp->if_snd)) - cue_start(ifp); + if_devstart(ifp); CUE_UNLOCK(sc); return; diff --git a/sys/dev/netif/dc/if_dc.c b/sys/dev/netif/dc/if_dc.c index cf6b8643e4..4a86e27a99 100644 --- a/sys/dev/netif/dc/if_dc.c +++ b/sys/dev/netif/dc/if_dc.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/pci/if_dc.c,v 1.9.2.45 2003/06/08 14:31:53 mux Exp $ - * $DragonFly: src/sys/dev/netif/dc/if_dc.c,v 1.55 2008/01/05 14:02:37 swildner Exp $ + * $DragonFly: src/sys/dev/netif/dc/if_dc.c,v 1.56 2008/05/14 11:59:19 sephe Exp $ */ /* @@ -98,6 +98,7 @@ #include #include #include +#include #include #include #include @@ -2180,6 +2181,9 @@ dc_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->dc_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return(0); fail: @@ -2763,7 +2767,7 @@ dc_tick(void *xsc) IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) { sc->dc_link++; if (!ifq_is_empty(&ifp->if_snd)) - dc_start(ifp); + if_devstart(ifp); } } @@ -2848,14 +2852,14 @@ dc_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) dc_rxeof(sc); dc_txeof(sc); if ((ifp->if_flags & IFF_OACTIVE) == 0 && !ifq_is_empty(&ifp->if_snd)) - dc_start(ifp); + if_devstart(ifp); break; case POLL_AND_CHECK_STATUS: sc->rxcycles = count; dc_rxeof(sc); dc_txeof(sc); if ((ifp->if_flags & IFF_OACTIVE) == 0 && !ifq_is_empty(&ifp->if_snd)) - dc_start(ifp); + if_devstart(ifp); status = CSR_READ_4(sc, DC_ISR); status &= (DC_ISR_RX_WATDOGTIMEO|DC_ISR_RX_NOBUF| DC_ISR_TX_NOBUF|DC_ISR_TX_IDLE|DC_ISR_TX_UNDERRUN| @@ -2967,9 +2971,7 @@ dc_intr(void *arg) CSR_WRITE_4(sc, DC_IMR, DC_INTRS); if (!ifq_is_empty(&ifp->if_snd)) - dc_start(ifp); - - return; + if_devstart(ifp); } /* @@ -3045,14 +3047,15 @@ static void dc_start(struct ifnet *ifp) { struct dc_softc *sc; - struct mbuf *m_head; - struct mbuf *m_defragged; + struct mbuf *m_head, *m_defragged; int idx, need_trans; sc = ifp->if_softc; - if (!sc->dc_link) + if (!sc->dc_link) { + ifq_purge(&ifp->if_snd); return; + } if (ifp->if_flags & IFF_OACTIVE) return; @@ -3062,13 +3065,12 @@ dc_start(struct ifnet *ifp) need_trans = 0; while(sc->dc_cdata.dc_tx_chain[idx] == NULL) { m_defragged = NULL; - m_head = ifq_poll(&ifp->if_snd); + m_head = ifq_dequeue(&ifp->if_snd, NULL); if (m_head == NULL) break; - if (sc->dc_flags & DC_TX_COALESCE && - (m_head->m_next != NULL || - sc->dc_flags & DC_TX_ALIGN)){ + if ((sc->dc_flags & DC_TX_COALESCE) && + (m_head->m_next != NULL || (sc->dc_flags & DC_TX_ALIGN))) { /* * Check first if coalescing allows us to queue * the packet. We don't want to loose it if @@ -3078,52 +3080,47 @@ dc_start(struct ifnet *ifp) idx != sc->dc_cdata.dc_tx_prod && idx == (DC_TX_LIST_CNT - 1)) { ifp->if_flags |= IFF_OACTIVE; + ifq_prepend(&ifp->if_snd, m_head); break; } if ((DC_TX_LIST_CNT - sc->dc_cdata.dc_tx_cnt) < 5) { ifp->if_flags |= IFF_OACTIVE; + ifq_prepend(&ifp->if_snd, m_head); break; } /* only coalesce if have >1 mbufs */ - m_defragged = m_defrag_nofree(m_head, MB_DONTWAIT); + m_defragged = m_defrag(m_head, MB_DONTWAIT); if (m_defragged == NULL) { ifp->if_flags |= IFF_OACTIVE; + ifq_prepend(&ifp->if_snd, m_head); break; } + m_head = m_defragged; } - if (dc_encap(sc, (m_defragged ? m_defragged : m_head), &idx)) { + if (dc_encap(sc, m_head, &idx)) { if (m_defragged) { /* * Throw away the original packet if the * defragged packet could not be encapsulated, * as well as the defragged packet. */ - ifq_dequeue(&ifp->if_snd, m_head); m_freem(m_head); - m_freem(m_defragged); + } else { + ifq_prepend(&ifp->if_snd, m_head); } ifp->if_flags |= IFF_OACTIVE; break; } - ifq_dequeue(&ifp->if_snd, m_head); - need_trans = 1; /* * If there's a BPF listener, bounce a copy of this frame * to him. */ - BPF_MTAP(ifp, (m_defragged ? m_defragged : m_head)); - - /* - * If we defragged the packet, m_head is not the one we - * encapsulated so we can throw it away. - */ - if (m_defragged) - m_freem(m_head); + BPF_MTAP(ifp, m_head); if (sc->dc_flags & DC_TX_ONE) { ifp->if_flags |= IFF_OACTIVE; @@ -3432,9 +3429,7 @@ dc_watchdog(struct ifnet *ifp) dc_init(sc); if (!ifq_is_empty(&ifp->if_snd)) - dc_start(ifp); - - return; + if_devstart(ifp); } /* @@ -3488,11 +3483,8 @@ dc_stop(struct dc_softc *sc) sc->dc_cdata.dc_tx_chain[i] = NULL; } } - bzero((char *)&sc->dc_ldata->dc_tx_list, sizeof(sc->dc_ldata->dc_tx_list)); - - return; } /* diff --git a/sys/dev/netif/de/if_de.c b/sys/dev/netif/de/if_de.c index 1bb0912fd6..aec41b39e1 100644 --- a/sys/dev/netif/de/if_de.c +++ b/sys/dev/netif/de/if_de.c @@ -1,7 +1,7 @@ /* $NetBSD: if_de.c,v 1.86 1999/06/01 19:17:59 thorpej Exp $ */ /* $FreeBSD: src/sys/pci/if_de.c,v 1.123.2.4 2000/08/04 23:25:09 peter Exp $ */ -/* $DragonFly: src/sys/dev/netif/de/if_de.c,v 1.47 2006/12/22 23:26:19 swildner Exp $ */ +/* $DragonFly: src/sys/dev/netif/de/if_de.c,v 1.48 2008/05/14 11:59:19 sephe Exp $ */ /*- * Copyright (c) 1994-1997 Matt Thomas (matt@3am-software.com) @@ -51,6 +51,7 @@ #include #include #include +#include #include "opt_inet.h" #include "opt_ipx.h" @@ -58,7 +59,7 @@ #include #include #include - +#include #include #ifdef INET @@ -3426,7 +3427,7 @@ tulip_intr_handler(tulip_softc_t *sc) if (sc->tulip_flags & (TULIP_WANTTXSTART|TULIP_TXPROBE_ACTIVE|TULIP_DOINGSETUP|TULIP_PROMISC)) { tulip_tx_intr(sc); if ((sc->tulip_flags & TULIP_TXPROBE_ACTIVE) == 0) - tulip_ifstart(&sc->tulip_if); + if_devstart(&sc->tulip_if); } } if (sc->tulip_flags & TULIP_NEEDRESET) { @@ -3854,9 +3855,15 @@ tulip_ifstart(struct ifnet *ifp) while (sc->tulip_if.if_snd.ifq_head != NULL) { struct mbuf *m; + + ALTQ_LOCK(&sc->tulip_if.if_snd); IF_DEQUEUE(&sc->tulip_if.if_snd, m); + ALTQ_UNLOCK(&sc->tulip_if.if_snd); + if ((m = tulip_txput(sc, m)) != NULL) { + ALTQ_LOCK(&sc->tulip_if.if_snd); IF_PREPEND(&sc->tulip_if.if_snd, m); + ALTQ_UNLOCK(&sc->tulip_if.if_snd); break; } } @@ -4191,6 +4198,10 @@ tulip_pci_attach(device_t dev) kfree((caddr_t) sc->tulip_txdescs, M_DEVBUF); return ENXIO; } + + sc->tulip_if.if_cpuid = ithread_cpuid(rman_get_start(res)); + KKASSERT(sc->tulip_if.if_cpuid >= 0 && + sc->tulip_if.if_cpuid < ncpus); } } return 0; diff --git a/sys/dev/netif/ed/if_ed.c b/sys/dev/netif/ed/if_ed.c index d80e41a9c6..60159a11db 100644 --- a/sys/dev/netif/ed/if_ed.c +++ b/sys/dev/netif/ed/if_ed.c @@ -25,7 +25,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/dev/ed/if_ed.c,v 1.224 2003/12/08 07:54:12 obrien Exp $ - * $DragonFly: src/sys/dev/netif/ed/if_ed.c,v 1.34 2007/05/13 18:33:57 swildner Exp $ + * $DragonFly: src/sys/dev/netif/ed/if_ed.c,v 1.35 2008/05/14 11:59:19 sephe Exp $ */ /* @@ -1991,7 +1991,7 @@ ed_init(void *xsc) /* * ...and attempt to start output */ - ed_start(ifp); + if_devstart(ifp); #ifndef ED_NO_MIIBUS callout_reset(&sc->ed_timer, hz, ed_tick, sc); @@ -2068,6 +2068,7 @@ ed_start(struct ifnet *ifp) if (sc->gone) { kprintf("ed_start(%p) GONE\n",ifp); + ifq_purge(&ifp->if_snd); return; } outloop: @@ -2580,7 +2581,7 @@ edintr(void *arg) * after handling the receiver to give the receiver priority. */ if ((ifp->if_flags & IFF_OACTIVE) == 0) - ed_start(ifp); + if_devstart(ifp); /* * return NIC CR to standard state: page 0, remote DMA diff --git a/sys/dev/netif/ed/if_ed_isa.c b/sys/dev/netif/ed/if_ed_isa.c index 78d5fdbbb0..3e7536e16a 100644 --- a/sys/dev/netif/ed/if_ed_isa.c +++ b/sys/dev/netif/ed/if_ed_isa.c @@ -25,7 +25,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/dev/ed/if_ed_isa.c,v 1.15 2003/10/31 18:31:58 brooks Exp $ - * $DragonFly: src/sys/dev/netif/ed/if_ed_isa.c,v 1.15 2006/10/25 20:55:56 dillon Exp $ + * $DragonFly: src/sys/dev/netif/ed/if_ed_isa.c,v 1.16 2008/05/14 11:59:19 sephe Exp $ */ #include @@ -34,6 +34,8 @@ #include #include #include +#include +#include #include #include @@ -130,11 +132,18 @@ ed_isa_attach(device_t dev) error = ed_attach(dev); if (error == 0) { + struct ifnet *ifp = &sc->arpcom.ac_if; + error = bus_setup_intr(dev, sc->irq_res, INTR_NETSAFE, edintr, sc, &sc->irq_handle, - sc->arpcom.ac_if.if_serializer); - if (error) + ifp->if_serializer); + if (error) { ed_isa_detach(dev); + } else { + ifp->if_cpuid = + ithread_cpuid(rman_get_start(sc->irq_res)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + } } else { ed_release_resources(dev); } diff --git a/sys/dev/netif/ed/if_ed_pccard.c b/sys/dev/netif/ed/if_ed_pccard.c index 25017e5f3e..cc31366b96 100644 --- a/sys/dev/netif/ed/if_ed_pccard.c +++ b/sys/dev/netif/ed/if_ed_pccard.c @@ -25,7 +25,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/dev/ed/if_ed_pccard.c,v 1.55 2003/12/31 04:25:00 kato Exp $ - * $DragonFly: src/sys/dev/netif/ed/if_ed_pccard.c,v 1.20 2007/08/14 15:32:32 sephe Exp $ + * $DragonFly: src/sys/dev/netif/ed/if_ed_pccard.c,v 1.21 2008/05/14 11:59:19 sephe Exp $ */ #include "opt_ed.h" @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -250,6 +251,7 @@ static int ed_pccard_attach(device_t dev) { struct ed_softc *sc = device_get_softc(dev); + struct ifnet *ifp = &sc->arpcom.ac_if; int error; int i; uint8_t sum; @@ -283,11 +285,14 @@ ed_pccard_attach(device_t dev) error = bus_setup_intr(dev, sc->irq_res, INTR_NETSAFE, edintr, sc, &sc->irq_handle, - sc->arpcom.ac_if.if_serializer); + ifp->if_serializer); if (error) { kprintf("setup intr failed %d \n", error); ed_release_resources(dev); return (error); + } else { + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->irq_res)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); } return (error); diff --git a/sys/dev/netif/ed/if_ed_pci.c b/sys/dev/netif/ed/if_ed_pci.c index 8a1b3cfc66..3bd6a867c1 100644 --- a/sys/dev/netif/ed/if_ed_pci.c +++ b/sys/dev/netif/ed/if_ed_pci.c @@ -18,7 +18,7 @@ * are met. * * $FreeBSD: src/sys/dev/ed/if_ed_pci.c,v 1.34 2003/10/31 18:31:58 brooks Exp $ - * $DragonFly: src/sys/dev/netif/ed/if_ed_pci.c,v 1.14 2006/10/25 20:55:56 dillon Exp $ + * $DragonFly: src/sys/dev/netif/ed/if_ed_pci.c,v 1.15 2008/05/14 11:59:19 sephe Exp $ */ #include @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -103,11 +104,18 @@ ed_pci_attach(device_t dev) error = ed_attach(dev); if (error == 0) { + struct ifnet *ifp = &sc->arpcom.ac_if; + error = bus_setup_intr(dev, sc->irq_res, INTR_NETSAFE, - edintr, sc, &sc->irq_handle, - sc->arpcom.ac_if.if_serializer); - if (error) + edintr, sc, &sc->irq_handle, + ifp->if_serializer); + if (error) { ed_pci_detach(dev); + } else { + ifp->if_cpuid = + ithread_cpuid(rman_get_start(sc->irq_res)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + } } else { ed_release_resources(dev); } diff --git a/sys/dev/netif/em/if_em.c b/sys/dev/netif/em/if_em.c index 07a7aa6bf1..9aebb86f69 100644 --- a/sys/dev/netif/em/if_em.c +++ b/sys/dev/netif/em/if_em.c @@ -64,7 +64,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sys/dev/netif/em/if_em.c,v 1.70 2008/05/02 07:40:32 sephe Exp $ + * $DragonFly: src/sys/dev/netif/em/if_em.c,v 1.71 2008/05/14 11:59:19 sephe Exp $ * $FreeBSD$ */ /* @@ -101,6 +101,7 @@ #include #include #include +#include #include #include #include @@ -469,12 +470,14 @@ static int em_attach(device_t dev) { struct adapter *adapter; + struct ifnet *ifp; int tsize, rsize; int error = 0; INIT_DEBUGOUT("em_attach: begin"); adapter = device_get_softc(dev); + ifp = &adapter->interface_data.ac_if; callout_init(&adapter->timer); callout_init(&adapter->tx_fifo_timer); @@ -678,14 +681,15 @@ em_attach(device_t dev) error = bus_setup_intr(dev, adapter->res_interrupt, INTR_NETSAFE, em_intr, adapter, - &adapter->int_handler_tag, - adapter->interface_data.ac_if.if_serializer); + &adapter->int_handler_tag, ifp->if_serializer); if (error) { device_printf(dev, "Error registering interrupt handler!\n"); - ether_ifdetach(&adapter->interface_data.ac_if); + ether_ifdetach(ifp); goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(adapter->res_interrupt)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); INIT_DEBUGOUT("em_attach: end"); return(0); @@ -792,8 +796,7 @@ em_resume(device_t dev) lwkt_serialize_enter(ifp->if_serializer); ifp->if_flags &= ~IFF_RUNNING; em_init(adapter); - if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) == (IFF_UP | IFF_RUNNING)) - em_start(ifp); + if_devstart(ifp); lwkt_serialize_exit(ifp->if_serializer); return bus_generic_resume(dev); @@ -819,20 +822,21 @@ em_start(struct ifnet *ifp) if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING) return; - if (!adapter->link_active) + if (!adapter->link_active) { + ifq_purge(&ifp->if_snd); return; + } while (!ifq_is_empty(&ifp->if_snd)) { - m_head = ifq_poll(&ifp->if_snd); - + m_head = ifq_dequeue(&ifp->if_snd, NULL); if (m_head == NULL) break; logif(pkt_txqueue); if (em_encap(adapter, m_head)) { ifp->if_flags |= IFF_OACTIVE; + ifq_prepend(&ifp->if_snd, m_head); break; } - ifq_dequeue(&ifp->if_snd, m_head); /* Send a copy of the frame to the BPF listener */ ETHER_BPF_MTAP(ifp, m_head); @@ -1179,7 +1183,7 @@ em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) em_txeof(adapter); if (!ifq_is_empty(&ifp->if_snd)) - em_start(ifp); + if_devstart(ifp); } break; } @@ -1246,7 +1250,8 @@ em_intr(void *arg) adapter->rx_overruns++; if ((ifp->if_flags & IFF_RUNNING) && !ifq_is_empty(&ifp->if_snd)) - em_start(ifp); + if_devstart(ifp); + logif(intr_end); } diff --git a/sys/dev/netif/ep/if_ep.c b/sys/dev/netif/ep/if_ep.c index 3381629576..cfd5c31716 100644 --- a/sys/dev/netif/ep/if_ep.c +++ b/sys/dev/netif/ep/if_ep.c @@ -39,7 +39,7 @@ /* * $FreeBSD: src/sys/dev/ep/if_ep.c,v 1.95.2.3 2002/03/06 07:26:35 imp Exp $ - * $DragonFly: src/sys/dev/netif/ep/if_ep.c,v 1.26 2006/12/22 23:26:19 swildner Exp $ + * $DragonFly: src/sys/dev/netif/ep/if_ep.c,v 1.27 2008/05/14 11:59:19 sephe Exp $ * * Promiscuous mode added and interrupt logic slightly changed * to reduce the number of adapter failures. Transceiver select @@ -405,7 +405,8 @@ ep_if_init(void *xsc) */ GO_WINDOW(1); - ep_if_start(ifp); + + if_devstart(ifp); crit_exit(); } @@ -422,6 +423,7 @@ ep_if_start(struct ifnet *ifp) int pad; if (sc->gone) { + ifq_purge(&ifp->if_snd); return; } @@ -434,7 +436,7 @@ ep_if_start(struct ifnet *ifp) startagain: /* Sneak a peek at the next packet */ - m = ifq_poll(&ifp->if_snd); + m = ifq_dequeue(&ifp->if_snd, NULL); if (m == NULL) { crit_exit(); return; @@ -454,7 +456,6 @@ startagain: if (len + pad > ETHER_MAX_LEN) { /* packet is obviously too large: toss it */ ++ifp->if_oerrors; - ifq_dequeue(&ifp->if_snd, m); m_freem(m); goto readcheck; } @@ -464,6 +465,7 @@ startagain: /* make sure */ if (inw(BASE + EP_W1_FREE_TX) < len + pad + 4) { ifp->if_flags |= IFF_OACTIVE; + ifq_prepend(&ifp->if_snd, top); crit_exit(); return; } @@ -471,8 +473,6 @@ startagain: outw(BASE + EP_COMMAND, SET_TX_AVAIL_THRESH | EP_THRESH_DISABLE); } - ifq_dequeue(&ifp->if_snd, m); - outw(BASE + EP_W1_TX_PIO_WR_1, len); outw(BASE + EP_W1_TX_PIO_WR_1, 0x0); /* Second dword meaningless */ @@ -553,7 +553,7 @@ rescan: ifp->if_flags &= ~IFF_OACTIVE; GO_WINDOW(1); inw(BASE + EP_W1_FREE_TX); - ep_if_start(ifp); + if_devstart(ifp); } if (status & S_CARD_FAILURE) { ifp->if_timer = 0; @@ -615,7 +615,7 @@ rescan: ifp->if_flags &= ~IFF_OACTIVE; GO_WINDOW(1); inw(BASE + EP_W1_FREE_TX); - ep_if_start(ifp); + if_devstart(ifp); } /* end TX_COMPLETE */ } @@ -881,7 +881,7 @@ ep_if_watchdog(struct ifnet *ifp) } ifp->if_flags &= ~IFF_OACTIVE; - ep_if_start(ifp); + if_devstart(ifp); ep_intr(ifp->if_softc); } diff --git a/sys/dev/netif/ep/if_ep_eisa.c b/sys/dev/netif/ep/if_ep_eisa.c index 0ff1fd2a47..235b2c4432 100644 --- a/sys/dev/netif/ep/if_ep_eisa.c +++ b/sys/dev/netif/ep/if_ep_eisa.c @@ -20,7 +20,7 @@ * are met. * * $FreeBSD: src/sys/dev/ep/if_ep_eisa.c,v 1.18 2000/01/14 07:14:00 peter Exp $ - * $DragonFly: src/sys/dev/netif/ep/if_ep_eisa.c,v 1.12 2006/10/25 20:55:56 dillon Exp $ + * $DragonFly: src/sys/dev/netif/ep/if_ep_eisa.c,v 1.13 2008/05/14 11:59:19 sephe Exp $ */ #include @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -178,6 +179,7 @@ static int ep_eisa_attach(device_t dev) { struct ep_softc * sc = device_get_softc(dev); + struct ifnet * ifp = &sc->arpcom.ac_if; struct resource * eisa_io = NULL; u_int32_t eisa_iobase; int irq; @@ -229,12 +231,15 @@ ep_eisa_attach(device_t dev) error = bus_setup_intr(dev, sc->irq, INTR_NETSAFE, ep_intr, sc, &sc->ep_intrhand, - sc->arpcom.ac_if.if_serializer); + ifp->if_serializer); if (error) { device_printf(dev, "bus_setup_intr() failed! (%d)\n", error); goto bad; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return (0); bad: diff --git a/sys/dev/netif/ep/if_ep_isa.c b/sys/dev/netif/ep/if_ep_isa.c index 7d927d83e1..b560573f7e 100644 --- a/sys/dev/netif/ep/if_ep_isa.c +++ b/sys/dev/netif/ep/if_ep_isa.c @@ -28,7 +28,7 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/ep/if_ep_isa.c,v 1.8.2.1 2000/12/16 03:47:57 nyan Exp $ - * $DragonFly: src/sys/dev/netif/ep/if_ep_isa.c,v 1.13 2006/10/25 20:55:56 dillon Exp $ + * $DragonFly: src/sys/dev/netif/ep/if_ep_isa.c,v 1.14 2008/05/14 11:59:19 sephe Exp $ */ #include @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -302,6 +303,7 @@ static int ep_isa_attach(device_t dev) { struct ep_softc * sc = device_get_softc(dev); + struct ifnet * ifp = &sc->arpcom.ac_if; int error = 0; if ((error = ep_alloc(dev))) { @@ -320,13 +322,15 @@ ep_isa_attach(device_t dev) } error = bus_setup_intr(dev, sc->irq, INTR_NETSAFE, ep_intr, - sc, &sc->ep_intrhand, - sc->arpcom.ac_if.if_serializer); + sc, &sc->ep_intrhand, ifp->if_serializer); if (error) { device_printf(dev, "bus_setup_intr() failed! (%d)\n", error); goto bad; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return (0); bad: ep_free(dev); diff --git a/sys/dev/netif/ep/if_ep_pccard.c b/sys/dev/netif/ep/if_ep_pccard.c index 133fbdd65d..e2822d0ab9 100644 --- a/sys/dev/netif/ep/if_ep_pccard.c +++ b/sys/dev/netif/ep/if_ep_pccard.c @@ -28,7 +28,7 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/ep/if_ep_pccard.c,v 1.12.2.2 2000/08/08 23:55:02 peter Exp $ - * $DragonFly: src/sys/dev/netif/ep/if_ep_pccard.c,v 1.11 2006/10/25 20:55:56 dillon Exp $ + * $DragonFly: src/sys/dev/netif/ep/if_ep_pccard.c,v 1.12 2008/05/14 11:59:19 sephe Exp $ */ /* @@ -42,6 +42,7 @@ #include #include #include +#include #include #include @@ -156,6 +157,7 @@ static int ep_pccard_attach(device_t dev) { struct ep_softc * sc = device_get_softc(dev); + struct ifnet * ifp = &sc->arpcom.ac_if; int error = 0; if ((error = ep_alloc(dev))) { @@ -216,6 +218,9 @@ ep_pccard_attach(device_t dev) goto bad; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return (0); bad: ep_free(dev); diff --git a/sys/dev/netif/et/if_et.c b/sys/dev/netif/et/if_et.c index 1d77433de5..a0321a09f0 100644 --- a/sys/dev/netif/et/if_et.c +++ b/sys/dev/netif/et/if_et.c @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sys/dev/netif/et/if_et.c,v 1.8 2007/12/21 19:13:35 swildner Exp $ + * $DragonFly: src/sys/dev/netif/et/if_et.c,v 1.9 2008/05/14 11:59:19 sephe Exp $ */ #include @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -378,6 +379,10 @@ et_attach(device_t dev) device_printf(dev, "can't setup intr\n"); goto fail; } + + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->sc_irq_res)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return 0; fail: et_detach(dev); @@ -1228,8 +1233,10 @@ et_start(struct ifnet *ifp) ASSERT_SERIALIZED(ifp->if_serializer); - if ((sc->sc_flags & ET_FLAG_TXRX_ENABLED) == 0) + if ((sc->sc_flags & ET_FLAG_TXRX_ENABLED) == 0) { + ifq_purge(&ifp->if_snd); return; + } if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING) return; @@ -1269,7 +1276,7 @@ et_watchdog(struct ifnet *ifp) if_printf(ifp, "watchdog timed out\n"); ifp->if_init(ifp->if_softc); - ifp->if_start(ifp); + if_devstart(ifp); } static int @@ -2141,7 +2148,7 @@ et_txeof(struct et_softc *sc) if (tbd->tbd_used + ET_NSEG_SPARE <= ET_TX_NDESC) ifp->if_flags &= ~IFF_OACTIVE; - ifp->if_start(ifp); + if_devstart(ifp); } static void @@ -2159,7 +2166,7 @@ et_tick(void *xsc) IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) { if_printf(ifp, "Link up, enable TX/RX\n"); if (et_enable_txrx(sc, 0) == 0) - ifp->if_start(ifp); + if_devstart(ifp); } callout_reset(&sc->sc_tick, hz, et_tick, sc); diff --git a/sys/dev/netif/ex/if_ex.c b/sys/dev/netif/ex/if_ex.c index fb4f2d9a87..cc66814127 100644 --- a/sys/dev/netif/ex/if_ex.c +++ b/sys/dev/netif/ex/if_ex.c @@ -25,7 +25,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/dev/ex/if_ex.c,v 1.26.2.3 2001/03/05 05:33:20 imp Exp $ - * $DragonFly: src/sys/dev/netif/ex/if_ex.c,v 1.24 2006/12/22 23:26:19 swildner Exp $ + * $DragonFly: src/sys/dev/netif/ex/if_ex.c,v 1.25 2008/05/14 11:59:19 sephe Exp $ * * MAINTAINER: Matthew N. Dodd * @@ -352,7 +352,8 @@ ex_init(void *xsc) DELAY(2); outb(iobase + CMD_REG, Rcv_Enable_CMD); - ex_start(ifp); + if (!ifq_is_empty(&ifp->if_snd)) + if_devstart(ifp); DODEBUG(Start_End, kprintf("ex_init%d: finish\n", ifp->if_dunit);); } @@ -375,7 +376,7 @@ ex_start(struct ifnet *ifp) * more packets left, or the card cannot accept any more yet. */ while ((ifp->if_flags & IFF_OACTIVE) == 0) { - opkt = ifq_poll(&ifp->if_snd); + opkt = ifq_dequeue(&ifp->if_snd, NULL); if (opkt == NULL) break; @@ -410,8 +411,6 @@ ex_start(struct ifnet *ifp) DODEBUG(Sent_Pkts, kprintf("i=%d, avail=%d\n", i, avail);); if (avail >= len + XMT_HEADER_LEN) { - ifq_dequeue(&ifp->if_snd, opkt); - #ifdef EX_PSA_INTR /* * Disable rx and tx interrupts, to avoid corruption @@ -525,6 +524,7 @@ ex_start(struct ifnet *ifp) m_freem(opkt); } else { ifp->if_flags |= IFF_OACTIVE; + ifq_prepend(&ifp->if_snd, opkt); DODEBUG(Status, kprintf("OACTIVE start\n");); } } @@ -596,7 +596,7 @@ ex_intr(void *arg) */ if (send_pkts && !ifq_is_empty(&ifp->if_snd)) - ex_start(ifp); + if_devstart(ifp); #ifdef EXDEBUG exintr_count--; @@ -827,7 +827,7 @@ ex_watchdog(struct ifnet *ifp) ifp->if_oerrors++; ex_reset(sc); - ex_start(ifp); + if_devstart(ifp); DODEBUG(Start_End, kprintf("ex_watchdog%d: finish\n", ifp->if_dunit);); diff --git a/sys/dev/netif/ex/if_ex_isa.c b/sys/dev/netif/ex/if_ex_isa.c index 2301c73193..c075403dd8 100644 --- a/sys/dev/netif/ex/if_ex_isa.c +++ b/sys/dev/netif/ex/if_ex_isa.c @@ -24,12 +24,13 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/dev/ex/if_ex_isa.c,v 1.3.2.1 2001/03/05 05:33:20 imp Exp $ - * $DragonFly: src/sys/dev/netif/ex/if_ex_isa.c,v 1.14 2007/04/30 07:18:50 dillon Exp $ + * $DragonFly: src/sys/dev/netif/ex/if_ex_isa.c,v 1.15 2008/05/14 11:59:19 sephe Exp $ */ #include #include #include +#include #include #include #include @@ -239,6 +240,7 @@ static int ex_isa_attach(device_t dev) { struct ex_softc * sc = device_get_softc(dev); + struct ifnet * ifp = &sc->arpcom.ac_if; int error = 0; u_int16_t temp; @@ -280,12 +282,15 @@ ex_isa_attach(device_t dev) error = bus_setup_intr(dev, sc->irq, INTR_NETSAFE, ex_intr, (void *)sc, &sc->ih, - sc->arpcom.ac_if.if_serializer); + ifp->if_serializer); if (error) { device_printf(dev, "bus_setup_intr() failed!\n"); goto bad; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return(0); bad: ex_release_resources(dev); diff --git a/sys/dev/netif/ex/if_ex_pccard.c b/sys/dev/netif/ex/if_ex_pccard.c index 6865afcf80..39723ab055 100644 --- a/sys/dev/netif/ex/if_ex_pccard.c +++ b/sys/dev/netif/ex/if_ex_pccard.c @@ -24,12 +24,13 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/dev/ex/if_ex_pccard.c,v 1.2.2.1 2001/03/05 05:33:20 imp Exp $ - * $DragonFly: src/sys/dev/netif/ex/if_ex_pccard.c,v 1.14 2007/08/14 15:32:32 sephe Exp $ + * $DragonFly: src/sys/dev/netif/ex/if_ex_pccard.c,v 1.15 2008/05/14 11:59:19 sephe Exp $ */ #include #include #include +#include #include #include #include @@ -125,6 +126,7 @@ static int ex_pccard_attach(device_t dev) { struct ex_softc * sc = device_get_softc(dev); + struct ifnet * ifp = &sc->arpcom.ac_if; int error = 0; int i; uint8_t sum; @@ -161,12 +163,15 @@ ex_pccard_attach(device_t dev) error = bus_setup_intr(dev, sc->irq, INTR_NETSAFE, ex_intr, (void *)sc, &sc->ih, - sc->arpcom.ac_if.if_serializer); + ifp->if_serializer); if (error) { device_printf(dev, "bus_setup_intr() failed!\n"); goto bad; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return(0); bad: ex_release_resources(dev); diff --git a/sys/dev/netif/fe/if_fe.c b/sys/dev/netif/fe/if_fe.c index c1e8e3c7b6..9578fc9c90 100644 --- a/sys/dev/netif/fe/if_fe.c +++ b/sys/dev/netif/fe/if_fe.c @@ -22,7 +22,7 @@ /* * $FreeBSD: src/sys/dev/fe/if_fe.c,v 1.65.2.1 2000/09/22 10:01:47 nyan Exp $ - * $DragonFly: src/sys/dev/netif/fe/if_fe.c,v 1.28 2006/12/22 23:26:20 swildner Exp $ + * $DragonFly: src/sys/dev/netif/fe/if_fe.c,v 1.29 2008/05/14 11:59:20 sephe Exp $ * * Device driver for Fujitsu MB86960A/MB86965A based Ethernet cards. * Contributed by M. Sekiguchi. @@ -821,6 +821,8 @@ fe_attach (device_t dev) return ENXIO; } + sc->sc_if.if_cpuid = ithread_cpuid(rman_get_start(sc->irq_res)); + KKASSERT(sc->sc_if.if_cpuid >= 0 && sc->sc_if.if_cpuid < ncpus); /* Print additional info when attached. */ device_printf(dev, "type %s%s\n", sc->typestr, @@ -1108,7 +1110,7 @@ fe_init (void * xsc) the interface keeping it idle. The upper layer will soon start the interface anyway, and there are no significant delay. */ - fe_start(&sc->sc_if); + if_devstart(&sc->sc_if); #endif } @@ -1727,7 +1729,7 @@ fe_intr (void *arg) * interrupt when the transmission buffer is full. */ if ((sc->sc_if.if_flags & IFF_OACTIVE) == 0) - fe_start(&sc->sc_if); + if_devstart(&sc->sc_if); } kprintf("fe%d: too many loops\n", sc->sc_unit); diff --git a/sys/dev/netif/fwe/if_fwe.c b/sys/dev/netif/fwe/if_fwe.c index a761869d5c..7da8fa31d3 100644 --- a/sys/dev/netif/fwe/if_fwe.c +++ b/sys/dev/netif/fwe/if_fwe.c @@ -32,7 +32,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/dev/firewire/if_fwe.c,v 1.27 2004/01/08 14:58:09 simokawa Exp $ - * $DragonFly: src/sys/dev/netif/fwe/if_fwe.c,v 1.30 2008/01/06 16:55:50 swildner Exp $ + * $DragonFly: src/sys/dev/netif/fwe/if_fwe.c,v 1.31 2008/05/14 11:59:20 sephe Exp $ */ #include "opt_inet.h" @@ -413,7 +413,7 @@ fwe_output_callback(struct fw_xfer *xfer) /* for queue full */ if (!ifq_is_empty(&ifp->if_snd)) - fwe_start(ifp); + if_devstart(ifp); lwkt_serialize_exit(ifp->if_serializer); } diff --git a/sys/dev/netif/fxp/if_fxp.c b/sys/dev/netif/fxp/if_fxp.c index f9b9d87168..5a5c90cc1f 100644 --- a/sys/dev/netif/fxp/if_fxp.c +++ b/sys/dev/netif/fxp/if_fxp.c @@ -26,7 +26,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/dev/fxp/if_fxp.c,v 1.110.2.30 2003/06/12 16:47:05 mux Exp $ - * $DragonFly: src/sys/dev/netif/fxp/if_fxp.c,v 1.50 2007/09/15 21:28:15 swildner Exp $ + * $DragonFly: src/sys/dev/netif/fxp/if_fxp.c,v 1.51 2008/05/14 11:59:20 sephe Exp $ */ /* @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -689,6 +690,9 @@ fxp_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return (0); failmem: @@ -1049,9 +1053,13 @@ fxp_start(struct ifnet *ifp) * of the command chain). */ if (sc->need_mcsetup) { + ifq_purge(&ifp->if_snd); return; } + if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING) + return; + txp = NULL; /* @@ -1068,7 +1076,7 @@ fxp_start(struct ifnet *ifp) * Grab a packet to transmit. The packet is dequeued, * once we are sure that we have enough free descriptors. */ - mb_head = ifq_poll(&ifp->if_snd); + mb_head = ifq_dequeue(&ifp->if_snd, NULL); if (mb_head == NULL) break; @@ -1101,20 +1109,22 @@ tbdinit: * mbuf chain first. Bail out if we can't get the * new buffers. */ - if (ntries > 0) + if (ntries > 0) { + ifq_prepend(&ifp->if_snd, mb_head); + ifp->if_flags |= IFF_OACTIVE; break; + } + mn = m_dup(mb_head, MB_DONTWAIT); - if (mn == NULL) + if (mn == NULL) { + ifq_prepend(&ifp->if_snd, mb_head); break; - /* We can transmit the packet, dequeue it. */ - ifq_dequeue(&ifp->if_snd, mb_head); + } + m_freem(mb_head); mb_head = mn; ntries = 1; goto tbdinit; - } else { - /* Nothing to worry about, just dequeue. */ - ifq_dequeue(&ifp->if_snd, mb_head); } txp->tbd_number = segment; @@ -1156,6 +1166,9 @@ tbdinit: BPF_MTAP(ifp, mb_head); } + if (sc->tx_queued >= FXP_NTXCB - 1) + ifp->if_flags |= IFF_OACTIVE; + /* * We're finished. If we added to the list, issue a RESUME to get DMA * going again if suspended. @@ -1269,7 +1282,9 @@ fxp_intr_body(struct fxp_softc *sc, u_int8_t statack, int count) */ if (statack & (FXP_SCB_STATACK_CXTNO | FXP_SCB_STATACK_CNA)) { struct fxp_cb_tx *txp; + int old_queued; + old_queued = sc->tx_queued; for (txp = sc->cbl_first; sc->tx_queued && (txp->cb_status & FXP_CB_STATUS_C) != 0; txp = txp->next) { @@ -1282,16 +1297,21 @@ fxp_intr_body(struct fxp_softc *sc, u_int8_t statack, int count) } } sc->cbl_first = txp; - ifp->if_timer = 0; + + ifp->if_timer = 0; /* XXX only if tx_queued is 0 */ + if (old_queued > sc->tx_queued) + ifp->if_flags &= ~IFF_OACTIVE; + if (sc->tx_queued == 0) { if (sc->need_mcsetup) fxp_mc_setup(sc); } + /* * Try to start more packets transmitting. */ if (!ifq_is_empty(&ifp->if_snd)) - (*ifp->if_start)(ifp); + if_devstart(ifp); } /* diff --git a/sys/dev/netif/ie/if_ie.c b/sys/dev/netif/ie/if_ie.c index 1ceba78871..e027d9172e 100644 --- a/sys/dev/netif/ie/if_ie.c +++ b/sys/dev/netif/ie/if_ie.c @@ -48,7 +48,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/dev/ie/if_ie.c,v 1.72.2.4 2003/03/27 21:01:49 mdodd Exp $ - * $DragonFly: src/sys/dev/netif/ie/if_ie.c,v 1.32 2007/07/11 23:46:58 dillon Exp $ + * $DragonFly: src/sys/dev/netif/ie/if_ie.c,v 1.33 2008/05/14 11:59:20 sephe Exp $ */ /* @@ -1010,7 +1010,7 @@ ietint(int unit, struct ie_softc *ie) /* Wish I knew why this seems to be necessary... */ ie->xmit_cmds[0]->ie_xmit_status |= IE_STAT_COMPL; - iestart(&ie->arpcom.ac_if); + if_devstart(&ie->arpcom.ac_if); return (0); /* shouldn't be necessary */ } diff --git a/sys/dev/netif/iwi/if_iwi.c b/sys/dev/netif/iwi/if_iwi.c index 9c3446024a..44c1ae0144 100644 --- a/sys/dev/netif/iwi/if_iwi.c +++ b/sys/dev/netif/iwi/if_iwi.c @@ -29,7 +29,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/dev/iwi/if_iwi.c,v 1.8.2.6 2006/02/23 02:06:46 sam Exp $ - * $DragonFly: src/sys/dev/netif/iwi/if_iwi.c,v 1.20 2008/05/10 07:11:28 sephe Exp $ + * $DragonFly: src/sys/dev/netif/iwi/if_iwi.c,v 1.21 2008/05/14 11:59:20 sephe Exp $ */ /*- @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -538,6 +539,9 @@ iwi_attach(device_t dev) goto fail1; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + if (bootverbose) ieee80211_announce(ic); @@ -958,7 +962,7 @@ iwi_resume(device_t dev) if (ifp->if_flags & IFF_UP) { ifp->if_init(ifp->if_softc); if (ifp->if_flags & IFF_RUNNING) - ifp->if_start(ifp); + if_devstart(ifp); } lwkt_serialize_exit(ifp->if_serializer); @@ -1538,7 +1542,7 @@ iwi_tx_intr(struct iwi_softc *sc, struct iwi_tx_ring *txq) sc->sc_tx_timer = 0; ifp->if_flags &= ~IFF_OACTIVE; - iwi_start(ifp); + if_devstart(ifp); } static void @@ -1839,12 +1843,11 @@ iwi_start(struct ifnet *ifp) struct ieee80211_node *ni; int ac; - if (ic->ic_state != IEEE80211_S_RUN) + ieee80211_drain_mgtq(&ic->ic_mgtq); + if (ic->ic_state != IEEE80211_S_RUN) { + ifq_purge(&ifp->if_snd); return; - - IF_POLL(&ic->ic_mgtq, m0); - if (m0 != NULL) - ieee80211_drain_mgtq(&ic->ic_mgtq); + } for (;;) { m0 = ifq_dequeue(&ifp->if_snd, NULL); diff --git a/sys/dev/netif/iwl/iwl2100.c b/sys/dev/netif/iwl/iwl2100.c index 3c333c9e55..c1af330b4b 100644 --- a/sys/dev/netif/iwl/iwl2100.c +++ b/sys/dev/netif/iwl/iwl2100.c @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sys/dev/netif/iwl/iwl2100.c,v 1.3 2008/03/08 06:43:52 sephe Exp $ + * $DragonFly: src/sys/dev/netif/iwl/iwl2100.c,v 1.4 2008/05/14 11:59:20 sephe Exp $ */ #include @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -313,6 +314,9 @@ iwl2100_attach(device_t dev) return ENXIO; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->sc_irq_res)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + /* * Attach radio tap */ @@ -846,14 +850,19 @@ iwl2100_start(struct ifnet *ifp) ASSERT_SERIALIZED(ifp->if_serializer); - if (sc->sc_flags & IWL2100_F_DETACH) + if (sc->sc_flags & IWL2100_F_DETACH) { + ieee80211_drain_mgtq(&ic->ic_mgtq); + ifq_purge(&ifp->if_snd); return; + } if ((ifp->if_flags & (IFF_OACTIVE | IFF_RUNNING)) != IFF_RUNNING) return; - if ((sc->sc_flags & IWL2100_F_IFSTART) == 0) + if ((sc->sc_flags & IWL2100_F_IFSTART) == 0) { + ifq_purge(&ifp->if_snd); goto back; + } while (tr->tr_used < IWL2100_TX_USED_MAX) { struct ieee80211_frame *wh; @@ -987,7 +996,9 @@ iwl2100_newstate_dispatch(struct netmsg *nmsg) struct iwlmsg *msg = (struct iwlmsg *)nmsg; struct iwl2100_softc *sc = msg->iwlm_softc; struct ieee80211com *ic = &sc->sc_ic; +#ifdef INVARIANTS struct ifnet *ifp = &ic->ic_if; +#endif enum ieee80211_state nstate, ostate; int arg = msg->iwlm_arg, error = 0; @@ -1738,7 +1749,9 @@ iwl2100_rxdesc_setup(struct iwl2100_softc *sc, int buf_idx) static int iwl2100_init_firmware(struct iwl2100_softc *sc) { +#ifdef INVARIANTS struct ifnet *ifp = &sc->sc_ic.ic_if; +#endif uint32_t intr; int i; @@ -2496,7 +2509,7 @@ next: } ifp->if_flags &= ~IFF_OACTIVE; - ifp->if_start(ifp); + if_devstart(ifp); } } @@ -2927,7 +2940,7 @@ iwl2100_rxeof_status(struct iwl2100_softc *sc, int i) DPRINTF(sc, IWL2100_DBG_RESTART, "%s", "restart done\n"); sc->sc_flags |= IWL2100_F_IFSTART; - ifp->if_start(ifp); + if_devstart(ifp); } else { KKASSERT(ic->ic_opmode == IEEE80211_M_IBSS); callout_reset(&sc->sc_ibss, (100 * hz) / 1000, @@ -3577,7 +3590,7 @@ iwl2100_ibss_bssid(void *xsc) IEEE80211_ADDR_COPY(ic->ic_bss->ni_bssid, bssid); sc->sc_flags |= IWL2100_F_IFSTART; - ifp->if_start(ifp); + if_devstart(ifp); } } back: diff --git a/sys/dev/netif/kue/if_kue.c b/sys/dev/netif/kue/if_kue.c index 671664884b..bd665d5398 100644 --- a/sys/dev/netif/kue/if_kue.c +++ b/sys/dev/netif/kue/if_kue.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/usb/if_kue.c,v 1.17.2.9 2003/04/13 02:39:25 murray Exp $ - * $DragonFly: src/sys/dev/netif/kue/if_kue.c,v 1.29 2008/01/06 16:55:50 swildner Exp $ + * $DragonFly: src/sys/dev/netif/kue/if_kue.c,v 1.30 2008/05/14 11:59:20 sephe Exp $ */ /* @@ -746,7 +746,7 @@ kue_txeof(usbd_xfer_handle xfer, usbd_private_handle priv, usbd_status status) ifp->if_opackets++; if (!ifq_is_empty(&ifp->if_snd)) - (*ifp->if_start)(ifp); + if_devstart(ifp); KUE_UNLOCK(sc); @@ -805,18 +805,18 @@ kue_start(struct ifnet *ifp) return; } - m_head = ifq_poll(&ifp->if_snd); + m_head = ifq_dequeue(&ifp->if_snd, NULL); if (m_head == NULL) { KUE_UNLOCK(sc); return; } if (kue_encap(sc, m_head, 0)) { + /* kue_encap() will free m_head, if we reach here */ ifp->if_flags |= IFF_OACTIVE; KUE_UNLOCK(sc); return; } - ifq_dequeue(&ifp->if_snd, m_head); /* * If there's a BPF listener, bounce a copy of this frame @@ -989,8 +989,8 @@ kue_watchdog(struct ifnet *ifp) usbd_get_xfer_status(c->kue_xfer, NULL, NULL, NULL, &stat); kue_txeof(c->kue_xfer, c, stat); - if (ifq_is_empty(&ifp->if_snd)) - kue_start(ifp); + if (!ifq_is_empty(&ifp->if_snd)) + if_devstart(ifp); KUE_UNLOCK(sc); return; diff --git a/sys/dev/netif/le/if_le.c b/sys/dev/netif/le/if_le.c index 74b3785115..ae3cb79c0c 100644 --- a/sys/dev/netif/le/if_le.c +++ b/sys/dev/netif/le/if_le.c @@ -22,7 +22,7 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/i386/isa/if_le.c,v 1.56.2.4 2002/06/05 23:24:10 paul Exp $ - * $DragonFly: src/sys/dev/netif/le/if_le.c,v 1.37 2006/12/22 23:26:20 swildner Exp $ + * $DragonFly: src/sys/dev/netif/le/if_le.c,v 1.38 2008/05/14 11:59:20 sephe Exp $ */ /* @@ -861,6 +861,8 @@ lemac_start(struct ifnet *ifp) } m = ifq_dequeue(&ifp->if_snd, NULL); + if (m == NULL) + break; LE_OUTB(sc, LEMAC_REG_MPN, tx_pg); /* Shift 2K window. */ /* @@ -902,7 +904,7 @@ lemac_tne_intr(struct le_softc *sc) sc->le_if.if_collisions++; } sc->le_if.if_flags &= ~IFF_OACTIVE; - lemac_start(&sc->le_if); + if_devstart(&sc->le_if); } static void @@ -1492,7 +1494,7 @@ lance_init(void *xsc) sc->le_if.if_flags |= IFF_RUNNING; LN_WRCSR(sc, LN_CSR0_START|LN_CSR0_INITDONE|LN_CSR0_ENABINTR); /* lance_dumpcsrs(sc, "lance_init: up"); */ - lance_start(&sc->le_if); + if_devstart(&sc->le_if); } else { /* lance_dumpcsrs(sc, "lance_init: down"); */ sc->le_if.if_flags &= ~IFF_RUNNING; @@ -1536,7 +1538,7 @@ lance_intr(struct le_softc *sc) if (oldcsr & LN_CSR0_TXINT) { if (lance_tx_intr(sc)) - lance_start(&sc->le_if); + if_devstart(&sc->le_if); } if (oldcsr == (LN_CSR0_PENDINTR|LN_CSR0_RXON|LN_CSR0_TXON)) @@ -1648,7 +1650,7 @@ lance_start(struct ifnet *ifp) return; for (;;) { - m = ifq_poll(&ifp->if_snd); + m = ifq_dequeue(&ifp->if_snd, NULL); if (m == NULL) break; @@ -1672,6 +1674,7 @@ lance_start(struct ifnet *ifp) /* * Try to see if we can free up anything off the transit ring. */ + ifq_prepend(&ifp->if_snd, m); if (lance_tx_intr(sc) > 0) { LN_STAT(tx_drains[0]++); continue; @@ -1690,6 +1693,7 @@ lance_start(struct ifnet *ifp) slop += ri->ri_heapend - ri->ri_outptr; if (len + slop > ri->ri_outsize) { LN_STAT(tx_nospc[1]++); + ifq_prepend(&ifp->if_snd, m); break; } /* @@ -1719,8 +1723,6 @@ lance_start(struct ifnet *ifp) if (m->m_pkthdr.len < len) LN_ZERO(sc, bp, len - m->m_pkthdr.len); - ifq_dequeue(&ifp->if_snd, m); - /* * Finally, copy out the descriptor and tell the * LANCE to transmit!. diff --git a/sys/dev/netif/lge/if_lge.c b/sys/dev/netif/lge/if_lge.c index e3e5922c2a..adc32680c1 100644 --- a/sys/dev/netif/lge/if_lge.c +++ b/sys/dev/netif/lge/if_lge.c @@ -31,7 +31,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/lge/if_lge.c,v 1.5.2.2 2001/12/14 19:49:23 jlemon Exp $ - * $DragonFly: src/sys/dev/netif/lge/if_lge.c,v 1.40 2008/01/05 14:02:37 swildner Exp $ + * $DragonFly: src/sys/dev/netif/lge/if_lge.c,v 1.41 2008/05/14 11:59:20 sephe Exp $ */ /* @@ -80,6 +80,7 @@ #include #include #include +#include #include #include #include @@ -557,6 +558,9 @@ lge_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->lge_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return(0); fail: @@ -1008,7 +1012,7 @@ lge_tick_serialized(void *xsc) kprintf("lge%d: gigabit link up\n", sc->lge_unit); if (!ifq_is_empty(&ifp->if_snd)) - (*ifp->if_start)(ifp); + if_devstart(ifp); } } @@ -1059,7 +1063,7 @@ lge_intr(void *arg) CSR_WRITE_4(sc, LGE_IMR, LGE_IMR_SETRST_CTL0|LGE_IMR_INTR_ENB); if (!ifq_is_empty(&ifp->if_snd)) - (*ifp->if_start)(ifp); + if_devstart(ifp); } /* @@ -1085,6 +1089,9 @@ lge_encap(struct lge_softc *sc, struct mbuf *m_head, uint32_t *txidx) for (m = m_head; m != NULL; m = m->m_next) { if (m->m_len != 0) { + if (frag == LGE_FRAG_CNT) + break; + tot_len += m->m_len; f = &cur_tx->lge_frags[frag]; f->lge_fraglen = m->m_len; @@ -1093,9 +1100,8 @@ lge_encap(struct lge_softc *sc, struct mbuf *m_head, uint32_t *txidx) frag++; } } - - if (m != NULL) - return(ENOBUFS); + /* Caller should make sure that 'm_head' is not excessive fragmented */ + KASSERT(m == NULL, ("too many fragments\n")); cur_tx->lge_mbuf = m_head; cur_tx->lge_ctl = LGE_TXCTL_WANTINTR|LGE_FRAGCNT(frag)|tot_len; @@ -1118,12 +1124,14 @@ static void lge_start(struct ifnet *ifp) { struct lge_softc *sc = ifp->if_softc; - struct mbuf *m_head = NULL; + struct mbuf *m_head = NULL, *m_defragged; uint32_t idx; int need_timer; - if (!sc->lge_link) + if (!sc->lge_link) { + ifq_purge(&ifp->if_snd); return; + } idx = sc->lge_cdata.lge_tx_prod; @@ -1132,18 +1140,46 @@ lge_start(struct ifnet *ifp) need_timer = 0; while(sc->lge_ldata->lge_tx_list[idx].lge_mbuf == NULL) { - if (CSR_READ_1(sc, LGE_TXCMDFREE_8BIT) == 0) + struct mbuf *m; + int frags; + + if (CSR_READ_1(sc, LGE_TXCMDFREE_8BIT) == 0) { + ifp->if_flags |= IFF_OACTIVE; break; + } - m_head = ifq_poll(&ifp->if_snd); + m_defragged = NULL; + m_head = ifq_dequeue(&ifp->if_snd, NULL); if (m_head == NULL) break; - if (lge_encap(sc, m_head, &idx)) { - ifp->if_flags |= IFF_OACTIVE; - break; +again: + frags = 0; + for (m = m_head; m != NULL; m = m->m_next) + ++frags; + if (frags > LGE_FRAG_CNT) { + if (m_defragged != NULL) { + /* + * Even after defragmentation, there + * are still too many fragments, so + * drop this packet. + */ + m_freem(m_head); + continue; + } + + m_defragged = m_defrag(m_head, MB_DONTWAIT); + if (m_defragged == NULL) { + m_freem(m_head); + continue; + } + m_head = m_defragged; + + /* Recount # of fragments */ + goto again; } - ifq_dequeue(&ifp->if_snd, m_head); + + lge_encap(sc, m_head, &idx); need_timer = 1; BPF_MTAP(ifp, m_head); @@ -1392,7 +1428,7 @@ lge_watchdog(struct ifnet *ifp) lge_init(sc); if (!ifq_is_empty(&ifp->if_snd)) - (*ifp->if_start)(ifp); + if_devstart(ifp); } /* diff --git a/sys/dev/netif/lge/if_lgereg.h b/sys/dev/netif/lge/if_lgereg.h index cf5ca40e97..f65cf19bb9 100644 --- a/sys/dev/netif/lge/if_lgereg.h +++ b/sys/dev/netif/lge/if_lgereg.h @@ -31,7 +31,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/lge/if_lgereg.h,v 1.2.2.1 2001/06/19 19:42:38 wpaul Exp $ - * $DragonFly: src/sys/dev/netif/lge/if_lgereg.h,v 1.10 2006/08/01 18:05:02 swildner Exp $ + * $DragonFly: src/sys/dev/netif/lge/if_lgereg.h,v 1.11 2008/05/14 11:59:20 sephe Exp $ */ @@ -400,10 +400,12 @@ struct lge_frag { uint16_t lge_rsvd1; }; +#define LGE_FRAG_CNT 10 + struct lge_tx_desc { /* Hardware descriptor section */ uint32_t lge_ctl; - struct lge_frag lge_frags[10]; + struct lge_frag lge_frags[LGE_FRAG_CNT]; uint32_t lge_rsvd0; union { struct mbuf *lge_mbuf; diff --git a/sys/dev/netif/lnc/am7990.c b/sys/dev/netif/lnc/am7990.c index ffdc4a89fc..2d7fd3f235 100644 --- a/sys/dev/netif/lnc/am7990.c +++ b/sys/dev/netif/lnc/am7990.c @@ -1,6 +1,6 @@ /* $NetBSD: am7990.c,v 1.68 2005/12/11 12:21:25 christos Exp $ */ /* $FreeBSD: src/sys/dev/le/am7990.c,v 1.3 2006/05/16 21:04:01 marius Exp $ */ -/* $DragonFly: src/sys/dev/netif/lnc/am7990.c,v 1.4 2007/03/24 08:29:57 sephe Exp $ */ +/* $DragonFly: src/sys/dev/netif/lnc/am7990.c,v 1.5 2008/05/14 11:59:20 sephe Exp $ */ /*- * Copyright (c) 1997, 1998 The NetBSD Foundation, Inc. @@ -479,8 +479,7 @@ am7990_intr(void *arg) (*sc->sc_wrcsr)(sc, LE_CSR0, LE_C0_INEA); if (!ifq_is_empty(&ifp->if_snd)) - am7990_start_locked(sc); - + if_devstart(ifp); } /* @@ -505,7 +504,6 @@ am7990_start_locked(struct lance_softc *sc) for (; sc->sc_no_td < sc->sc_ntbuf && !ifq_is_empty(&ifp->if_snd);) { - m = ifq_poll(&ifp->if_snd); rp = LE_TMDADDR(sc, bix); (*sc->sc_copyfromdesc)(sc, &tmd, rp, sizeof(tmd)); @@ -516,8 +514,8 @@ am7990_start_locked(struct lance_softc *sc) sc->sc_no_td, sc->sc_last_td); } - ifq_dequeue(&ifp->if_snd, m); - if (m == 0) + m = ifq_dequeue(&ifp->if_snd, NULL); + if (m == NULL) break; /* diff --git a/sys/dev/netif/lnc/am79900.c b/sys/dev/netif/lnc/am79900.c index 085b0219ef..562270791a 100644 --- a/sys/dev/netif/lnc/am79900.c +++ b/sys/dev/netif/lnc/am79900.c @@ -1,6 +1,6 @@ /* $NetBSD: am79900.c,v 1.17 2005/12/24 20:27:29 perry Exp $ */ /* $FreeBSD: src/sys/dev/le/am79900.c,v 1.3 2006/05/16 21:04:01 marius Exp $ */ -/* $DragonFly: src/sys/dev/netif/lnc/am79900.c,v 1.4 2007/03/24 08:29:57 sephe Exp $ */ +/* $DragonFly: src/sys/dev/netif/lnc/am79900.c,v 1.5 2008/05/14 11:59:20 sephe Exp $ */ /*- @@ -517,8 +517,7 @@ am79900_intr(void *arg) (*sc->sc_wrcsr)(sc, LE_CSR0, LE_C0_INEA); if (!ifq_is_empty(&ifp->if_snd)) - am79900_start_locked(sc); - + if_devstart(ifp); } /* @@ -543,7 +542,6 @@ am79900_start_locked(struct lance_softc *sc) for (; sc->sc_no_td < sc->sc_ntbuf && !ifq_is_empty(&ifp->if_snd);) { - m = ifq_poll(&ifp->if_snd); rp = LE_TMDADDR(sc, bix); (*sc->sc_copyfromdesc)(sc, &tmd, rp, sizeof(tmd)); @@ -554,8 +552,8 @@ am79900_start_locked(struct lance_softc *sc) sc->sc_no_td, sc->sc_last_td); } - ifq_dequeue(&ifp->if_snd, m); - if (m == 0) + m = ifq_dequeue(&ifp->if_snd, NULL); + if (m == NULL) break; /* diff --git a/sys/dev/netif/lnc/if_lnc_isa.c b/sys/dev/netif/lnc/if_lnc_isa.c index 5fdc07f628..8ec9a6d49d 100644 --- a/sys/dev/netif/lnc/if_lnc_isa.c +++ b/sys/dev/netif/lnc/if_lnc_isa.c @@ -1,6 +1,6 @@ /* $NetBSD: if_le_isa.c,v 1.41 2005/12/24 20:27:41 perry Exp $ */ /* $FreeBSD: src/sys/dev/le/if_le_isa.c,v 1.1 2006/05/17 21:25:22 marius Exp $ */ -/* $DragonFly: src/sys/dev/netif/lnc/if_lnc_isa.c,v 1.10 2006/10/25 20:55:57 dillon Exp $ */ +/* $DragonFly: src/sys/dev/netif/lnc/if_lnc_isa.c,v 1.11 2008/05/14 11:59:20 sephe Exp $ */ /*- * Copyright (c) 1997, 1998 The NetBSD Foundation, Inc. @@ -78,6 +78,7 @@ #include #include #include +#include #include #include #include @@ -430,6 +431,9 @@ le_isa_attach(device_t dev) goto fail_am7990; } + sc->ifp->if_cpuid = ithread_cpuid(rman_get_start(lesc->sc_ires)); + KKASSERT(sc->ifp->if_cpuid >= 0 && sc->ifp->if_cpuid < ncpus); + return (0); fail_am7990: diff --git a/sys/dev/netif/lnc/if_lnc_pci.c b/sys/dev/netif/lnc/if_lnc_pci.c index 32857c401d..a7f1990129 100644 --- a/sys/dev/netif/lnc/if_lnc_pci.c +++ b/sys/dev/netif/lnc/if_lnc_pci.c @@ -1,6 +1,6 @@ /* $NetBSD: if_le_pci.c,v 1.43 2005/12/11 12:22:49 christos Exp $ */ /* $FreeBSD: src/sys/dev/le/if_le_pci.c,v 1.4 2006/06/05 15:14:14 marius Exp $ */ -/* $DragonFly: src/sys/dev/netif/lnc/if_lnc_pci.c,v 1.11 2006/10/25 20:55:57 dillon Exp $ */ +/* $DragonFly: src/sys/dev/netif/lnc/if_lnc_pci.c,v 1.12 2008/05/14 11:59:20 sephe Exp $ */ /*- @@ -79,6 +79,7 @@ #include #include #include +#include #include #include #include @@ -443,6 +444,9 @@ le_pci_attach(device_t dev) goto fail_am79900; } + sc->ifp->if_cpuid = ithread_cpuid(rman_get_start(lesc->sc_ires)); + KKASSERT(sc->ifp->if_cpuid >= 0 && sc->ifp->if_cpuid < ncpus); + return (0); fail_am79900: diff --git a/sys/dev/netif/lnc/lance.c b/sys/dev/netif/lnc/lance.c index a44d1c64e7..90350de5d4 100644 --- a/sys/dev/netif/lnc/lance.c +++ b/sys/dev/netif/lnc/lance.c @@ -1,6 +1,6 @@ /* $NetBSD: lance.c,v 1.34 2005/12/24 20:27:30 perry Exp $ */ /* $FreeBSD: src/sys/dev/le/lance.c,v 1.2 2006/05/16 21:04:01 marius Exp $ */ -/* $DragonFly: src/sys/dev/netif/lnc/lance.c,v 1.6 2006/12/29 15:26:14 corecode Exp $ */ +/* $DragonFly: src/sys/dev/netif/lnc/lance.c,v 1.7 2008/05/14 11:59:20 sephe Exp $ */ /*- @@ -299,7 +299,7 @@ lance_init_locked(struct lance_softc *sc) ifp->if_flags |= IFF_RUNNING; ifp->if_flags &= ~IFF_OACTIVE; ifp->if_timer = 0; - (*sc->sc_start_locked)(sc); + if_devstart(ifp); } else if_printf(ifp, "controller failed to initialize\n"); diff --git a/sys/dev/netif/msk/if_msk.c b/sys/dev/netif/msk/if_msk.c index de8bedad41..9a57fc2975 100644 --- a/sys/dev/netif/msk/if_msk.c +++ b/sys/dev/netif/msk/if_msk.c @@ -93,7 +93,7 @@ */ /* $FreeBSD: src/sys/dev/msk/if_msk.c,v 1.26 2007/12/05 09:41:58 remko Exp $ */ -/* $DragonFly: src/sys/dev/netif/msk/if_msk.c,v 1.3 2008/03/23 09:48:20 sephe Exp $ */ +/* $DragonFly: src/sys/dev/netif/msk/if_msk.c,v 1.4 2008/05/14 11:59:20 sephe Exp $ */ /* * Device driver for the Marvell Yukon II Ethernet controller. @@ -106,6 +106,7 @@ #include #include #include +#include #include #include #include @@ -1440,7 +1441,7 @@ static int mskc_attach(device_t dev) { struct msk_softc *sc; - int error, *port; + int error, *port, cpuid; sc = device_get_softc(dev); sc->msk_dev = dev; @@ -1652,6 +1653,14 @@ mskc_attach(device_t dev) device_printf(dev, "couldn't set up interrupt handler\n"); goto fail; } + + cpuid = ithread_cpuid(rman_get_start(sc->msk_irq)); + KKASSERT(cpuid >= 0 && cpuid < ncpus); + + if (sc->msk_if[0] != NULL) + sc->msk_if[0]->msk_ifp->if_cpuid = cpuid; + if (sc->msk_if[1] != NULL) + sc->msk_if[1]->msk_ifp->if_cpuid = cpuid; return 0; fail: mskc_detach(dev); @@ -2631,8 +2640,12 @@ msk_start(struct ifnet *ifp) ASSERT_SERIALIZED(ifp->if_serializer); - if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != - IFF_RUNNING || sc_if->msk_link == 0) + if (!sc_if->msk_link) { + ifq_purge(&ifp->if_snd); + return; + } + + if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING) return; for (enq = 0; !ifq_is_empty(&ifp->if_snd) && @@ -2704,7 +2717,7 @@ msk_watchdog(struct ifnet *ifp) if_printf(ifp, "watchdog timeout (missed Tx interrupts) " "-- recovering\n"); if (!ifq_is_empty(&ifp->if_snd)) - ifp->if_start(ifp); + if_devstart(ifp); return; } } @@ -2713,7 +2726,7 @@ msk_watchdog(struct ifnet *ifp) ifp->if_oerrors++; msk_init(sc_if); if (!ifq_is_empty(&ifp->if_snd)) - ifp->if_start(ifp); + if_devstart(ifp); } static int @@ -3330,10 +3343,10 @@ mskc_intr(void *xsc) if (ifp0 != NULL && (ifp0->if_flags & IFF_RUNNING) != 0 && !ifq_is_empty(&ifp0->if_snd)) - ifp0->if_start(ifp0); + if_devstart(ifp0); if (ifp1 != NULL && (ifp1->if_flags & IFF_RUNNING) != 0 && !ifq_is_empty(&ifp1->if_snd)) - ifp1->if_start(ifp1); + if_devstart(ifp1); } static void diff --git a/sys/dev/netif/my/if_my.c b/sys/dev/netif/my/if_my.c index a1879aa65b..cacd25cdaa 100644 --- a/sys/dev/netif/my/if_my.c +++ b/sys/dev/netif/my/if_my.c @@ -26,7 +26,7 @@ * Written by: yen_cw@myson.com.tw available at: http://www.myson.com.tw/ * * $FreeBSD: src/sys/dev/my/if_my.c,v 1.2.2.4 2002/04/17 02:05:27 julian Exp $ - * $DragonFly: src/sys/dev/netif/my/if_my.c,v 1.29 2008/01/06 16:55:50 swildner Exp $ + * $DragonFly: src/sys/dev/netif/my/if_my.c,v 1.30 2008/05/14 11:59:21 sephe Exp $ * * Myson fast ethernet PCI NIC driver * @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -539,7 +540,7 @@ my_autoneg_mii(struct my_softc * sc, int flag, int verbose) if (sc->my_tx_pend) { sc->my_autoneg = 0; sc->my_tx_pend = 0; - my_start(ifp); + if_devstart(ifp); } } @@ -960,6 +961,9 @@ my_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->my_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return (0); fail: @@ -1261,7 +1265,7 @@ my_intr(void *arg) /* Re-enable interrupts. */ CSR_WRITE_4(sc, MY_IMR, MY_INTRS); if (!ifq_is_empty(&ifp->if_snd)) - my_start(ifp); + if_devstart(ifp); } /* @@ -1334,6 +1338,7 @@ my_start(struct ifnet * ifp) crit_enter(); if (sc->my_autoneg) { + ifq_purge(&ifp->if_snd); sc->my_tx_pend = 1; crit_exit(); return; @@ -1628,7 +1633,7 @@ my_watchdog(struct ifnet * ifp) my_reset(sc); my_init(sc); if (!ifq_is_empty(&ifp->if_snd)) - my_start(ifp); + if_devstart(ifp); crit_exit(); } diff --git a/sys/dev/netif/ndis/if_ndis.c b/sys/dev/netif/ndis/if_ndis.c index 6bc953c357..b3bb39b640 100644 --- a/sys/dev/netif/ndis/if_ndis.c +++ b/sys/dev/netif/ndis/if_ndis.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/if_ndis/if_ndis.c,v 1.65 2004/07/07 17:46:30 wpaul Exp $ - * $DragonFly: src/sys/dev/netif/ndis/if_ndis.c,v 1.21 2007/08/14 13:30:35 sephe Exp $ + * $DragonFly: src/sys/dev/netif/ndis/if_ndis.c,v 1.22 2008/05/14 11:59:21 sephe Exp $ */ #include @@ -1065,7 +1065,7 @@ ndis_starttask(void *arg) ifp = arg; if (!ifq_is_empty(&ifp->if_snd)) - ndis_start(ifp); + if_devstart(ifp); } /* @@ -1099,16 +1099,16 @@ ndis_start(struct ifnet *ifp) p0 = &sc->ndis_txarray[sc->ndis_txidx]; while(sc->ndis_txpending) { - m = ifq_poll(&ifp->if_snd); + m = ifq_dequeue(&ifp->if_snd, NULL); if (m == NULL) break; sc->ndis_txarray[sc->ndis_txidx] = NULL; if (ndis_mtop(m, &sc->ndis_txarray[sc->ndis_txidx])) { + m_freem(m); return; } - ifq_dequeue(&ifp->if_snd, m); /* * Save pointer to original mbuf diff --git a/sys/dev/netif/nfe/if_nfe.c b/sys/dev/netif/nfe/if_nfe.c index 7ca3a865f1..a793402582 100644 --- a/sys/dev/netif/nfe/if_nfe.c +++ b/sys/dev/netif/nfe/if_nfe.c @@ -1,5 +1,5 @@ /* $OpenBSD: if_nfe.c,v 1.63 2006/06/17 18:00:43 brad Exp $ */ -/* $DragonFly: src/sys/dev/netif/nfe/if_nfe.c,v 1.19 2008/03/10 12:59:51 sephe Exp $ */ +/* $DragonFly: src/sys/dev/netif/nfe/if_nfe.c,v 1.20 2008/05/14 11:59:21 sephe Exp $ */ /* * Copyright (c) 2006 The DragonFly Project. All rights reserved. @@ -61,6 +61,7 @@ #include #include #include +#include #include #include #include @@ -527,6 +528,9 @@ nfe_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->sc_irq_res)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return 0; fail: nfe_detach(dev); @@ -1018,7 +1022,7 @@ skip: if (data != NULL) { /* at least one slot freed */ ifp->if_flags &= ~IFF_OACTIVE; - ifp->if_start(ifp); + if_devstart(ifp); } } @@ -1178,10 +1182,7 @@ nfe_start(struct ifnet *ifp) int count = 0; struct mbuf *m0; - if (ifp->if_flags & IFF_OACTIVE) - return; - - if (ifq_is_empty(&ifp->if_snd)) + if ((ifp->if_flags & (IFF_OACTIVE | IFF_RUNNING)) != IFF_RUNNING) return; for (;;) { @@ -1393,7 +1394,7 @@ nfe_init(void *xsc) * so we are not going to get an interrupt, jump-start any pending * output. */ - ifp->if_start(ifp); + if_devstart(ifp); } static void diff --git a/sys/dev/netif/nge/if_nge.c b/sys/dev/netif/nge/if_nge.c index 8d51ca4f75..df0f62e03e 100644 --- a/sys/dev/netif/nge/if_nge.c +++ b/sys/dev/netif/nge/if_nge.c @@ -31,7 +31,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/nge/if_nge.c,v 1.13.2.13 2003/02/05 22:03:57 mbr Exp $ - * $DragonFly: src/sys/dev/netif/nge/if_nge.c,v 1.46 2008/03/10 12:59:51 sephe Exp $ + * $DragonFly: src/sys/dev/netif/nge/if_nge.c,v 1.47 2008/05/14 11:59:21 sephe Exp $ */ /* @@ -96,6 +96,7 @@ #include #include #include +#include #include #include #include @@ -907,6 +908,9 @@ nge_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->nge_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return(0); fail: nge_detach(dev); @@ -1369,7 +1373,7 @@ nge_tick(void *xsc) nge_miibus_statchg(sc->nge_miibus); sc->nge_link++; if (!ifq_is_empty(&ifp->if_snd)) - nge_start(ifp); + if_devstart(ifp); } } } else { @@ -1385,7 +1389,7 @@ nge_tick(void *xsc) kprintf("nge%d: gigabit link up\n", sc->nge_unit); if (!ifq_is_empty(&ifp->if_snd)) - nge_start(ifp); + if_devstart(ifp); } } } @@ -1422,7 +1426,7 @@ nge_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) nge_rxeof(sc); nge_txeof(sc); if (!ifq_is_empty(&ifp->if_snd)) - nge_start(ifp); + if_devstart(ifp); if (sc->rxcycles > 0 || cmd == POLL_AND_CHECK_STATUS) { uint32_t status; @@ -1511,7 +1515,7 @@ nge_intr(void *arg) CSR_WRITE_4(sc, NGE_IER, 1); if (!ifq_is_empty(&ifp->if_snd)) - nge_start(ifp); + if_devstart(ifp); /* Data LED off for TBI mode */ @@ -1536,14 +1540,13 @@ nge_encap(struct nge_softc *sc, struct mbuf *m_head, uint32_t *txidx) * the fragment pointers. Stop when we run out * of fragments or hit the end of the mbuf chain. */ - m = m_head; cur = frag = *txidx; for (m = m_head; m != NULL; m = m->m_next) { if (m->m_len != 0) { if ((NGE_TX_LIST_CNT - (sc->nge_cdata.nge_tx_cnt + cnt)) < 2) - return(ENOBUFS); + break; f = &sc->nge_ldata->nge_tx_list[frag]; f->nge_ctl = NGE_CMDSTS_MORE | m->m_len; f->nge_ptr = vtophys(mtod(m, vm_offset_t)); @@ -1554,9 +1557,8 @@ nge_encap(struct nge_softc *sc, struct mbuf *m_head, uint32_t *txidx) cnt++; } } - - if (m != NULL) - return(ENOBUFS); + /* Caller should make sure that 'm_head' is not excessive fragmented */ + KASSERT(m == NULL, ("too many fragments\n")); sc->nge_ldata->nge_tx_list[*txidx].nge_extsts = 0; if (m_head->m_pkthdr.csum_flags) { @@ -1596,29 +1598,59 @@ static void nge_start(struct ifnet *ifp) { struct nge_softc *sc = ifp->if_softc; - struct mbuf *m_head = NULL; + struct mbuf *m_head = NULL, *m_defragged; uint32_t idx; int need_trans; - if (!sc->nge_link) + if (!sc->nge_link) { + ifq_purge(&ifp->if_snd); return; + } idx = sc->nge_cdata.nge_tx_prod; - if (ifp->if_flags & IFF_OACTIVE) + if ((ifp->if_flags & (IFF_OACTIVE | IFF_RUNNING)) != IFF_RUNNING) return; need_trans = 0; - while(sc->nge_ldata->nge_tx_list[idx].nge_mbuf == NULL) { - m_head = ifq_poll(&ifp->if_snd); + while (sc->nge_ldata->nge_tx_list[idx].nge_mbuf == NULL) { + struct mbuf *m; + int cnt; + + m_defragged = NULL; + m_head = ifq_dequeue(&ifp->if_snd, NULL); if (m_head == NULL) break; - if (nge_encap(sc, m_head, &idx)) { - ifp->if_flags |= IFF_OACTIVE; - break; +again: + cnt = 0; + for (m = m_head; m != NULL; m = m->m_next) + ++cnt; + if ((NGE_TX_LIST_CNT - + (sc->nge_cdata.nge_tx_cnt + cnt)) < 2) { + if (m_defragged != NULL) { + /* + * Even after defragmentation, there + * are still too many fragments, so + * drop this packet. + */ + m_freem(m_head); + ifp->if_flags |= IFF_OACTIVE; + break; + } + + m_defragged = m_defrag(m_head, MB_DONTWAIT); + if (m_defragged == NULL) { + m_freem(m_head); + continue; + } + m_head = m_defragged; + + /* Recount # of fragments */ + goto again; } - ifq_dequeue(&ifp->if_snd, m_head); + + nge_encap(sc, m_head, &idx); need_trans = 1; ETHER_BPF_MTAP(ifp, m_head); @@ -2007,7 +2039,7 @@ nge_watchdog(struct ifnet *ifp) nge_init(sc); if (!ifq_is_empty(&ifp->if_snd)) - nge_start(ifp); + if_devstart(ifp); } /* diff --git a/sys/dev/netif/pcn/if_pcn.c b/sys/dev/netif/pcn/if_pcn.c index 958d38ac1c..2d1f82f742 100644 --- a/sys/dev/netif/pcn/if_pcn.c +++ b/sys/dev/netif/pcn/if_pcn.c @@ -31,7 +31,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/pci/if_pcn.c,v 1.5.2.10 2003/03/05 18:42:33 njl Exp $ - * $DragonFly: src/sys/dev/netif/pcn/if_pcn.c,v 1.32 2006/12/22 23:26:21 swildner Exp $ + * $DragonFly: src/sys/dev/netif/pcn/if_pcn.c,v 1.33 2008/05/14 11:59:21 sephe Exp $ */ /* @@ -62,6 +62,7 @@ #include #include #include +#include #include #include #include @@ -623,6 +624,9 @@ pcn_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->pcn_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return (0); fail: pcn_detach(dev); @@ -890,7 +894,7 @@ pcn_tick(void *xsc) IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) sc->pcn_link++; if (!ifq_is_empty(&ifp->if_snd)) - pcn_start(ifp); + if_devstart(ifp); } callout_reset(&sc->pcn_stat_timer, hz, pcn_tick, sc); @@ -931,9 +935,7 @@ pcn_intr(void *arg) } if (!ifq_is_empty(&ifp->if_snd)) - pcn_start(ifp); - - return; + if_devstart(ifp); } /* @@ -952,14 +954,13 @@ pcn_encap(struct pcn_softc *sc, struct mbuf *m_head, u_int32_t *txidx) * the fragment pointers. Stop when we run out * of fragments or hit the end of the mbuf chain. */ - m = m_head; cur = frag = *txidx; for (m = m_head; m != NULL; m = m->m_next) { if (m->m_len != 0) { if ((PCN_TX_LIST_CNT - (sc->pcn_cdata.pcn_tx_cnt + cnt)) < 2) - return(ENOBUFS); + break; f = &sc->pcn_ldata->pcn_tx_list[frag]; f->pcn_txctl = (~(m->m_len) + 1) & PCN_TXCTL_BUFSZ; f->pcn_txctl |= PCN_TXCTL_MBO; @@ -973,9 +974,8 @@ pcn_encap(struct pcn_softc *sc, struct mbuf *m_head, u_int32_t *txidx) cnt++; } } - - if (m != NULL) - return(ENOBUFS); + /* Caller should make sure that 'm_head' is not excessive fragmented */ + KASSERT(m == NULL, ("too many fragments\n")); sc->pcn_cdata.pcn_tx_chain[cur] = m_head; sc->pcn_ldata->pcn_tx_list[cur].pcn_txctl |= @@ -997,31 +997,61 @@ static void pcn_start(struct ifnet *ifp) { struct pcn_softc *sc; - struct mbuf *m_head = NULL; + struct mbuf *m_head = NULL, *m_defragged; u_int32_t idx; int need_trans; sc = ifp->if_softc; - if (!sc->pcn_link) + if (!sc->pcn_link) { + ifq_purge(&ifp->if_snd); return; + } idx = sc->pcn_cdata.pcn_tx_prod; - if (ifp->if_flags & IFF_OACTIVE) + if ((ifp->if_flags & (IFF_OACTIVE | IFF_RUNNING)) != IFF_RUNNING) return; need_trans = 0; - while(sc->pcn_cdata.pcn_tx_chain[idx] == NULL) { - m_head = ifq_poll(&ifp->if_snd); + while (sc->pcn_cdata.pcn_tx_chain[idx] == NULL) { + struct mbuf *m; + int cnt; + + m_defragged = NULL; + m_head = ifq_dequeue(&ifp->if_snd, NULL); if (m_head == NULL) break; - if (pcn_encap(sc, m_head, &idx)) { - ifp->if_flags |= IFF_OACTIVE; - break; +again: + cnt = 0; + for (m = m_head; m != NULL; m = m->m_next) + ++cnt; + if ((PCN_TX_LIST_CNT - + (sc->pcn_cdata.pcn_tx_cnt + cnt)) < 2) { + if (m_defragged != NULL) { + /* + * Even after defragmentation, there + * are still too many fragments, so + * drop this packet. + */ + m_freem(m_head); + ifp->if_flags |= IFF_OACTIVE; + break; + } + + m_defragged = m_defrag(m_head, MB_DONTWAIT); + if (m_defragged == NULL) { + m_freem(m_head); + continue; + } + m_head = m_defragged; + + /* Recount # of fragments */ + goto again; } - ifq_dequeue(&ifp->if_snd, m_head); + + pcn_encap(sc, m_head, &idx); need_trans = 1; BPF_MTAP(ifp, m_head); @@ -1281,9 +1311,7 @@ pcn_watchdog(struct ifnet *ifp) pcn_init(sc); if (!ifq_is_empty(&ifp->if_snd)) - pcn_start(ifp); - - return; + if_devstart(ifp); } /* diff --git a/sys/dev/netif/ral/rt2560.c b/sys/dev/netif/ral/rt2560.c index 9afd32b7ed..01204bb335 100644 --- a/sys/dev/netif/ral/rt2560.c +++ b/sys/dev/netif/ral/rt2560.c @@ -15,7 +15,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. * * $FreeBSD: src/sys/dev/ral/rt2560.c,v 1.3 2006/03/21 21:15:43 damien Exp $ - * $DragonFly: src/sys/dev/netif/ral/rt2560.c,v 1.35 2008/02/08 09:42:29 sephe Exp $ + * $DragonFly: src/sys/dev/netif/ral/rt2560.c,v 1.36 2008/05/14 11:59:21 sephe Exp $ */ /* @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -402,6 +403,9 @@ rt2560_attach(device_t dev, int id) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->sc_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + if (bootverbose) ieee80211_announce(ic); return 0; @@ -1133,7 +1137,7 @@ rt2560_tx_intr(struct rt2560_softc *sc) if ((sc->sc_flags & (RT2560_FLAG_DATA_OACT | RT2560_FLAG_PRIO_OACT)) == 0) ifp->if_flags &= ~IFF_OACTIVE; - rt2560_start(ifp); + ifp->if_start(ifp); } } @@ -1206,7 +1210,7 @@ rt2560_prio_intr(struct rt2560_softc *sc) if ((sc->sc_flags & (RT2560_FLAG_DATA_OACT | RT2560_FLAG_PRIO_OACT)) == 0) ifp->if_flags &= ~IFF_OACTIVE; - rt2560_start(ifp); + ifp->if_start(ifp); } } @@ -1966,19 +1970,20 @@ rt2560_start(struct ifnet *ifp) if (rt2560_tx_mgt(sc, m0, ni) != 0) break; - } else { - if (ic->ic_state != IEEE80211_S_RUN) - break; - m0 = ifq_poll(&ifp->if_snd); - if (m0 == NULL) + if (ic->ic_state != IEEE80211_S_RUN) { + ifq_purge(&ifp->if_snd); break; + } + if (sc->txq.queued >= RT2560_TX_RING_COUNT - 1) { ifp->if_flags |= IFF_OACTIVE; sc->sc_flags |= RT2560_FLAG_DATA_OACT; break; } - m0 = ifq_dequeue(&ifp->if_snd, m0); + m0 = ifq_dequeue(&ifp->if_snd, NULL); + if (m0 == NULL) + break; if (m0->m_len < sizeof (struct ether_header) && !(m0 = m_pullup(m0, sizeof (struct ether_header)))) diff --git a/sys/dev/netif/ral/rt2661.c b/sys/dev/netif/ral/rt2661.c index 56d618f927..433c373096 100644 --- a/sys/dev/netif/ral/rt2661.c +++ b/sys/dev/netif/ral/rt2661.c @@ -15,7 +15,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. * * $FreeBSD: src/sys/dev/ral/rt2661.c,v 1.4 2006/03/21 21:15:43 damien Exp $ - * $DragonFly: src/sys/dev/netif/ral/rt2661.c,v 1.29 2008/02/08 09:42:30 sephe Exp $ + * $DragonFly: src/sys/dev/netif/ral/rt2661.c,v 1.30 2008/05/14 11:59:21 sephe Exp $ */ /* @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -528,6 +529,9 @@ rt2661_attach(device_t dev, int id) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->sc_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + if (bootverbose) ieee80211_announce(ic); return 0; @@ -1194,7 +1198,7 @@ rt2661_tx_dma_intr(struct rt2661_softc *sc, struct rt2661_tx_ring *txq) sc->sc_tx_timer = 0; ifp->if_flags &= ~IFF_OACTIVE; - rt2661_start(ifp); + ifp->if_start(ifp); } } @@ -1925,8 +1929,10 @@ rt2661_start(struct ifnet *ifp) if (rt2661_tx_mgt(sc, m0, ni) != 0) break; } else { - if (ic->ic_state != IEEE80211_S_RUN) + if (ic->ic_state != IEEE80211_S_RUN) { + ifq_purge(&ifp->if_snd); break; + } m0 = ifq_dequeue(&ifp->if_snd, NULL); if (m0 == NULL) diff --git a/sys/dev/netif/re/if_re.c b/sys/dev/netif/re/if_re.c index 4f4f50982f..78c89794be 100644 --- a/sys/dev/netif/re/if_re.c +++ b/sys/dev/netif/re/if_re.c @@ -33,7 +33,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/re/if_re.c,v 1.25 2004/06/09 14:34:01 naddy Exp $ - * $DragonFly: src/sys/dev/netif/re/if_re.c,v 1.41 2008/04/27 15:10:37 sephe Exp $ + * $DragonFly: src/sys/dev/netif/re/if_re.c,v 1.42 2008/05/14 11:59:21 sephe Exp $ */ /* @@ -117,9 +117,9 @@ #include #include #include +#include #include #include -/* #include */ #include #include #include @@ -1286,6 +1286,9 @@ re_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->re_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + fail: if (error) re_detach(dev); @@ -1721,7 +1724,7 @@ re_tick_serialized(void *xsc) IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) { sc->re_link = 1; if (!ifq_is_empty(&ifp->if_snd)) - ifp->if_start(ifp); + if_devstart(ifp); } } @@ -1750,7 +1753,7 @@ re_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) re_txeof(sc); if (!ifq_is_empty(&ifp->if_snd)) - (*ifp->if_start)(ifp); + if_devstart(ifp); if (cmd == POLL_AND_CHECK_STATUS) { /* also check status register */ uint16_t status; @@ -1816,7 +1819,7 @@ re_intr(void *arg) } if (!ifq_is_empty(&ifp->if_snd)) - (*ifp->if_start)(ifp); + if_devstart(ifp); } static int @@ -1828,10 +1831,9 @@ re_encap(struct re_softc *sc, struct mbuf **m_head, int *idx, int *called_defrag bus_dmamap_t map; int error; - *called_defrag = 0; - if (sc->re_ldata.re_tx_free <= 4) - return(EFBIG); + KASSERT(sc->re_ldata.re_tx_free > 4, ("not enough free TX desc\n")); + *called_defrag = 0; m = *m_head; /* @@ -1977,16 +1979,27 @@ re_start(struct ifnet *ifp) struct mbuf *m_head2; int called_defrag, idx, need_trans; - if (!sc->re_link || (ifp->if_flags & IFF_OACTIVE)) + if (!sc->re_link) { + ifq_purge(&ifp->if_snd); + return; + } + + if ((ifp->if_flags & (IFF_OACTIVE | IFF_RUNNING)) != IFF_RUNNING) return; idx = sc->re_ldata.re_tx_prodidx; need_trans = 0; while (sc->re_ldata.re_tx_mbuf[idx] == NULL) { - m_head = ifq_poll(&ifp->if_snd); + if (sc->re_ldata.re_tx_free <= 4) { + ifp->if_flags |= IFF_OACTIVE; + break; + } + + m_head = ifq_dequeue(&ifp->if_snd, NULL); if (m_head == NULL) break; + m_head2 = m_head; if (re_encap(sc, &m_head2, &idx, &called_defrag)) { /* @@ -1994,10 +2007,8 @@ re_start(struct ifnet *ifp) * the returned m_head2 is garbage and we must dequeue * and throw away the original packet. */ - if (called_defrag) { - ifq_dequeue(&ifp->if_snd, m_head); + if (called_defrag) m_freem(m_head); - } ifp->if_flags |= IFF_OACTIVE; break; } @@ -2008,7 +2019,6 @@ re_start(struct ifnet *ifp) * and the original must be thrown away. Otherwise m_head2 * *IS* the original. */ - ifq_dequeue(&ifp->if_snd, m_head); if (called_defrag) m_freem(m_head); need_trans = 1; @@ -2322,7 +2332,7 @@ re_watchdog(struct ifnet *ifp) re_init(sc); if (!ifq_is_empty(&ifp->if_snd)) - ifp->if_start(ifp); + if_devstart(ifp); } /* diff --git a/sys/dev/netif/rl/if_rl.c b/sys/dev/netif/rl/if_rl.c index ecafe26dda..c17afa8401 100644 --- a/sys/dev/netif/rl/if_rl.c +++ b/sys/dev/netif/rl/if_rl.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/pci/if_rl.c,v 1.38.2.16 2003/03/05 18:42:33 njl Exp $ - * $DragonFly: src/sys/dev/netif/rl/if_rl.c,v 1.36 2007/06/26 07:47:28 hasso Exp $ + * $DragonFly: src/sys/dev/netif/rl/if_rl.c,v 1.37 2008/05/14 11:59:21 sephe Exp $ */ /* @@ -99,6 +99,7 @@ #include #include #include +#include #include #include @@ -937,6 +938,9 @@ rl_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->rl_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return(0); fail: @@ -1245,7 +1249,7 @@ rl_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) rl_rxeof(sc); rl_txeof(sc); if (!ifq_is_empty(&ifp->if_snd)) - rl_start(ifp); + if_devstart(ifp); if (cmd == POLL_AND_CHECK_STATUS) { /* also check status register */ uint16_t status; @@ -1313,7 +1317,7 @@ rl_intr(void *arg) } if (!ifq_is_empty(&ifp->if_snd)) - rl_start(ifp); + if_devstart(ifp); } /* @@ -1365,12 +1369,13 @@ rl_encap(struct rl_softc *sc, struct mbuf *m_head) static void rl_start(struct ifnet *ifp) { - struct rl_softc *sc; + struct rl_softc *sc = ifp->if_softc; struct mbuf *m_head = NULL; - sc = ifp->if_softc; + if ((ifp->if_flags & (IFF_OACTIVE | IFF_RUNNING)) != IFF_RUNNING) + return; - while(RL_CUR_TXMBUF(sc) == NULL) { + while (RL_CUR_TXMBUF(sc) == NULL) { m_head = ifq_dequeue(&ifp->if_snd, NULL); if (m_head == NULL) break; diff --git a/sys/dev/netif/rtw/rtw.c b/sys/dev/netif/rtw/rtw.c index 0f145143fc..0c06dceaf7 100644 --- a/sys/dev/netif/rtw/rtw.c +++ b/sys/dev/netif/rtw/rtw.c @@ -32,7 +32,7 @@ * SUCH DAMAGE. * * $NetBSD: rtw.c,v 1.72 2006/03/28 00:48:10 dyoung Exp $ - * $DragonFly: src/sys/dev/netif/rtw/rtw.c,v 1.12 2008/01/15 09:01:13 sephe Exp $ + * $DragonFly: src/sys/dev/netif/rtw/rtw.c,v 1.13 2008/05/14 11:59:21 sephe Exp $ */ /* @@ -75,6 +75,7 @@ #include #include #include +#include #include #include #include @@ -2954,17 +2955,20 @@ rtw_dequeue(struct ifnet *ifp, struct rtw_txsoft_blk **tsbp, *mp = NULL; if (sc->sc_ic.ic_state != IEEE80211_S_RUN) { + ifq_purge(&ifp->if_snd); DPRINTF(sc, RTW_DEBUG_XMIT, ("%s: not running\n", ifp->if_xname)); return 0; } - m0 = ifq_poll(&ifp->if_snd); + m0 = ifq_dequeue(&ifp->if_snd, NULL); if (m0 == NULL) { DPRINTF(sc, RTW_DEBUG_XMIT, ("%s: no frame ready\n", ifp->if_xname)); return 0; } + DPRINTF(sc, RTW_DEBUG_XMIT, + ("%s: dequeue data frame\n", ifp->if_xname)); pri = ((m0->m_flags & M_PWR_SAV) != 0) ? RTW_TXPRIHI : RTW_TXPRIMD; @@ -2972,14 +2976,11 @@ rtw_dequeue(struct ifnet *ifp, struct rtw_txsoft_blk **tsbp, DPRINTF(sc, RTW_DEBUG_XMIT_RSRC, ("%s: no ring %d descriptor\n", ifp->if_xname, pri)); *if_flagsp |= IFF_OACTIVE; + ifq_prepend(&ifp->if_snd, m0); sc->sc_if.if_timer = 1; return 0; } - ifq_dequeue(&ifp->if_snd, m0); - DPRINTF(sc, RTW_DEBUG_XMIT, - ("%s: dequeue data frame\n", ifp->if_xname)); - BPF_MTAP(ifp, m0); eh = mtod(m0, struct ether_header *); @@ -3903,6 +3904,9 @@ rtw_attach(device_t dev) goto err; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->sc_irq_res)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + device_printf(dev, "hardware version %c\n", sc->sc_hwverid); if (bootverbose) ieee80211_announce(ic); diff --git a/sys/dev/netif/rue/if_rue.c b/sys/dev/netif/rue/if_rue.c index dec7bf07d1..e731a60b7a 100644 --- a/sys/dev/netif/rue/if_rue.c +++ b/sys/dev/netif/rue/if_rue.c @@ -55,7 +55,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/usb/if_rue.c,v 1.14 2004/06/09 14:34:03 naddy Exp $ - * $DragonFly: src/sys/dev/netif/rue/if_rue.c,v 1.12 2007/11/06 07:37:00 hasso Exp $ + * $DragonFly: src/sys/dev/netif/rue/if_rue.c,v 1.13 2008/05/14 11:59:21 sephe Exp $ */ /* @@ -901,7 +901,7 @@ rue_txeof(usbd_xfer_handle xfer, usbd_private_handle priv, usbd_status status) ifp->if_opackets++; if (!ifq_is_empty(&ifp->if_snd)) - ifp->if_start(ifp); + if_devstart(ifp); RUE_UNLOCK(sc); } @@ -930,7 +930,7 @@ rue_tick(void *xsc) IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) { sc->rue_link++; if (!ifq_is_empty(&ifp->if_snd)) - rue_start(ifp); + if_devstart(ifp); } callout_reset(&sc->rue_stat_ch, hz, rue_tick, sc); @@ -988,6 +988,7 @@ rue_start(struct ifnet *ifp) RUE_LOCK(sc); if (!sc->rue_link) { + ifq_purge(&ifp->if_snd); RUE_UNLOCK(sc); return; } @@ -997,18 +998,18 @@ rue_start(struct ifnet *ifp) return; } - m_head = ifq_poll(&ifp->if_snd); + m_head = ifq_dequeue(&ifp->if_snd, NULL); if (m_head == NULL) { RUE_UNLOCK(sc); return; } if (rue_encap(sc, m_head, 0)) { + /* rue_encap() will free m_head, if we reach here */ ifp->if_flags |= IFF_OACTIVE; RUE_UNLOCK(sc); return; } - ifq_dequeue(&ifp->if_snd, m_head); /* * If there's a BPF listener, bounce a copy of this frame @@ -1252,7 +1253,7 @@ rue_watchdog(struct ifnet *ifp) rue_txeof(c->rue_xfer, c, stat); if (!ifq_is_empty(&ifp->if_snd)) - rue_start(ifp); + if_devstart(ifp); RUE_UNLOCK(sc); } diff --git a/sys/dev/netif/rum/if_rum.c b/sys/dev/netif/rum/if_rum.c index ee6522f71d..abd8b22e6d 100644 --- a/sys/dev/netif/rum/if_rum.c +++ b/sys/dev/netif/rum/if_rum.c @@ -1,5 +1,5 @@ /* $OpenBSD: if_rum.c,v 1.40 2006/09/18 16:20:20 damien Exp $ */ -/* $DragonFly: src/sys/dev/netif/rum/if_rum.c,v 1.27 2008/01/15 09:45:35 sephe Exp $ */ +/* $DragonFly: src/sys/dev/netif/rum/if_rum.c,v 1.28 2008/05/14 11:59:21 sephe Exp $ */ /*- * Copyright (c) 2005, 2006 Damien Bergamini @@ -1125,8 +1125,10 @@ rum_start(struct ifnet *ifp) ASSERT_SERIALIZED(ifp->if_serializer); - if (sc->sc_stopped) + if (sc->sc_stopped) { + ifq_purge(&ifp->if_snd); return; + } crit_enter(); @@ -1158,17 +1160,19 @@ rum_start(struct ifnet *ifp) } else { struct ether_header *eh; - if (ic->ic_state != IEEE80211_S_RUN) + if (ic->ic_state != IEEE80211_S_RUN) { + ifq_purge(&ifp->if_snd); break; + } - m0 = ifq_poll(&ifp->if_snd); - if (m0 == NULL) - break; if (sc->tx_queued >= RT2573_TX_LIST_COUNT) { ifp->if_flags |= IFF_OACTIVE; break; } - ifq_dequeue(&ifp->if_snd, m0); + + m0 = ifq_dequeue(&ifp->if_snd, NULL); + if (m0 == NULL) + break; if (m0->m_len < sizeof(struct ether_header)) { m0 = m_pullup(m0, sizeof(struct ether_header)); diff --git a/sys/dev/netif/sbni/if_sbni.c b/sys/dev/netif/sbni/if_sbni.c index d6f55b7052..862e9f0d23 100644 --- a/sys/dev/netif/sbni/if_sbni.c +++ b/sys/dev/netif/sbni/if_sbni.c @@ -25,7 +25,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/dev/sbni/if_sbni.c,v 1.1.2.4 2002/08/11 09:32:00 fjoe Exp $ - * $DragonFly: src/sys/dev/netif/sbni/if_sbni.c,v 1.27 2008/03/10 08:36:30 sephe Exp $ + * $DragonFly: src/sys/dev/netif/sbni/if_sbni.c,v 1.28 2008/05/14 11:59:21 sephe Exp $ */ /* @@ -278,7 +278,7 @@ sbni_init(void *xsc) ifp->if_flags &= ~IFF_OACTIVE; /* attempt to start output */ - sbni_start(ifp); + if_devstart(ifp); } diff --git a/sys/dev/netif/sbni/if_sbni_pci.c b/sys/dev/netif/sbni/if_sbni_pci.c index 9744673d25..7c869c5fd9 100644 --- a/sys/dev/netif/sbni/if_sbni_pci.c +++ b/sys/dev/netif/sbni/if_sbni_pci.c @@ -25,7 +25,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/dev/sbni/if_sbni_pci.c,v 1.6 2002/09/28 20:59:59 phk Exp $ - * $DragonFly: src/sys/dev/netif/sbni/if_sbni_pci.c,v 1.14 2006/12/22 23:26:21 swildner Exp $ + * $DragonFly: src/sys/dev/netif/sbni/if_sbni_pci.c,v 1.15 2008/05/14 11:59:21 sephe Exp $ */ @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -139,12 +140,17 @@ sbni_pci_attach(device_t dev) sbni_attach(sc->slave_sc, next_sbni_unit++, flags); if (sc->irq_res) { + struct ifnet *ifp = &sc->arpcom.ac_if; + error = bus_setup_intr(dev, sc->irq_res, INTR_NETSAFE, - sbni_intr, sc, - &sc->irq_handle, - sc->arpcom.ac_if.if_serializer); + sbni_intr, sc, &sc->irq_handle, + ifp->if_serializer); if (error) { kprintf("sbni%d: bus_setup_intr\n", next_sbni_unit); + } else { + ifp->if_cpuid = + ithread_cpuid(rman_get_start(sc->irq_res)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); } } else { kprintf("\nsbni%d: cannot claim irq!\n", next_sbni_unit); diff --git a/sys/dev/netif/sbsh/if_sbsh.c b/sys/dev/netif/sbsh/if_sbsh.c index f80c04b593..e7ee68b77c 100644 --- a/sys/dev/netif/sbsh/if_sbsh.c +++ b/sys/dev/netif/sbsh/if_sbsh.c @@ -24,7 +24,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/dev/sbsh/if_sbsh.c,v 1.3.2.1 2003/04/15 18:15:07 fjoe Exp $ - * $DragonFly: src/sys/dev/netif/sbsh/if_sbsh.c,v 1.26 2008/03/10 08:39:53 sephe Exp $ + * $DragonFly: src/sys/dev/netif/sbsh/if_sbsh.c,v 1.27 2008/05/14 11:59:21 sephe Exp $ */ #include @@ -40,6 +40,7 @@ #include #include #include +#include #include #include @@ -274,6 +275,9 @@ sbsh_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->irq_res)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return(0); fail: diff --git a/sys/dev/netif/sf/if_sf.c b/sys/dev/netif/sf/if_sf.c index 0866d58d13..2007f2274c 100644 --- a/sys/dev/netif/sf/if_sf.c +++ b/sys/dev/netif/sf/if_sf.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/pci/if_sf.c,v 1.18.2.8 2001/12/16 15:46:07 luigi Exp $ - * $DragonFly: src/sys/dev/netif/sf/if_sf.c,v 1.31 2008/01/05 14:02:37 swildner Exp $ + * $DragonFly: src/sys/dev/netif/sf/if_sf.c,v 1.32 2008/05/14 11:59:21 sephe Exp $ */ /* @@ -86,6 +86,7 @@ #include #include #include +#include #include #include #include @@ -786,6 +787,9 @@ sf_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->sf_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return(0); fail: @@ -1101,9 +1105,7 @@ sf_intr(void *arg) csr_write_4(sc, SF_IMR, SF_INTRS); if (!ifq_is_empty(&ifp->if_snd)) - sf_start(ifp); - - return; + if_devstart(ifp); } static void @@ -1217,9 +1219,7 @@ sf_encap(struct sf_softc *sc, struct sf_tx_bufdesc_type0 *c, struct sf_frag *f = NULL; struct mbuf *m; - m = m_head; - - for (m = m_head, frag = 0; m != NULL; m = m->m_next) { + for (m = m_head; m != NULL; m = m->m_next) { if (m->m_len != 0) { if (frag == SF_MAXFRAGS) break; @@ -1231,35 +1231,8 @@ sf_encap(struct sf_softc *sc, struct sf_tx_bufdesc_type0 *c, frag++; } } - - if (m != NULL) { - struct mbuf *m_new = NULL; - - MGETHDR(m_new, MB_DONTWAIT, MT_DATA); - if (m_new == NULL) { - kprintf("sf%d: no memory for tx list", sc->sf_unit); - return(1); - } - - if (m_head->m_pkthdr.len > MHLEN) { - MCLGET(m_new, MB_DONTWAIT); - if (!(m_new->m_flags & M_EXT)) { - m_freem(m_new); - kprintf("sf%d: no memory for tx list", - sc->sf_unit); - return(1); - } - } - m_copydata(m_head, 0, m_head->m_pkthdr.len, - mtod(m_new, caddr_t)); - m_new->m_pkthdr.len = m_new->m_len = m_head->m_pkthdr.len; - m_freem(m_head); - m_head = m_new; - f = &c->sf_frags[0]; - f->sf_fraglen = f->sf_pktlen = m_head->m_pkthdr.len; - f->sf_addr = vtophys(mtod(m_head, caddr_t)); - frag = 1; - } + /* Caller should make sure that 'm_head' is not excessive fragmented */ + KASSERT(m == NULL, ("too many fragments\n")); c->sf_mbuf = m_head; c->sf_id = SF_TX_BUFDESC_ID; @@ -1276,15 +1249,17 @@ sf_start(struct ifnet *ifp) { struct sf_softc *sc; struct sf_tx_bufdesc_type0 *cur_tx = NULL; - struct mbuf *m_head = NULL; - int i, txprod; + struct mbuf *m_head = NULL, *m_defragged; + int i, txprod, need_trans = 0; sc = ifp->if_softc; - if (!sc->sf_link) + if (!sc->sf_link) { + ifq_purge(&ifp->if_snd); return; + } - if (ifp->if_flags & IFF_OACTIVE) + if ((ifp->if_flags & (IFF_OACTIVE | IFF_RUNNING)) != IFF_RUNNING) return; txprod = csr_read_4(sc, SF_TXDQ_PRODIDX); @@ -1297,35 +1272,65 @@ sf_start(struct ifnet *ifp) i = SF_IDX_HI(txprod) >> 4; } - while(sc->sf_ldata->sf_tx_dlist[i].sf_mbuf == NULL) { + while (sc->sf_ldata->sf_tx_dlist[i].sf_mbuf == NULL) { + struct mbuf *m; + int frag; + + /* + * Don't get the TX DMA queue get too full. + */ + if (sc->sf_tx_cnt > 64) { + ifp->if_flags |= IFF_OACTIVE; + break; + } +#ifdef foo if (sc->sf_tx_cnt >= (SF_TX_DLIST_CNT - 5)) { ifp->if_flags |= IFF_OACTIVE; - cur_tx = NULL; break; } - m_head = ifq_poll(&ifp->if_snd); +#endif + + m_defragged = NULL; + m_head = ifq_dequeue(&ifp->if_snd, NULL); if (m_head == NULL) break; - cur_tx = &sc->sf_ldata->sf_tx_dlist[i]; - if (sf_encap(sc, cur_tx, m_head)) { - ifp->if_flags |= IFF_OACTIVE; - cur_tx = NULL; - break; +again: + frag = 0; + for (m = m_head; m != NULL; m = m->m_next) + ++frag; + if (frag > SF_MAXFRAGS) { + if (m_defragged != NULL) { + /* + * Even after defragmentation, there + * are still too many fragments, so + * drop this packet. + */ + m_freem(m_head); + continue; + } + + m_defragged = m_defrag(m_head, MB_DONTWAIT); + if (m_defragged == NULL) { + m_freem(m_head); + continue; + } + m_head = m_defragged; + + /* Recount # of fragments */ + goto again; } - ifq_dequeue(&ifp->if_snd, m_head); + + cur_tx = &sc->sf_ldata->sf_tx_dlist[i]; + sf_encap(sc, cur_tx, m_head); BPF_MTAP(ifp, cur_tx->sf_mbuf); SF_INC(i, SF_TX_DLIST_CNT); sc->sf_tx_cnt++; - /* - * Don't get the TX DMA queue get too full. - */ - if (sc->sf_tx_cnt > 64) - break; + need_trans = 1; } - if (cur_tx == NULL) + if (!need_trans) return; /* Transmit */ @@ -1334,8 +1339,6 @@ sf_start(struct ifnet *ifp) ((i << 20) & 0xFFFF0000)); ifp->if_timer = 5; - - return; } static void @@ -1415,10 +1418,11 @@ sf_stats_update(void *xsc) if (!sc->sf_link) { mii_pollstat(mii); if (mii->mii_media_status & IFM_ACTIVE && - IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) + IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) { sc->sf_link++; if (!ifq_is_empty(&ifp->if_snd)) - sf_start(ifp); + if_devstart(ifp); + } } callout_reset(&sc->sf_stat_timer, hz, sf_stats_update, sc); @@ -1441,9 +1445,7 @@ sf_watchdog(struct ifnet *ifp) sf_init(sc); if (!ifq_is_empty(&ifp->if_snd)) - sf_start(ifp); - - return; + if_devstart(ifp); } static void diff --git a/sys/dev/netif/sf/if_sfreg.h b/sys/dev/netif/sf/if_sfreg.h index 778a30b883..4537818631 100644 --- a/sys/dev/netif/sf/if_sfreg.h +++ b/sys/dev/netif/sf/if_sfreg.h @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/pci/if_sfreg.h,v 1.6.2.1 2001/08/16 20:35:04 wpaul Exp $ - * $DragonFly: src/sys/dev/netif/sf/if_sfreg.h,v 1.5 2006/08/01 18:08:24 swildner Exp $ + * $DragonFly: src/sys/dev/netif/sf/if_sfreg.h,v 1.6 2008/05/14 11:59:21 sephe Exp $ */ /* @@ -829,7 +829,7 @@ struct sf_tx_bufdesc_type0 { u_int8_t sf_fragcnt; u_int8_t sf_rsvd2; u_int16_t sf_rsvd1; - struct sf_frag sf_frags[14]; + struct sf_frag sf_frags[SF_MAXFRAGS]; }; /* diff --git a/sys/dev/netif/sis/if_sis.c b/sys/dev/netif/sis/if_sis.c index 3386bda966..21ce1b43fb 100644 --- a/sys/dev/netif/sis/if_sis.c +++ b/sys/dev/netif/sis/if_sis.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/pci/if_sis.c,v 1.13.4.24 2003/03/05 18:42:33 njl Exp $ - * $DragonFly: src/sys/dev/netif/sis/if_sis.c,v 1.36 2006/10/25 20:55:59 dillon Exp $ + * $DragonFly: src/sys/dev/netif/sis/if_sis.c,v 1.37 2008/05/14 11:59:21 sephe Exp $ */ /* @@ -72,6 +72,7 @@ #include #include #include +#include #include #include @@ -1310,6 +1311,9 @@ sis_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->sis_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + fail: if (error) sis_detach(dev); @@ -1635,7 +1639,7 @@ sis_tick(void *xsc) IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) sc->sis_link++; if (!ifq_is_empty(&ifp->if_snd)) - sis_start(ifp); + if_devstart(ifp); } callout_reset(&sc->sis_timer, hz, sis_tick, sc); @@ -1670,7 +1674,7 @@ sis_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) sis_rxeof(sc); sis_txeof(sc); if (!ifq_is_empty(&ifp->if_snd)) - sis_start(ifp); + if_devstart(ifp); if (sc->rxcycles > 0 || cmd == POLL_AND_CHECK_STATUS) { uint32_t status; @@ -1745,7 +1749,7 @@ sis_intr(void *arg) CSR_WRITE_4(sc, SIS_IER, 1); if (!ifq_is_empty(&ifp->if_snd)) - sis_start(ifp); + if_devstart(ifp); } /* @@ -1759,25 +1763,18 @@ sis_encap(struct sis_softc *sc, struct mbuf *m_head, uint32_t *txidx) struct mbuf *m; int frag, cur, cnt = 0; - /* - * If there's no way we can send any packets, return now. - */ - if (SIS_TX_LIST_CNT - sc->sis_cdata.sis_tx_cnt < 2) - return (ENOBUFS); - /* * Start packing the mbufs in this chain into * the fragment pointers. Stop when we run out * of fragments or hit the end of the mbuf chain. */ - m = m_head; cur = frag = *txidx; for (m = m_head; m != NULL; m = m->m_next) { if (m->m_len != 0) { if ((SIS_TX_LIST_CNT - (sc->sis_cdata.sis_tx_cnt + cnt)) < 2) - return(ENOBUFS); + break; f = &sc->sis_ldata.sis_tx_list[frag]; f->sis_ctl = SIS_CMDSTS_MORE | m->m_len; bus_dmamap_create(sc->sis_tag, 0, &f->sis_map); @@ -1793,9 +1790,8 @@ sis_encap(struct sis_softc *sc, struct mbuf *m_head, uint32_t *txidx) cnt++; } } - - if (m != NULL) - return(ENOBUFS); + /* Caller should make sure that 'm_head' is not excessive fragmented */ + KASSERT(m == NULL, ("too many fragments\n")); sc->sis_ldata.sis_tx_list[cur].sis_mbuf = m_head; sc->sis_ldata.sis_tx_list[cur].sis_ctl &= ~SIS_CMDSTS_MORE; @@ -1817,31 +1813,69 @@ static void sis_start(struct ifnet *ifp) { struct sis_softc *sc; - struct mbuf *m_head = NULL; + struct mbuf *m_head = NULL, *m_defragged; uint32_t idx; int need_trans; sc = ifp->if_softc; - if (!sc->sis_link) + if (!sc->sis_link) { + ifq_purge(&ifp->if_snd); return; + } idx = sc->sis_cdata.sis_tx_prod; - if (ifp->if_flags & IFF_OACTIVE) + if ((ifp->if_flags & (IFF_OACTIVE | IFF_RUNNING)) != IFF_RUNNING) return; need_trans = 0; - while(sc->sis_ldata.sis_tx_list[idx].sis_mbuf == NULL) { - m_head = ifq_poll(&ifp->if_snd); - if (m_head == NULL) - break; + while (sc->sis_ldata.sis_tx_list[idx].sis_mbuf == NULL) { + struct mbuf *m; + int cnt; - if (sis_encap(sc, m_head, &idx)) { + /* + * If there's no way we can send any packets, return now. + */ + if (SIS_TX_LIST_CNT - sc->sis_cdata.sis_tx_cnt < 2) { ifp->if_flags |= IFF_OACTIVE; break; } - ifq_dequeue(&ifp->if_snd, m_head); + + m_defragged = NULL; + m_head = ifq_dequeue(&ifp->if_snd, NULL); + if (m_head == NULL) + break; + +again: + cnt = 0; + for (m = m_head; m != NULL; m = m->m_next) + ++cnt; + if ((SIS_TX_LIST_CNT - + (sc->sis_cdata.sis_tx_cnt + cnt)) < 2) { + if (m_defragged != NULL) { + /* + * Even after defragmentation, there + * are still too many fragments, so + * drop this packet. + */ + m_freem(m_head); + ifp->if_flags |= IFF_OACTIVE; + break; + } + + m_defragged = m_defrag(m_head, MB_DONTWAIT); + if (m_defragged == NULL) { + m_freem(m_head); + continue; + } + m_head = m_defragged; + + /* Recount # of fragments */ + goto again; + } + + sis_encap(sc, m_head, &idx); need_trans = 1; /* @@ -2122,7 +2156,7 @@ sis_watchdog(struct ifnet *ifp) sis_init(sc); if (!ifq_is_empty(&ifp->if_snd)) - sis_start(ifp); + if_devstart(ifp); } /* diff --git a/sys/dev/netif/sk/if_sk.c b/sys/dev/netif/sk/if_sk.c index 45d5254e29..f092091146 100644 --- a/sys/dev/netif/sk/if_sk.c +++ b/sys/dev/netif/sk/if_sk.c @@ -31,7 +31,7 @@ * * $OpenBSD: if_sk.c,v 1.129 2006/10/16 12:30:08 tom Exp $ * $FreeBSD: /c/ncvs/src/sys/pci/if_sk.c,v 1.20 2000/04/22 02:16:37 wpaul Exp $ - * $DragonFly: src/sys/dev/netif/sk/if_sk.c,v 1.55 2008/03/16 15:50:22 hasso Exp $ + * $DragonFly: src/sys/dev/netif/sk/if_sk.c,v 1.56 2008/05/14 11:59:22 sephe Exp $ */ /* @@ -91,6 +91,7 @@ #include #include #include +#include #include #include #include @@ -1322,7 +1323,7 @@ skc_attach(device_t dev) struct sk_softc *sc = device_get_softc(dev); uint8_t skrs; int *port; - int error; + int error, cpuid; DPRINTFN(2, ("begin skc_attach\n")); @@ -1513,6 +1514,15 @@ skc_attach(device_t dev) device_printf(dev, "couldn't set up irq\n"); goto fail; } + + cpuid = ithread_cpuid(rman_get_start(sc->sk_irq)); + KKASSERT(cpuid >= 0 && cpuid < ncpus); + + if (sc->sk_if[0] != NULL) + sc->sk_if[0]->arpcom.ac_if.if_cpuid = cpuid; + if (sc->sk_if[1] != NULL) + sc->sk_if[1]->arpcom.ac_if.if_cpuid = cpuid; + return 0; fail: skc_detach(dev); @@ -1704,8 +1714,16 @@ sk_start(struct ifnet *ifp) DPRINTFN(2, ("sk_start\n")); + if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING) + return; + while (sc_if->sk_cdata.sk_tx_mbuf[idx] == NULL) { - m_head = ifq_poll(&ifp->if_snd); + if ((SK_TX_RING_CNT - sc_if->sk_cdata.sk_tx_cnt) <= 2) { + ifp->if_flags |= IFF_OACTIVE; + break; + } + + m_head = ifq_dequeue(&ifp->if_snd, NULL); if (m_head == NULL) break; @@ -1716,13 +1734,11 @@ sk_start(struct ifnet *ifp) */ if (sk_encap(sc_if, m_head, &idx)) { ifp->if_flags |= IFF_OACTIVE; + ifq_prepend(&ifp->if_snd, m_head); break; } - /* now we are committed to transmit the packet */ - ifq_dequeue(&ifp->if_snd, m_head); pkts++; - BPF_MTAP(ifp, m_head); } if (pkts == 0) @@ -2296,9 +2312,9 @@ sk_intr(void *xsc) CSR_WRITE_4(sc, SK_IMR, sc->sk_intrmask); if (ifp0 != NULL && !ifq_is_empty(&ifp0->if_snd)) - sk_start(ifp0); + if_devstart(ifp0); if (ifp1 != NULL && !ifq_is_empty(&ifp1->if_snd)) - sk_start(ifp1); + if_devstart(ifp1); } static void diff --git a/sys/dev/netif/sln/if_sln.c b/sys/dev/netif/sln/if_sln.c index 040ce0a087..a0c7dd2bf9 100644 --- a/sys/dev/netif/sln/if_sln.c +++ b/sys/dev/netif/sln/if_sln.c @@ -29,12 +29,13 @@ * SUCH DAMAGE. * * $FreeBSD-4.7: /usr/src/sys/pci/silan.c,v 1.0 2003/01/10 gaoyonghong $ - * $DragonFly: src/sys/dev/netif/sln/if_sln.c,v 1.1 2008/02/28 18:39:20 swildner Exp $ + * $DragonFly: src/sys/dev/netif/sln/if_sln.c,v 1.2 2008/05/14 11:59:22 sephe Exp $ */ #include #include #include +#include #include #include #include @@ -264,6 +265,10 @@ sln_attach(device_t dev) device_printf(dev, "couldn't set up irq\n"); goto fail; } + + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->sln_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return 0; fail: return error; @@ -733,8 +738,12 @@ sln_tx(struct ifnet *ifp) ASSERT_SERIALIZED(ifp->if_serializer); - if (!sc->connect || - (ifp->if_flags & (IFF_OACTIVE | IFF_RUNNING)) != IFF_RUNNING) + if (!sc->connect) { + ifq_purge(&ifp->if_snd); + return; + } + + if ((ifp->if_flags & (IFF_OACTIVE | IFF_RUNNING)) != IFF_RUNNING) return; while (SL_CUR_TXBUF(sc) == NULL) { /* SL_CUR_TXBUF(x) = x->sln_bufdata.sln_tx_buf[x->sln_bufdata.cur_tx] */ @@ -1084,7 +1093,7 @@ sln_interrupt(void *arg) /* Data in Tx buffer waiting for transimission */ if (!ifq_is_empty(&ifp->if_snd)) - sln_tx(ifp); + if_devstart(ifp); back: /* Re-enable interrupts. */ SLN_WRITE_4(sc, SL_INT_MASK, SL_INRTS); @@ -1147,7 +1156,7 @@ sln_watchdog(struct ifnet *ifp) sln_init(sc); if (!ifq_is_empty(&ifp->if_snd)) - sln_tx(ifp); + if_devstart(ifp); } /* Stop all chip I/O */ diff --git a/sys/dev/netif/sn/if_sn.c b/sys/dev/netif/sn/if_sn.c index f0efbe48a8..6b57803d6d 100644 --- a/sys/dev/netif/sn/if_sn.c +++ b/sys/dev/netif/sn/if_sn.c @@ -29,7 +29,7 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/sn/if_sn.c,v 1.7.2.3 2001/02/04 04:38:38 toshi Exp $ - * $DragonFly: src/sys/dev/netif/sn/if_sn.c,v 1.27 2006/12/22 23:26:22 swildner Exp $ + * $DragonFly: src/sys/dev/netif/sn/if_sn.c,v 1.28 2008/05/14 11:59:22 sephe Exp $ */ /* @@ -85,6 +85,7 @@ #include #include #include +#include #include #include #include @@ -235,6 +236,9 @@ sn_attach(device_t dev) return error; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->irq_res)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return 0; } @@ -332,7 +336,7 @@ sninit(void *xsc) /* * Attempt to push out any waiting packets. */ - snstart(ifp); + if_devstart(ifp); } @@ -350,12 +354,14 @@ snstart(struct ifnet *ifp) u_char packet_no; int time_out; - if (ifp->if_flags & IFF_OACTIVE) { + if ((ifp->if_flags & (IFF_OACTIVE | IFF_RUNNING)) != IFF_RUNNING) return; - } + if (sc->pages_wanted != -1) { + /* XXX should never happen */ kprintf("%s: snstart() while memory allocation pending\n", ifp->if_xname); + ifp->if_flags |= IFF_OACTIVE; return; } startagain: @@ -363,10 +369,10 @@ startagain: /* * Sneak a peek at the next packet */ - m = ifq_poll(&ifp->if_snd); - if (m == 0) { + m = ifq_dequeue(&ifp->if_snd, NULL); + if (m == NULL) return; - } + /* * Compute the frame length and set pad to give an overall even * number of bytes. Below we assume that the packet length is even. @@ -383,8 +389,7 @@ startagain: if (len + pad > ETHER_MAX_LEN - ETHER_CRC_LEN) { kprintf("%s: large packet discarded (A)\n", ifp->if_xname); ++sc->arpcom.ac_if.if_oerrors; - ifq_dequeue(&ifp->if_snd, m); - m_freem(m); + m_freem(top); goto readcheck; } #ifdef SW_PAD @@ -443,6 +448,7 @@ startagain: ifp->if_timer = 1; ifp->if_flags |= IFF_OACTIVE; sc->pages_wanted = numPages; + ifq_prepend(&ifp->if_snd, top); return; } @@ -452,6 +458,7 @@ startagain: packet_no = inb(BASE + ALLOC_RESULT_REG_B); if (packet_no & ARR_FAILED) { kprintf("%s: Memory allocation failed\n", ifp->if_xname); + ifq_prepend(&ifp->if_snd, top); goto startagain; } /* @@ -472,16 +479,10 @@ startagain: outb(BASE + DATA_REG_B, (length + 6) & 0xFF); outb(BASE + DATA_REG_B, (length + 6) >> 8); - /* - * Get the packet from the kernel. This will include the Ethernet - * frame header, MAC Addresses etc. - */ - ifq_dequeue(&ifp->if_snd, m); - /* * Push out the data to the card. */ - for (top = m; m != 0; m = m->m_next) { + for (m = top; m != NULL; m = m->m_next) { /* * Push out words. @@ -574,11 +575,13 @@ snresume(struct ifnet *ifp) /* * Sneak a peek at the next packet */ - m = ifq_poll(&ifp->if_snd); + m = ifq_dequeue(&ifp->if_snd, NULL); if (m == NULL) { - kprintf("%s: snresume() with nothing to send\n", ifp->if_xname); + kprintf("%s: snresume() with nothing to send\n", + ifp->if_xname); return; } + /* * Compute the frame length and set pad to give an overall even * number of bytes. Below we assume that the packet length is even. @@ -595,8 +598,7 @@ snresume(struct ifnet *ifp) if (len + pad > ETHER_MAX_LEN - ETHER_CRC_LEN) { kprintf("%s: large packet discarded (B)\n", ifp->if_xname); ++ifp->if_oerrors; - ifq_dequeue(&ifp->if_snd, m); - m_freem(m); + m_freem(top); return; } #ifdef SW_PAD @@ -631,6 +633,7 @@ snresume(struct ifnet *ifp) if (packet_no & ARR_FAILED) { kprintf("%s: Memory allocation failed. Weird.\n", ifp->if_xname); ifp->if_timer = 1; + ifq_prepend(&ifp->if_snd, top); goto try_start; } /* @@ -652,6 +655,7 @@ snresume(struct ifnet *ifp) ; outw(BASE + MMU_CMD_REG_W, MMUCR_FREEPKT); + ifq_prepend(&ifp->if_snd, top); return; } /* @@ -667,16 +671,10 @@ snresume(struct ifnet *ifp) outb(BASE + DATA_REG_B, (length + 6) & 0xFF); outb(BASE + DATA_REG_B, (length + 6) >> 8); - /* - * Get the packet from the kernel. This will include the Ethernet - * frame header, MAC Addresses etc. - */ - ifq_dequeue(&ifp->if_snd, m); - /* * Push out the data to the card. */ - for (top = m; m != 0; m = m->m_next) { + for (m = top; m != NULL; m = m->m_next) { /* * Push out words. @@ -727,7 +725,7 @@ try_start: * Now pass control to snstart() to queue any additional packets */ ifp->if_flags &= ~IFF_OACTIVE; - snstart(ifp); + if_devstart(ifp); /* * We've sent something, so we're active. Set a watchdog in case the @@ -891,7 +889,7 @@ sn_intr(void *arg) * Attempt to queue more transmits. */ sc->arpcom.ac_if.if_flags &= ~IFF_OACTIVE; - snstart(&sc->arpcom.ac_if); + if_devstart(&sc->arpcom.ac_if); } /* * Transmit underrun. We use this opportunity to update transmit @@ -929,7 +927,7 @@ sn_intr(void *arg) * Attempt to enqueue some more stuff. */ sc->arpcom.ac_if.if_flags &= ~IFF_OACTIVE; - snstart(&sc->arpcom.ac_if); + if_devstart(&sc->arpcom.ac_if); } /* * Some other error. Try to fix it by resetting the adapter. diff --git a/sys/dev/netif/ste/if_ste.c b/sys/dev/netif/ste/if_ste.c index dfe22f2354..078fb5e2fe 100644 --- a/sys/dev/netif/ste/if_ste.c +++ b/sys/dev/netif/ste/if_ste.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/pci/if_ste.c,v 1.14.2.9 2003/02/05 22:03:57 mbr Exp $ - * $DragonFly: src/sys/dev/netif/ste/if_ste.c,v 1.36 2008/01/05 14:02:37 swildner Exp $ + * $DragonFly: src/sys/dev/netif/ste/if_ste.c,v 1.37 2008/05/14 11:59:22 sephe Exp $ */ #include @@ -44,6 +44,7 @@ #include #include #include +#include #include #include @@ -621,9 +622,7 @@ ste_intr(void *xsc) CSR_WRITE_2(sc, STE_IMR, STE_INTRS); if (!ifq_is_empty(&ifp->if_snd)) - ste_start(ifp); - - return; + if_devstart(ifp); } /* @@ -805,7 +804,7 @@ ste_stats_update(void *xsc) */ ste_miibus_statchg(sc->ste_dev); if (!ifq_is_empty(&ifp->if_snd)) - ste_start(ifp); + if_devstart(ifp); } } @@ -978,6 +977,9 @@ ste_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->ste_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return 0; fail: @@ -1415,10 +1417,12 @@ ste_start(struct ifnet *ifp) sc = ifp->if_softc; - if (!sc->ste_link) + if (!sc->ste_link) { + ifq_purge(&ifp->if_snd); return; + } - if (ifp->if_flags & IFF_OACTIVE) + if ((ifp->if_flags & (IFF_OACTIVE | IFF_RUNNING)) != IFF_RUNNING) return; idx = sc->ste_cdata.ste_tx_prod; @@ -1471,8 +1475,6 @@ ste_start(struct ifnet *ifp) ifp->if_timer = 5; sc->ste_cdata.ste_tx_prod = idx; } - - return; } static void @@ -1492,9 +1494,7 @@ ste_watchdog(struct ifnet *ifp) ste_init(sc); if (!ifq_is_empty(&ifp->if_snd)) - ste_start(ifp); - - return; + if_devstart(ifp); } static void diff --git a/sys/dev/netif/stge/if_stge.c b/sys/dev/netif/stge/if_stge.c index 7ece21f54d..e788306a97 100644 --- a/sys/dev/netif/stge/if_stge.c +++ b/sys/dev/netif/stge/if_stge.c @@ -1,6 +1,6 @@ /* $NetBSD: if_stge.c,v 1.32 2005/12/11 12:22:49 christos Exp $ */ /* $FreeBSD: src/sys/dev/stge/if_stge.c,v 1.2 2006/08/12 01:21:36 yongari Exp $ */ -/* $DragonFly: src/sys/dev/netif/stge/if_stge.c,v 1.4 2008/03/10 12:59:52 sephe Exp $ */ +/* $DragonFly: src/sys/dev/netif/stge/if_stge.c,v 1.5 2008/05/14 11:59:22 sephe Exp $ */ /*- * Copyright (c) 2001 The NetBSD Foundation, Inc. @@ -49,6 +49,7 @@ #include #include #include +#include #include #include #include @@ -806,6 +807,9 @@ stge_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->sc_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + fail: if (error != 0) stge_detach(dev); @@ -1324,9 +1328,10 @@ stge_start(struct ifnet *ifp) * for the NIC to drain the ring. */ if (stge_encap(sc, &m_head)) { - if (m_head == NULL) - break; - ifp->if_flags |= IFF_OACTIVE; + if (m_head != NULL) { + m_freem(m_head); + ifp->if_flags |= IFF_OACTIVE; + } break; } @@ -1568,7 +1573,7 @@ force_init: /* Try to get more packets going. */ if (!ifq_is_empty(&ifp->if_snd)) - ifp->if_start(ifp); + if_devstart(ifp); } /* @@ -1855,7 +1860,7 @@ stge_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) } if (!ifq_is_empty(&ifp->if_snd)) - ifp->if_start(ifp); + if_devstart(ifp); } } #endif /* DEVICE_POLLING */ diff --git a/sys/dev/netif/ti/if_ti.c b/sys/dev/netif/ti/if_ti.c index 0c5e740520..5d46cc8bfd 100644 --- a/sys/dev/netif/ti/if_ti.c +++ b/sys/dev/netif/ti/if_ti.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/pci/if_ti.c,v 1.25.2.14 2002/02/15 04:20:20 silby Exp $ - * $DragonFly: src/sys/dev/netif/ti/if_ti.c,v 1.50 2008/03/10 12:59:52 sephe Exp $ + * $DragonFly: src/sys/dev/netif/ti/if_ti.c,v 1.51 2008/05/14 11:59:22 sephe Exp $ */ /* @@ -91,6 +91,7 @@ #include #include #include +#include #include #include @@ -1571,8 +1572,11 @@ ti_attach(device_t dev) ether_ifdetach(ifp); goto fail; } - return 0; + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->ti_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + + return 0; fail: ti_detach(dev); return(error); @@ -1793,7 +1797,7 @@ ti_intr(void *xsc) CSR_WRITE_4(sc, TI_MB_HOSTINTR, 0); if ((ifp->if_flags & IFF_RUNNING) && !ifq_is_empty(&ifp->if_snd)) - ti_start(ifp); + if_devstart(ifp); } static void @@ -1916,7 +1920,7 @@ ti_start(struct ifnet *ifp) need_trans = 0; while(sc->ti_cdata.ti_tx_chain[prodidx] == NULL) { - m_head = ifq_poll(&ifp->if_snd); + m_head = ifq_dequeue(&ifp->if_snd, NULL); if (m_head == NULL) break; @@ -1933,6 +1937,7 @@ ti_start(struct ifnet *ifp) if ((TI_TX_RING_CNT - sc->ti_txcnt) < m_head->m_pkthdr.csum_data + 16) { ifp->if_flags |= IFF_OACTIVE; + ifq_prepend(&ifp->if_snd, m_head); break; } } @@ -1944,9 +1949,9 @@ ti_start(struct ifnet *ifp) */ if (ti_encap(sc, m_head, &prodidx)) { ifp->if_flags |= IFF_OACTIVE; + ifq_prepend(&ifp->if_snd, m_head); break; } - ifq_dequeue(&ifp->if_snd, m_head); need_trans = 1; ETHER_BPF_MTAP(ifp, m_head); @@ -2249,7 +2254,7 @@ ti_watchdog(struct ifnet *ifp) ifp->if_oerrors++; if (!ifq_is_empty(&ifp->if_snd)) - ifp->if_start(ifp); + if_devstart(ifp); } /* diff --git a/sys/dev/netif/tl/if_tl.c b/sys/dev/netif/tl/if_tl.c index d12ecb0442..afe4cb1827 100644 --- a/sys/dev/netif/tl/if_tl.c +++ b/sys/dev/netif/tl/if_tl.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/pci/if_tl.c,v 1.51.2.5 2001/12/16 15:46:08 luigi Exp $ - * $DragonFly: src/sys/dev/netif/tl/if_tl.c,v 1.38 2008/01/05 14:02:37 swildner Exp $ + * $DragonFly: src/sys/dev/netif/tl/if_tl.c,v 1.39 2008/05/14 11:59:22 sephe Exp $ */ /* @@ -190,6 +190,7 @@ #include #include #include +#include #include #include @@ -1247,6 +1248,9 @@ tl_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->tl_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return(0); fail: @@ -1665,9 +1669,7 @@ tl_intr(void *xsc) } if (!ifq_is_empty(&ifp->if_snd)) - tl_start(ifp); - - return; + if_devstart(ifp); } static diff --git a/sys/dev/netif/tx/if_tx.c b/sys/dev/netif/tx/if_tx.c index 386dc5fddc..8b05daf4bf 100644 --- a/sys/dev/netif/tx/if_tx.c +++ b/sys/dev/netif/tx/if_tx.c @@ -24,7 +24,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/dev/tx/if_tx.c,v 1.61.2.1 2002/10/29 01:43:49 semenu Exp $ - * $DragonFly: src/sys/dev/netif/tx/if_tx.c,v 1.41 2007/05/17 08:19:02 swildner Exp $ + * $DragonFly: src/sys/dev/netif/tx/if_tx.c,v 1.42 2008/05/14 11:59:22 sephe Exp $ */ /* @@ -51,6 +51,7 @@ #include #include #include +#include #include #include @@ -280,6 +281,9 @@ epic_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return(0); fail: @@ -681,7 +685,7 @@ epic_intr(void *arg) if (status & (INTSTAT_TXC|INTSTAT_TCC|INTSTAT_TQE)) { epic_tx_done(sc); if (!ifq_is_empty(&sc->sc_if.if_snd)) - epic_ifstart(&sc->sc_if); + if_devstart(&sc->sc_if); } /* Check for rare errors */ @@ -783,7 +787,7 @@ epic_ifwatchdog(struct ifnet *ifp) /* Start output */ if (!ifq_is_empty(&ifp->if_snd)) - epic_ifstart(ifp); + if_devstart(ifp); } /* diff --git a/sys/dev/netif/txp/if_txp.c b/sys/dev/netif/txp/if_txp.c index f5a6b4b046..edb507396d 100644 --- a/sys/dev/netif/txp/if_txp.c +++ b/sys/dev/netif/txp/if_txp.c @@ -1,6 +1,6 @@ /* $OpenBSD: if_txp.c,v 1.48 2001/06/27 06:34:50 kjc Exp $ */ /* $FreeBSD: src/sys/dev/txp/if_txp.c,v 1.4.2.4 2001/12/14 19:50:43 jlemon Exp $ */ -/* $DragonFly: src/sys/dev/netif/txp/if_txp.c,v 1.47 2008/03/10 12:59:52 sephe Exp $ */ +/* $DragonFly: src/sys/dev/netif/txp/if_txp.c,v 1.48 2008/05/14 11:59:22 sephe Exp $ */ /* * Copyright (c) 2001 @@ -51,6 +51,7 @@ #include #include #include +#include #include #include @@ -320,6 +321,9 @@ txp_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->sc_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return(0); fail: @@ -642,9 +646,7 @@ txp_intr(void *vsc) /* unmask all interrupts */ WRITE_REG(sc, TXP_IMR, TXP_INT_A2H_3); - txp_start(&sc->sc_arpcom.ac_if); - - return; + if_devstart(&sc->sc_arpcom.ac_if); } static void @@ -722,12 +724,10 @@ txp_rx_reclaim(struct txp_softc *sc, struct txp_rx_ring *r) m->m_pkthdr.csum_data = 0xffff; } - lwkt_serialize_enter(ifp->if_serializer); if (rxd->rx_stat & RX_STAT_VLAN) VLAN_INPUT_TAG(m, htons(rxd->rx_vlan >> 16)); else ifp->if_input(ifp, m); - lwkt_serialize_exit(ifp->if_serializer); next: @@ -1190,7 +1190,7 @@ txp_start(struct ifnet *ifp) struct txp_tx_ring *r = &sc->sc_txhir; struct txp_tx_desc *txd; struct txp_frag_desc *fxd; - struct mbuf *m, *m0; + struct mbuf *m, *m0, *m_defragged; struct txp_swdesc *sd; u_int32_t firstprod, firstcnt, prod, cnt; @@ -1201,21 +1201,48 @@ txp_start(struct ifnet *ifp) cnt = r->r_cnt; while (1) { - m = ifq_poll(&ifp->if_snd); - if (m == NULL) - break; + int frag; firstprod = prod; firstcnt = cnt; - sd = sc->sc_txd + prod; - sd->sd_mbuf = m; - if ((TX_ENTRIES - cnt) < 4) goto oactive; - txd = r->r_desc + prod; + m_defragged = NULL; + m = ifq_dequeue(&ifp->if_snd, NULL); + if (m == NULL) + break; +again: + frag = 1; /* Extra desc */ + for (m0 = m; m0 != NULL; m0 = m0->m_next) + ++frag; + if ((cnt + frag) >= (TX_ENTRIES - 4)) { + if (m_defragged != NULL) { + /* + * Even after defragmentation, there + * are still too many fragments, so + * drop this packet. + */ + m_freem(m); + goto oactive; + } + m_defragged = m_defrag(m, MB_DONTWAIT); + if (m_defragged == NULL) { + m_freem(m); + continue; + } + m = m_defragged; + + /* Recount # of fragments */ + goto again; + } + + sd = sc->sc_txd + prod; + sd->sd_mbuf = m; + + txd = r->r_desc + prod; txd->tx_flags = TX_FLAGS_TYPE_DATA; txd->tx_numdesc = 0; txd->tx_addrlo = 0; @@ -1226,8 +1253,8 @@ txp_start(struct ifnet *ifp) if (++prod == TX_ENTRIES) prod = 0; - if (++cnt >= (TX_ENTRIES - 4)) - goto oactive; + ++cnt; + KASSERT(cnt < (TX_ENTRIES - 4), ("too many frag\n")); if (m->m_flags & M_VLANTAG) { txd->tx_pflags = TX_PFLAGS_VLAN | @@ -1249,8 +1276,9 @@ txp_start(struct ifnet *ifp) for (m0 = m; m0 != NULL; m0 = m0->m_next) { if (m0->m_len == 0) continue; - if (++cnt >= (TX_ENTRIES - 4)) - goto oactive; + + ++cnt; + KASSERT(cnt < (TX_ENTRIES - 4), ("too many frag\n")); txd->tx_numdesc++; @@ -1266,12 +1294,10 @@ txp_start(struct ifnet *ifp) prod = 0; } else fxd++; - } ifp->if_timer = 5; - ifq_dequeue(&ifp->if_snd, m); ETHER_BPF_MTAP(ifp, m); WRITE_REG(sc, r->r_reg, TXP_IDX2OFFSET(prod)); } diff --git a/sys/dev/netif/ural/if_ural.c b/sys/dev/netif/ural/if_ural.c index d437d2b1cc..f2e9d7e7d9 100644 --- a/sys/dev/netif/ural/if_ural.c +++ b/sys/dev/netif/ural/if_ural.c @@ -1,5 +1,5 @@ /* $FreeBSD: src/sys/dev/usb/if_ural.c,v 1.10.2.8 2006/07/08 07:48:43 maxim Exp $ */ -/* $DragonFly: src/sys/dev/netif/ural/if_ural.c,v 1.25 2008/01/15 09:01:13 sephe Exp $ */ +/* $DragonFly: src/sys/dev/netif/ural/if_ural.c,v 1.26 2008/05/14 11:59:22 sephe Exp $ */ /*- * Copyright (c) 2005, 2006 @@ -1369,8 +1369,10 @@ ural_start(struct ifnet *ifp) ASSERT_SERIALIZED(ifp->if_serializer); - if (sc->sc_stopped) + if (sc->sc_stopped) { + ifq_purge(&ifp->if_snd); return; + } crit_enter(); @@ -1403,17 +1405,19 @@ ural_start(struct ifnet *ifp) } else { struct ether_header *eh; - if (ic->ic_state != IEEE80211_S_RUN) - break; - m0 = ifq_poll(&ifp->if_snd); - if (m0 == NULL) + if (ic->ic_state != IEEE80211_S_RUN) { + ifq_purge(&ifp->if_snd); break; + } + if (sc->tx_queued >= RAL_TX_LIST_COUNT) { ifp->if_flags |= IFF_OACTIVE; break; } - ifq_dequeue(&ifp->if_snd, m0); + m0 = ifq_dequeue(&ifp->if_snd, NULL); + if (m0 == NULL) + break; if (m0->m_len < sizeof (struct ether_header)) { m0 = m_pullup(m0, sizeof (struct ether_header)); diff --git a/sys/dev/netif/vge/if_vge.c b/sys/dev/netif/vge/if_vge.c index 7420468216..8662788797 100644 --- a/sys/dev/netif/vge/if_vge.c +++ b/sys/dev/netif/vge/if_vge.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/vge/if_vge.c,v 1.24 2006/02/14 12:44:56 glebius Exp $ - * $DragonFly: src/sys/dev/netif/vge/if_vge.c,v 1.7 2008/03/10 12:59:52 sephe Exp $ + * $DragonFly: src/sys/dev/netif/vge/if_vge.c,v 1.8 2008/05/14 11:59:22 sephe Exp $ */ /* @@ -95,6 +95,7 @@ #include #include #include +#include #include #include @@ -1033,6 +1034,9 @@ vge_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->vge_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return 0; fail: vge_detach(dev); @@ -1470,7 +1474,7 @@ vge_tick(struct vge_softc *sc) IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) { sc->vge_link = 1; if (!ifq_is_empty(&ifp->if_snd)) - ifp->if_start(ifp); + if_devstart(ifp); } } } @@ -1496,7 +1500,7 @@ vge_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) vge_txeof(sc); if (!ifq_is_empty(&ifp->if_snd)) - ifp->if_start(ifp); + if_devstart(ifp); /* XXX copy & paste from vge_intr */ if (cmd == POLL_AND_CHECK_STATUS) { @@ -1574,7 +1578,7 @@ vge_intr(void *arg) CSR_WRITE_1(sc, VGE_CRS3, VGE_CR3_INT_GMSK); if (!ifq_is_empty(&ifp->if_snd)) - ifp->if_start(ifp); + if_devstart(ifp); } static int @@ -1667,10 +1671,12 @@ vge_start(struct ifnet *ifp) ASSERT_SERIALIZED(ifp->if_serializer); - if (!sc->vge_link || (ifp->if_flags & IFF_OACTIVE)) + if (!sc->vge_link) { + ifq_purge(&ifp->if_snd); return; + } - if (ifq_is_empty(&ifp->if_snd)) + if ((ifp->if_flags & (IFF_OACTIVE | IFF_RUNNING)) != IFF_RUNNING) return; idx = sc->vge_ldata.vge_tx_prodidx; @@ -1680,16 +1686,14 @@ vge_start(struct ifnet *ifp) pidx = VGE_TX_DESC_CNT - 1; while (sc->vge_ldata.vge_tx_mbuf[idx] == NULL) { - m_head = ifq_poll(&ifp->if_snd); - if (m_head == NULL) - break; - if (sc->vge_ldata.vge_tx_free <= 2) { ifp->if_flags |= IFF_OACTIVE; break; } - m_head = ifq_dequeue(&ifp->if_snd, m_head); + m_head = ifq_dequeue(&ifp->if_snd, NULL); + if (m_head == NULL) + break; if (vge_encap(sc, m_head, idx)) { /* If vge_encap() failed, it will free m_head for us */ diff --git a/sys/dev/netif/vr/if_vr.c b/sys/dev/netif/vr/if_vr.c index fdc2b08b27..625f1be31d 100644 --- a/sys/dev/netif/vr/if_vr.c +++ b/sys/dev/netif/vr/if_vr.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/pci/if_vr.c,v 1.26.2.13 2003/02/06 04:46:20 silby Exp $ - * $DragonFly: src/sys/dev/netif/vr/if_vr.c,v 1.46 2007/09/09 06:21:23 sephe Exp $ + * $DragonFly: src/sys/dev/netif/vr/if_vr.c,v 1.47 2008/05/14 11:59:22 sephe Exp $ */ /* @@ -73,6 +73,7 @@ #include #include #include +#include #include #include @@ -791,12 +792,15 @@ vr_attach(device_t dev) error = bus_setup_intr(dev, sc->vr_irq, INTR_NETSAFE, vr_intr, sc, &sc->vr_intrhand, ifp->if_serializer); - if (error) { device_printf(dev, "couldn't set up irq\n"); ether_ifdetach(ifp); goto fail; } + + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->vr_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return 0; fail: @@ -1250,7 +1254,7 @@ vr_intr(void *arg) CSR_WRITE_2(sc, VR_IMR, VR_INTRS); if (!ifq_is_empty(&ifp->if_snd)) - vr_start(ifp); + if_devstart(ifp); } /* @@ -1317,7 +1321,7 @@ vr_start(struct ifnet *ifp) struct vr_chain *tx_chain; int cur_tx_idx, start_tx_idx, prev_tx_idx; - if (ifp->if_flags & IFF_OACTIVE) + if ((ifp->if_flags & (IFF_OACTIVE | IFF_RUNNING)) != IFF_RUNNING) return; sc = ifp->if_softc; @@ -1333,11 +1337,11 @@ vr_start(struct ifnet *ifp) return; } - while(tx_chain[cd->vr_tx_free_idx].vr_buf == NULL) { + while (tx_chain[cd->vr_tx_free_idx].vr_buf == NULL) { struct mbuf *m_head; struct vr_chain *cur_tx; - m_head = ifq_poll(&ifp->if_snd); + m_head = ifq_dequeue(&ifp->if_snd, NULL); if (m_head == NULL) break; @@ -1352,8 +1356,6 @@ vr_start(struct ifnet *ifp) break; } - ifq_dequeue(&ifp->if_snd, m_head); - /* XXX */ if (cur_tx_idx != start_tx_idx) VR_TXOWN(cur_tx) = VR_TXSTAT_OWN; @@ -1583,7 +1585,7 @@ vr_watchdog(struct ifnet *ifp) vr_init(sc); if (!ifq_is_empty(&ifp->if_snd)) - vr_start(ifp); + if_devstart(ifp); } /* diff --git a/sys/dev/netif/vx/if_vx.c b/sys/dev/netif/vx/if_vx.c index 16e367a792..3d9d79066a 100644 --- a/sys/dev/netif/vx/if_vx.c +++ b/sys/dev/netif/vx/if_vx.c @@ -28,7 +28,7 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/vx/if_vx.c,v 1.25.2.6 2002/02/13 00:43:10 dillon Exp $ - * $DragonFly: src/sys/dev/netif/vx/if_vx.c,v 1.28 2006/12/22 23:26:22 swildner Exp $ + * $DragonFly: src/sys/dev/netif/vx/if_vx.c,v 1.29 2008/05/14 11:59:22 sephe Exp $ * */ @@ -229,7 +229,7 @@ vxinit(void *xsc) ifp->if_flags &= ~IFF_OACTIVE; /* Attempt to start output, if any. */ - vxstart(ifp); + if_devstart(ifp); } static void @@ -394,14 +394,15 @@ vxstart(struct ifnet *ifp) startagain: /* Sneak a peek at the next packet */ - m0 = ifq_poll(&ifp->if_snd); + m0 = ifq_dequeue(&ifp->if_snd, NULL); if (m0 == NULL) return; + /* We need to use m->m_pkthdr.len, so require the header */ - M_ASSERTPKTHDR(m0); - len = m0->m_pkthdr.len; + M_ASSERTPKTHDR(m0); + len = m0->m_pkthdr.len; - pad = (4 - len) & 3; + pad = (4 - len) & 3; /* * The 3c509 automatically pads short packets to minimum ethernet length, @@ -411,7 +412,6 @@ startagain: if (len + pad > ETHER_MAX_LEN) { /* packet is obviously too large: toss it */ ++ifp->if_oerrors; - ifq_dequeue(&ifp->if_snd, m0); m_freem(m0); goto readcheck; } @@ -422,11 +422,11 @@ startagain: if (CSR_READ_2(sc, VX_W1_FREE_TX) < len + pad + 4) { /* make sure */ ifp->if_flags |= IFF_OACTIVE; ifp->if_timer = 1; + ifq_prepend(&ifp->if_snd, m0); return; } } CSR_WRITE_2(sc, VX_COMMAND, SET_TX_AVAIL_THRESH | (8188 >> 2)); - ifq_dequeue(&ifp->if_snd, m0); VX_BUSY_WAIT; CSR_WRITE_2(sc, VX_COMMAND, SET_TX_START_THRESH | @@ -604,7 +604,7 @@ vxintr(void *voidsc) if (status & S_TX_AVAIL) { ifp->if_timer = 0; ifp->if_flags &= ~IFF_OACTIVE; - vxstart(ifp); + if_devstart(ifp); } if (status & S_CARD_FAILURE) { if_printf(ifp, "adapter failure (%x)\n", status); @@ -615,7 +615,7 @@ vxintr(void *voidsc) if (status & S_TX_COMPLETE) { ifp->if_timer = 0; vxtxstat(sc); - vxstart(ifp); + if_devstart(ifp); } } /* no more interrupts */ @@ -893,7 +893,7 @@ vxwatchdog(struct ifnet *ifp) if (ifp->if_flags & IFF_DEBUG) if_printf(ifp, "device timeout\n"); ifp->if_flags &= ~IFF_OACTIVE; - vxstart(ifp); + if_devstart(ifp); vxintr(sc); } diff --git a/sys/dev/netif/vx/if_vx_eisa.c b/sys/dev/netif/vx/if_vx_eisa.c index 6ecc884dc9..92ee0adec7 100644 --- a/sys/dev/netif/vx/if_vx_eisa.c +++ b/sys/dev/netif/vx/if_vx_eisa.c @@ -27,7 +27,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/dev/vx/if_vx_eisa.c,v 1.14 2000/01/29 14:50:31 peter Exp $ - * $DragonFly: src/sys/dev/netif/vx/if_vx_eisa.c,v 1.15 2006/10/25 20:55:59 dillon Exp $ + * $DragonFly: src/sys/dev/netif/vx/if_vx_eisa.c,v 1.16 2008/05/14 11:59:22 sephe Exp $ */ #include @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -103,12 +104,11 @@ vx_eisa_probe(device_t dev) static int vx_eisa_attach(device_t dev) { - struct vx_softc *sc; + struct vx_softc *sc = device_get_softc(dev); + struct ifnet *ifp = &sc->arpcom.ac_if; struct resource *eisa_io = NULL; int rid; - sc = device_get_softc(dev); - /* * The addresses are sorted in increasing order * so we know the port to pass to the core ep @@ -144,11 +144,15 @@ vx_eisa_attach(device_t dev) if (bus_setup_intr(dev, sc->vx_irq, INTR_NETSAFE, vxintr, sc, &sc->vx_intrhand, - sc->arpcom.ac_if.if_serializer) + ifp->if_serializer) ) { ether_ifdetach(&sc->arpcom.ac_if); goto bad; } + + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->vx_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return 0; bad: diff --git a/sys/dev/netif/vx/if_vx_pci.c b/sys/dev/netif/vx/if_vx_pci.c index 0558e66090..df5d572321 100644 --- a/sys/dev/netif/vx/if_vx_pci.c +++ b/sys/dev/netif/vx/if_vx_pci.c @@ -27,7 +27,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/dev/vx/if_vx_pci.c,v 1.21 2000/05/28 15:59:52 peter Exp $ - * $DragonFly: src/sys/dev/netif/vx/if_vx_pci.c,v 1.15 2006/12/22 23:26:22 swildner Exp $ + * $DragonFly: src/sys/dev/netif/vx/if_vx_pci.c,v 1.16 2008/05/14 11:59:22 sephe Exp $ */ #include @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -132,11 +133,10 @@ vx_pci_probe(device_t dev) static int vx_pci_attach(device_t dev) { - struct vx_softc *sc; + struct vx_softc *sc = device_get_softc(dev); + struct ifnet *ifp = &sc->arpcom.ac_if; int rid; - sc = device_get_softc(dev); - rid = PCIR_MAPS; sc->vx_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid, RF_ACTIVE); @@ -174,12 +174,15 @@ vx_pci_attach(device_t dev) if (bus_setup_intr(dev, sc->vx_irq, INTR_NETSAFE, vxintr, sc, &sc->vx_intrhand, - sc->arpcom.ac_if.if_serializer) + ifp->if_serializer) ) { ether_ifdetach(&sc->arpcom.ac_if); goto bad; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->vx_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return(0); bad: diff --git a/sys/dev/netif/wb/if_wb.c b/sys/dev/netif/wb/if_wb.c index 21f88a5746..3a35f2e2d5 100644 --- a/sys/dev/netif/wb/if_wb.c +++ b/sys/dev/netif/wb/if_wb.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/pci/if_wb.c,v 1.26.2.6 2003/03/05 18:42:34 njl Exp $ - * $DragonFly: src/sys/dev/netif/wb/if_wb.c,v 1.40 2008/01/05 14:02:37 swildner Exp $ + * $DragonFly: src/sys/dev/netif/wb/if_wb.c,v 1.41 2008/05/14 11:59:22 sephe Exp $ */ /* @@ -96,6 +96,7 @@ #include #include #include +#include #include #include @@ -819,6 +820,9 @@ wb_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->wb_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return(0); fail: @@ -1180,7 +1184,7 @@ wb_intr(void *arg) CSR_WRITE_4(sc, WB_IMR, WB_INTRS); if (!ifq_is_empty(&ifp->if_snd)) - wb_start(ifp); + if_devstart(ifp); } static void @@ -1566,7 +1570,7 @@ wb_watchdog(struct ifnet *ifp) wb_init(sc); if (!ifq_is_empty(&ifp->if_snd)) - wb_start(ifp); + if_devstart(ifp); } /* diff --git a/sys/dev/netif/wi/if_wi.c b/sys/dev/netif/wi/if_wi.c index 41935804b2..34a966db24 100644 --- a/sys/dev/netif/wi/if_wi.c +++ b/sys/dev/netif/wi/if_wi.c @@ -32,7 +32,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/dev/wi/if_wi.c,v 1.180.2.7 2005/10/05 13:16:29 avatar Exp $ - * $DragonFly: src/sys/dev/netif/wi/if_wi.c,v 1.40 2007/04/08 09:43:57 sephe Exp $ + * $DragonFly: src/sys/dev/netif/wi/if_wi.c,v 1.41 2008/05/14 11:59:22 sephe Exp $ */ /* @@ -85,6 +85,7 @@ #include #include #include +#include #include @@ -500,6 +501,9 @@ wi_attach(device_t dev) goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + if (bootverbose) ieee80211_announce(ic); @@ -575,9 +579,8 @@ wi_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) } if ((ifp->if_flags & IFF_OACTIVE) == 0 && - (sc->sc_flags & WI_FLAGS_OUTRANGE) == 0 && !ifq_is_empty(&ifp->if_snd)) { - wi_start(ifp); - } + (sc->sc_flags & WI_FLAGS_OUTRANGE) == 0) + ifp->if_start(ifp); break; } } @@ -609,14 +612,11 @@ wi_intr(void *arg) if (status & WI_EV_INFO) wi_info_intr(sc); if ((ifp->if_flags & IFF_OACTIVE) == 0 && - (sc->sc_flags & WI_FLAGS_OUTRANGE) == 0 && - !ifq_is_empty(&ifp->if_snd)) - wi_start(ifp); + (sc->sc_flags & WI_FLAGS_OUTRANGE) == 0) + ifp->if_start(ifp); /* Re-enable interrupts. */ CSR_WRITE_2(sc, WI_INT_EN, WI_INTRS); - - return; } void @@ -850,8 +850,11 @@ wi_start(struct ifnet *ifp) struct wi_frame frmhdr; int cur, fid, off, error; - if (sc->wi_gone || (sc->sc_flags & WI_FLAGS_OUTRANGE)) + if (sc->wi_gone || (sc->sc_flags & WI_FLAGS_OUTRANGE)) { + ieee80211_drain_mgtq(&ic->ic_mgtq); + ifq_purge(&ifp->if_snd); return; + } memset(&frmhdr, 0, sizeof(frmhdr)); cur = sc->sc_txnext; @@ -882,17 +885,20 @@ wi_start(struct ifnet *ifp) } else { struct ether_header *eh; - if (ic->ic_state != IEEE80211_S_RUN) - break; - m0 = ifq_poll(&ifp->if_snd); - if (m0 == NULL) + if (ic->ic_state != IEEE80211_S_RUN) { + ifq_purge(&ifp->if_snd); break; + } + if (sc->sc_txd[cur].d_len != 0) { ifp->if_flags |= IFF_OACTIVE; break; } - ifq_dequeue(&ifp->if_snd, m0); + m0 = ifq_dequeue(&ifp->if_snd, NULL); + if (m0 == NULL) + break; + if (m0->m_len < sizeof(struct ether_header)) { m0 = m_pullup(m0, sizeof(struct ether_header)); if (m0 == NULL) { diff --git a/sys/dev/netif/wl/if_wl.c b/sys/dev/netif/wl/if_wl.c index aca643ae72..740a17404a 100644 --- a/sys/dev/netif/wl/if_wl.c +++ b/sys/dev/netif/wl/if_wl.c @@ -1,5 +1,5 @@ /* $FreeBSD: src/sys/i386/isa/if_wl.c,v 1.27.2.2 2000/07/17 21:24:32 archie Exp $ */ -/* $DragonFly: src/sys/dev/netif/wl/if_wl.c,v 1.30 2006/12/22 23:26:22 swildner Exp $ */ +/* $DragonFly: src/sys/dev/netif/wl/if_wl.c,v 1.31 2008/05/14 11:59:22 sephe Exp $ */ /* * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -205,6 +205,7 @@ WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include #include #include +#include #include #include @@ -560,6 +561,9 @@ wlattach(device_t dev) return error; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->res_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + if (bootverbose) wldump(sc); return 0; @@ -803,7 +807,7 @@ wlinit(void *xsc) sc->tbusy = 0; callout_stop(&sc->watchdog_ch); - wlstart(ifp); + if_devstart(ifp); } else { if_printf(ifp, "init(): trouble resetting board.\n"); } @@ -1645,7 +1649,7 @@ wlintr(void *arg) sc->tbusy = 0; callout_stop(&sc->watchdog_ch); ifp->if_flags &= ~IFF_OACTIVE; - wlstart(ifp); + if_devstart(ifp); } } } diff --git a/sys/dev/netif/xe/if_xe.c b/sys/dev/netif/xe/if_xe.c index b0647771d7..ac35d64d17 100644 --- a/sys/dev/netif/xe/if_xe.c +++ b/sys/dev/netif/xe/if_xe.c @@ -25,7 +25,7 @@ * * $Id: if_xe.c,v 1.20 1999/06/13 19:17:40 scott Exp $ * $FreeBSD: src/sys/dev/xe/if_xe.c,v 1.39 2003/10/14 22:51:35 rsm Exp $ - * $DragonFly: src/sys/dev/netif/xe/if_xe.c,v 1.34 2006/12/22 23:26:22 swildner Exp $ + * $DragonFly: src/sys/dev/netif/xe/if_xe.c,v 1.35 2008/05/14 11:59:23 sephe Exp $ */ /* @@ -110,6 +110,7 @@ #include #include #include +#include #include #include @@ -330,14 +331,17 @@ xe_attach (device_t dev) err = bus_setup_intr(dev, scp->irq_res, INTR_NETSAFE, xe_intr, scp, &scp->intrhand, - scp->arpcom.ac_if.if_serializer); + scp->ifp->if_serializer); if (err) { device_printf(dev, "Setup intr failed\n"); - ether_ifdetach(&scp->arpcom.ac_if); + ether_ifdetach(scp->ifp); xe_deactivate(dev); return err; } + scp->ifp->if_cpuid = ithread_cpuid(rman_get_start(scp->irq_res)); + KKASSERT(scp->ifp->if_cpuid >= 0 && scp->ifp->if_cpuid < ncpus); + /* Done */ return 0; } @@ -482,15 +486,15 @@ xe_start(struct ifnet *ifp) { * Loop while there are packets to be sent, and space to send them. */ while (1) { - mbp = ifq_poll(&ifp->if_snd); /* Suck a packet off the send queue */ - + /* Suck a packet off the send queue */ + mbp = ifq_dequeue(&ifp->if_snd, NULL); if (mbp == NULL) { /* - * We are using the !OACTIVE flag to indicate to the outside world that - * we can accept an additional packet rather than that the transmitter - * is _actually_ active. Indeed, the transmitter may be active, but if - * we haven't filled all the buffers with data then we still want to - * accept more. + * We are using the !OACTIVE flag to indicate to the outside + * world that we can accept an additional packet rather than + * that the transmitter is _actually_ active. Indeed, the + * transmitter may be active, but if we haven't filled all + * the buffers with data then we still want to accept more. */ ifp->if_flags &= ~IFF_OACTIVE; return; @@ -498,13 +502,13 @@ xe_start(struct ifnet *ifp) { if (xe_pio_write_packet(scp, mbp) != 0) { ifp->if_flags |= IFF_OACTIVE; + ifq_prepend(&ifp->if_snd, mbp); return; } - ifq_dequeue(&ifp->if_snd, mbp); BPF_MTAP(ifp, mbp); - ifp->if_timer = 5; /* In case we don't hear from the card again */ + ifp->if_timer = 5; /* In case we don't hear from the card again */ scp->tx_queued++; m_freem(mbp); @@ -1175,7 +1179,7 @@ xe_setmedia_serialized(void *xscp) /* Restart output? */ xe_enable_intr(scp); scp->ifp->if_flags &= ~IFF_OACTIVE; - xe_start(scp->ifp); + if_devstart(scp->ifp); } diff --git a/sys/dev/netif/xl/if_xl.c b/sys/dev/netif/xl/if_xl.c index af68415487..c096c1f785 100644 --- a/sys/dev/netif/xl/if_xl.c +++ b/sys/dev/netif/xl/if_xl.c @@ -30,7 +30,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/pci/if_xl.c,v 1.72.2.28 2003/10/08 06:01:57 murray Exp $ - * $DragonFly: src/sys/dev/netif/xl/if_xl.c,v 1.49 2007/08/14 13:30:35 sephe Exp $ + * $DragonFly: src/sys/dev/netif/xl/if_xl.c,v 1.50 2008/05/14 11:59:23 sephe Exp $ */ /* @@ -112,6 +112,7 @@ #include #include #include +#include #include #include @@ -214,6 +215,7 @@ static void xl_txeoc (struct xl_softc *); static void xl_intr (void *); static void xl_start_body (struct ifnet *, int); static void xl_start (struct ifnet *); +static void xl_start_poll (struct ifnet *); static void xl_start_90xB (struct ifnet *); static int xl_ioctl (struct ifnet *, u_long, caddr_t, struct ucred *); @@ -1605,6 +1607,9 @@ done: goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->xl_irq)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); + return 0; fail: @@ -2334,8 +2339,12 @@ xl_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) switch (cmd) { case POLL_REGISTER: xl_enable_intrs(sc, 0); + if (sc->xl_type != XL_TYPE_905B) + ifp->if_start = xl_start_poll; break; case POLL_DEREGISTER: + if (sc->xl_type != XL_TYPE_905B) + ifp->if_start = xl_start; xl_enable_intrs(sc, XL_INTRS); break; case POLL_ONLY: @@ -2346,12 +2355,8 @@ xl_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) else xl_txeof(sc); - if (!ifq_is_empty(&ifp->if_snd)) { - if (sc->xl_type == XL_TYPE_905B) - xl_start_90xB(ifp); - else - xl_start_body(ifp, 0); - } + if (!ifq_is_empty(&ifp->if_snd)) + if_devstart(ifp); if (cmd == POLL_AND_CHECK_STATUS) { uint16_t status; @@ -2437,9 +2442,7 @@ xl_intr(void *arg) } if (!ifq_is_empty(&ifp->if_snd)) - (*ifp->if_start)(ifp); - - return; + if_devstart(ifp); } static void @@ -2583,6 +2586,12 @@ xl_start(struct ifnet *ifp) xl_start_body(ifp, 1); } +static void +xl_start_poll(struct ifnet *ifp) +{ + xl_start_body(ifp, 0); +} + /* * Main transmit routine. To avoid having to do mbuf copies, we put pointers * to the mbuf data regions directly in the transmit lists. We also save a @@ -2725,7 +2734,7 @@ xl_start_90xB(struct ifnet *ifp) sc = ifp->if_softc; - if (ifp->if_flags & IFF_OACTIVE) + if ((ifp->if_flags & (IFF_OACTIVE | IFF_RUNNING)) != IFF_RUNNING) return; idx = sc->xl_cdata.xl_tx_prod; @@ -3212,7 +3221,7 @@ xl_watchdog(struct ifnet *ifp) xl_init(sc); if (!ifq_is_empty(&ifp->if_snd)) - (*ifp->if_start)(ifp); + if_devstart(ifp); } /* diff --git a/sys/kern/kern_intr.c b/sys/kern/kern_intr.c index 7fb21fd375..ab4acc98d5 100644 --- a/sys/kern/kern_intr.c +++ b/sys/kern/kern_intr.c @@ -24,7 +24,7 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/kern/kern_intr.c,v 1.24.2.1 2001/10/14 20:05:50 luigi Exp $ - * $DragonFly: src/sys/kern/kern_intr.c,v 1.52 2008/03/30 14:40:13 hasso Exp $ + * $DragonFly: src/sys/kern/kern_intr.c,v 1.53 2008/05/14 11:59:23 sephe Exp $ * */ @@ -951,6 +951,19 @@ emergency_intr_timer_callback(systimer_t info, struct intrframe *frame __unused) lwkt_schedule(info->data); } +int +ithread_cpuid(int intr) +{ + const struct intr_info *info; + + KKASSERT(intr >= 0 && intr < MAX_INTS); + info = &intr_info_ary[intr]; + + if (info->i_state == ISTATE_NOTHREAD) + return -1; + return info->i_thread.td_gd->gd_cpuid; +} + /* * Sysctls used by systat and others: hw.intrnames and hw.intrcnt. * The data for this machine dependent, and the declarations are in machine diff --git a/sys/kern/lwkt_serialize.c b/sys/kern/lwkt_serialize.c index e7e251e631..2b4ff0b80e 100644 --- a/sys/kern/lwkt_serialize.c +++ b/sys/kern/lwkt_serialize.c @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sys/kern/lwkt_serialize.c,v 1.15 2008/05/05 12:35:03 sephe Exp $ + * $DragonFly: src/sys/kern/lwkt_serialize.c,v 1.16 2008/05/14 11:59:23 sephe Exp $ */ /* * This API provides a fast locked-bus-cycle-based serializer. It's @@ -95,12 +95,11 @@ KTR_INFO(KTR_SERIALIZER, slz, exit_beg, 11, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE); #define logslz_spinbo(slz, bo1, bo) KTR_LOG(slz_spinbo, slz, bo1, bo) static void lwkt_serialize_sleep(void *info); -#ifdef SMP -static void lwkt_serialize_adaptive_sleep(void *bo); -#endif static void lwkt_serialize_wakeup(void *info); #ifdef SMP +static void lwkt_serialize_adaptive_sleep(void *bo); + static int slz_backoff_limit = 128; SYSCTL_INT(_debug, OID_AUTO, serialize_bolimit, CTLFLAG_RW, &slz_backoff_limit, 0, ""); @@ -119,9 +118,8 @@ void lwkt_serialize_init(lwkt_serialize_t s) { atomic_intr_init(&s->interlock); -#ifdef INVARIANTS s->last_td = (void *)-4; -#endif + s->sleep_cnt = 0; s->tryfail_cnt = 0; s->enter_cnt = 0; @@ -129,7 +127,6 @@ lwkt_serialize_init(lwkt_serialize_t s) } #ifdef SMP - void lwkt_serialize_adaptive_enter(lwkt_serialize_t s) { @@ -139,34 +136,29 @@ lwkt_serialize_adaptive_enter(lwkt_serialize_t s) bo.round = 0; bo.s = s; -#ifdef INVARIANTS - KKASSERT(s->last_td != curthread); -#endif + ASSERT_NOT_SERIALIZED(s); + logslz(enter_beg, s); atomic_intr_cond_enter(&s->interlock, lwkt_serialize_adaptive_sleep, &bo); logslz(enter_end, s); -#ifdef INVARIANTS + s->last_td = curthread; -#endif #ifdef PROFILE_SERIALIZER s->enter_cnt++; #endif } - #endif /* SMP */ void lwkt_serialize_enter(lwkt_serialize_t s) { -#ifdef INVARIANTS - KKASSERT(s->last_td != curthread); -#endif + ASSERT_NOT_SERIALIZED(s); + logslz(enter_beg, s); atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s); logslz(enter_end, s); -#ifdef INVARIANTS + s->last_td = curthread; -#endif #ifdef PROFILE_SERIALIZER s->enter_cnt++; #endif @@ -180,17 +172,14 @@ lwkt_serialize_try(lwkt_serialize_t s) { int error; -#ifdef INVARIANTS - KKASSERT(s->last_td != curthread); -#endif + ASSERT_NOT_SERIALIZED(s); + #ifdef PROFILE_SERIALIZER s->try_cnt++; #endif logslz(try, s); if ((error = atomic_intr_cond_try(&s->interlock)) == 0) { -#ifdef INVARIANTS s->last_td = curthread; -#endif logslz(tryok, s); return(1); } @@ -204,10 +193,9 @@ lwkt_serialize_try(lwkt_serialize_t s) void lwkt_serialize_exit(lwkt_serialize_t s) { -#ifdef INVARIANTS - KKASSERT(s->last_td == curthread); + ASSERT_SERIALIZED(s); s->last_td = (void *)-2; -#endif + logslz(exit_beg, s); atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s); logslz(exit_end, s); @@ -241,18 +229,17 @@ lwkt_serialize_handler_call(lwkt_serialize_t s, void (*func)(void *, void *), logslz(enter_beg, s); atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s); logslz(enter_end, s); -#ifdef INVARIANTS + s->last_td = curthread; -#endif #ifdef PROFILE_SERIALIZER s->enter_cnt++; #endif if (atomic_intr_handler_is_enabled(&s->interlock) == 0) func(arg, frame); -#ifdef INVARIANTS - KKASSERT(s->last_td == curthread); + + ASSERT_SERIALIZED(s); s->last_td = (void *)-2; -#endif + logslz(exit_beg, s); atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s); logslz(exit_end, s); @@ -277,15 +264,14 @@ lwkt_serialize_handler_try(lwkt_serialize_t s, void (*func)(void *, void *), #endif logslz(try, s); if (atomic_intr_cond_try(&s->interlock) == 0) { -#ifdef INVARIANTS s->last_td = curthread; -#endif logslz(tryok, s); + func(arg, frame); -#ifdef INVARIANTS - KKASSERT(s->last_td == curthread); + + ASSERT_SERIALIZED(s); s->last_td = (void *)-2; -#endif + logslz(exit_beg, s); atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s); logslz(exit_end, s); diff --git a/sys/net/altq/altq_cbq.c b/sys/net/altq/altq_cbq.c index 60cd22f4d9..c24611b159 100644 --- a/sys/net/altq/altq_cbq.c +++ b/sys/net/altq/altq_cbq.c @@ -1,5 +1,5 @@ /* $KAME: altq_cbq.c,v 1.20 2004/04/17 10:54:48 kjc Exp $ */ -/* $DragonFly: src/sys/net/altq/altq_cbq.c,v 1.6 2006/12/22 23:44:55 swildner Exp $ */ +/* $DragonFly: src/sys/net/altq/altq_cbq.c,v 1.7 2008/05/14 11:59:23 sephe Exp $ */ /* * Copyright (c) Sun Microsystems, Inc. 1993-1998 All rights reserved. @@ -201,18 +201,10 @@ get_class_stats(class_stats_t *statsp, struct rm_class *cl) } int -cbq_pfattach(struct pf_altq *a) +cbq_pfattach(struct pf_altq *a, struct ifaltq *ifq) { - struct ifnet *ifp; - int error; - - if ((ifp = ifunit(a->ifname)) == NULL || a->altq_disc == NULL) - return (EINVAL); - crit_enter(); - error = altq_attach(&ifp->if_snd, ALTQT_CBQ, a->altq_disc, + return altq_attach(ifq, ALTQT_CBQ, a->altq_disc, cbq_enqueue, cbq_dequeue, cbq_request, NULL, NULL); - crit_exit(); - return (error); } int @@ -261,19 +253,15 @@ cbq_remove_altq(struct pf_altq *a) return (0); } -int -cbq_add_queue(struct pf_altq *a) +static int +cbq_add_queue_locked(struct pf_altq *a, cbq_state_t *cbqp) { struct rm_class *borrow, *parent; - cbq_state_t *cbqp; struct rm_class *cl; struct cbq_opts *opts; int i; - if ((cbqp = a->altq_disc) == NULL) - return (EINVAL); - if (a->qid == 0) - return (EINVAL); + KKASSERT(a->qid != 0); /* * find a free slot in the class table. if the slot matching @@ -372,14 +360,32 @@ cbq_add_queue(struct pf_altq *a) } int -cbq_remove_queue(struct pf_altq *a) +cbq_add_queue(struct pf_altq *a) { - struct rm_class *cl; - cbq_state_t *cbqp; - int i; + cbq_state_t *cbqp; + struct ifaltq *ifq; + int error; + + if (a->qid == 0) + return (EINVAL); + /* XXX not MP safe */ if ((cbqp = a->altq_disc) == NULL) return (EINVAL); + ifq = cbqp->ifnp.ifq_; + + ALTQ_LOCK(ifq); + error = cbq_add_queue_locked(a, cbqp); + ALTQ_UNLOCK(ifq); + + return error; +} + +static int +cbq_remove_queue_locked(struct pf_altq *a, cbq_state_t *cbqp) +{ + struct rm_class *cl; + int i; if ((cl = clh_to_clp(cbqp, a->qid)) == NULL) return (EINVAL); @@ -407,6 +413,25 @@ cbq_remove_queue(struct pf_altq *a) return (0); } +int +cbq_remove_queue(struct pf_altq *a) +{ + cbq_state_t *cbqp; + struct ifaltq *ifq; + int error; + + /* XXX not MP safe */ + if ((cbqp = a->altq_disc) == NULL) + return (EINVAL); + ifq = cbqp->ifnp.ifq_; + + ALTQ_LOCK(ifq); + error = cbq_remove_queue_locked(a, cbqp); + ALTQ_UNLOCK(ifq); + + return error; +} + int cbq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) { @@ -414,18 +439,27 @@ cbq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) struct rm_class *cl; class_stats_t stats; int error = 0; + struct ifaltq *ifq; + + if (*nbytes < sizeof(stats)) + return (EINVAL); + /* XXX not MP safe */ if ((cbqp = altq_lookup(a->ifname, ALTQT_CBQ)) == NULL) return (EBADF); + ifq = cbqp->ifnp.ifq_; - if ((cl = clh_to_clp(cbqp, a->qid)) == NULL) - return (EINVAL); + ALTQ_LOCK(ifq); - if (*nbytes < sizeof(stats)) + if ((cl = clh_to_clp(cbqp, a->qid)) == NULL) { + ALTQ_UNLOCK(ifq); return (EINVAL); + } get_class_stats(&stats, cl); + ALTQ_UNLOCK(ifq); + if ((error = copyout((caddr_t)&stats, ubuf, sizeof(stats))) != 0) return (error); *nbytes = sizeof(stats); diff --git a/sys/net/altq/altq_fairq.c b/sys/net/altq/altq_fairq.c index f07f342ac5..e171affecf 100644 --- a/sys/net/altq/altq_fairq.c +++ b/sys/net/altq/altq_fairq.c @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sys/net/altq/altq_fairq.c,v 1.1 2008/04/06 18:58:15 dillon Exp $ + * $DragonFly: src/sys/net/altq/altq_fairq.c,v 1.2 2008/05/14 11:59:23 sephe Exp $ */ /* * Matt: I gutted altq_priq.c and used it as a skeleton on which to build @@ -133,18 +133,10 @@ static void get_class_stats(struct fairq_classstats *, struct fairq_class *); static struct fairq_class *clh_to_clp(struct fairq_if *, uint32_t); int -fairq_pfattach(struct pf_altq *a) +fairq_pfattach(struct pf_altq *a, struct ifaltq *ifq) { - struct ifnet *ifp; - int error; - - if ((ifp = ifunit(a->ifname)) == NULL || a->altq_disc == NULL) - return (EINVAL); - crit_enter(); - error = altq_attach(&ifp->if_snd, ALTQT_FAIRQ, a->altq_disc, + return altq_attach(ifq, ALTQT_FAIRQ, a->altq_disc, fairq_enqueue, fairq_dequeue, fairq_request, NULL, NULL); - crit_exit(); - return (error); } int @@ -185,20 +177,14 @@ fairq_remove_altq(struct pf_altq *a) return (0); } -int -fairq_add_queue(struct pf_altq *a) +static int +fairq_add_queue_locked(struct pf_altq *a, struct fairq_if *pif) { - struct fairq_if *pif; struct fairq_class *cl; - if ((pif = a->altq_disc) == NULL) - return (EINVAL); + KKASSERT(a->priority < FAIRQ_MAXPRI); + KKASSERT(a->qid != 0); - /* check parameters */ - if (a->priority >= FAIRQ_MAXPRI) - return (EINVAL); - if (a->qid == 0) - return (EINVAL); if (pif->pif_classes[a->priority] != NULL) return (EBUSY); if (clh_to_clp(pif, a->qid) != NULL) @@ -213,13 +199,34 @@ fairq_add_queue(struct pf_altq *a) } int -fairq_remove_queue(struct pf_altq *a) +fairq_add_queue(struct pf_altq *a) { struct fairq_if *pif; - struct fairq_class *cl; + struct ifaltq *ifq; + int error; + + /* check parameters */ + if (a->priority >= FAIRQ_MAXPRI) + return (EINVAL); + if (a->qid == 0) + return (EINVAL); + /* XXX not MP safe */ if ((pif = a->altq_disc) == NULL) return (EINVAL); + ifq = pif->pif_ifq; + + ALTQ_LOCK(ifq); + error = fairq_add_queue_locked(a, pif); + ALTQ_UNLOCK(ifq); + + return error; +} + +static int +fairq_remove_queue_locked(struct pf_altq *a, struct fairq_if *pif) +{ + struct fairq_class *cl; if ((cl = clh_to_clp(pif, a->qid)) == NULL) return (EINVAL); @@ -227,25 +234,53 @@ fairq_remove_queue(struct pf_altq *a) return (fairq_class_destroy(cl)); } +int +fairq_remove_queue(struct pf_altq *a) +{ + struct fairq_if *pif; + struct ifaltq *ifq; + int error; + + /* XXX not MP safe */ + if ((pif = a->altq_disc) == NULL) + return (EINVAL); + ifq = pif->pif_ifq; + + ALTQ_LOCK(ifq); + error = fairq_remove_queue_locked(a, pif); + ALTQ_UNLOCK(ifq); + + return error; +} + int fairq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) { struct fairq_if *pif; struct fairq_class *cl; struct fairq_classstats stats; + struct ifaltq *ifq; int error = 0; + if (*nbytes < sizeof(stats)) + return (EINVAL); + + /* XXX not MP safe */ if ((pif = altq_lookup(a->ifname, ALTQT_FAIRQ)) == NULL) return (EBADF); + ifq = pif->pif_ifq; - if ((cl = clh_to_clp(pif, a->qid)) == NULL) - return (EINVAL); + ALTQ_LOCK(ifq); - if (*nbytes < sizeof(stats)) + if ((cl = clh_to_clp(pif, a->qid)) == NULL) { + ALTQ_UNLOCK(ifq); return (EINVAL); + } get_class_stats(&stats, cl); + ALTQ_UNLOCK(ifq); + if ((error = copyout((caddr_t)&stats, ubuf, sizeof(stats))) != 0) return (error); *nbytes = sizeof(stats); diff --git a/sys/net/altq/altq_hfsc.c b/sys/net/altq/altq_hfsc.c index e81dba8e61..45c7b4869f 100644 --- a/sys/net/altq/altq_hfsc.c +++ b/sys/net/altq/altq_hfsc.c @@ -1,5 +1,5 @@ /* $KAME: altq_hfsc.c,v 1.25 2004/04/17 10:54:48 kjc Exp $ */ -/* $DragonFly: src/sys/net/altq/altq_hfsc.c,v 1.8 2006/12/22 23:44:55 swildner Exp $ */ +/* $DragonFly: src/sys/net/altq/altq_hfsc.c,v 1.9 2008/05/14 11:59:23 sephe Exp $ */ /* * Copyright (c) 1997-1999 Carnegie Mellon University. All Rights Reserved. @@ -139,18 +139,10 @@ static struct hfsc_class *clh_to_clp(struct hfsc_if *, uint32_t); #define HT_INFINITY 0xffffffffffffffffLL /* infinite time value */ int -hfsc_pfattach(struct pf_altq *a) +hfsc_pfattach(struct pf_altq *a, struct ifaltq *ifq) { - struct ifnet *ifp; - int error; - - if ((ifp = ifunit(a->ifname)) == NULL || a->altq_disc == NULL) - return (EINVAL); - crit_enter(); - error = altq_attach(&ifp->if_snd, ALTQT_HFSC, a->altq_disc, + return altq_attach(ifq, ALTQT_HFSC, a->altq_disc, hfsc_enqueue, hfsc_dequeue, hfsc_request, NULL, NULL); - crit_exit(); - return (error); } int @@ -195,16 +187,14 @@ hfsc_remove_altq(struct pf_altq *a) return (0); } -int -hfsc_add_queue(struct pf_altq *a) +static int +hfsc_add_queue_locked(struct pf_altq *a, struct hfsc_if *hif) { - struct hfsc_if *hif; struct hfsc_class *cl, *parent; struct hfsc_opts *opts; struct service_curve rtsc, lssc, ulsc; - if ((hif = a->altq_disc) == NULL) - return (EINVAL); + KKASSERT(a->qid != 0); opts = &a->pq_u.hfsc_opts; @@ -213,9 +203,6 @@ hfsc_add_queue(struct pf_altq *a) else if ((parent = clh_to_clp(hif, a->parent_qid)) == NULL) return (EINVAL); - if (a->qid == 0) - return (EINVAL); - if (clh_to_clp(hif, a->qid) != NULL) return (EBUSY); @@ -238,13 +225,31 @@ hfsc_add_queue(struct pf_altq *a) } int -hfsc_remove_queue(struct pf_altq *a) +hfsc_add_queue(struct pf_altq *a) { struct hfsc_if *hif; - struct hfsc_class *cl; + struct ifaltq *ifq; + int error; + if (a->qid == 0) + return (EINVAL); + + /* XXX not MP safe */ if ((hif = a->altq_disc) == NULL) return (EINVAL); + ifq = hif->hif_ifq; + + ALTQ_LOCK(ifq); + error = hfsc_add_queue_locked(a, hif); + ALTQ_UNLOCK(ifq); + + return error; +} + +static int +hfsc_remove_queue_locked(struct pf_altq *a, struct hfsc_if *hif) +{ + struct hfsc_class *cl; if ((cl = clh_to_clp(hif, a->qid)) == NULL) return (EINVAL); @@ -252,25 +257,53 @@ hfsc_remove_queue(struct pf_altq *a) return (hfsc_class_destroy(cl)); } +int +hfsc_remove_queue(struct pf_altq *a) +{ + struct hfsc_if *hif; + struct ifaltq *ifq; + int error; + + /* XXX not MP safe */ + if ((hif = a->altq_disc) == NULL) + return (EINVAL); + ifq = hif->hif_ifq; + + ALTQ_LOCK(ifq); + error = hfsc_remove_queue_locked(a, hif); + ALTQ_UNLOCK(ifq); + + return error; +} + int hfsc_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) { struct hfsc_if *hif; struct hfsc_class *cl; struct hfsc_classstats stats; + struct ifaltq *ifq; int error = 0; + if (*nbytes < sizeof(stats)) + return (EINVAL); + + /* XXX not MP safe */ if ((hif = altq_lookup(a->ifname, ALTQT_HFSC)) == NULL) return (EBADF); + ifq = hif->hif_ifq; - if ((cl = clh_to_clp(hif, a->qid)) == NULL) - return (EINVAL); + ALTQ_LOCK(ifq); - if (*nbytes < sizeof(stats)) + if ((cl = clh_to_clp(hif, a->qid)) == NULL) { + ALTQ_UNLOCK(ifq); return (EINVAL); + } get_class_stats(&stats, cl); + ALTQ_UNLOCK(ifq); + if ((error = copyout((caddr_t)&stats, ubuf, sizeof(stats))) != 0) return (error); *nbytes = sizeof(stats); diff --git a/sys/net/altq/altq_priq.c b/sys/net/altq/altq_priq.c index 1f31864aa9..0ca183effa 100644 --- a/sys/net/altq/altq_priq.c +++ b/sys/net/altq/altq_priq.c @@ -1,5 +1,5 @@ /* $KAME: altq_priq.c,v 1.12 2004/04/17 10:54:48 kjc Exp $ */ -/* $DragonFly: src/sys/net/altq/altq_priq.c,v 1.8 2006/12/22 23:44:55 swildner Exp $ */ +/* $DragonFly: src/sys/net/altq/altq_priq.c,v 1.9 2008/05/14 11:59:23 sephe Exp $ */ /* * Copyright (C) 2000-2003 @@ -78,18 +78,10 @@ static void get_class_stats(struct priq_classstats *, struct priq_class *); static struct priq_class *clh_to_clp(struct priq_if *, uint32_t); int -priq_pfattach(struct pf_altq *a) +priq_pfattach(struct pf_altq *a, struct ifaltq *ifq) { - struct ifnet *ifp; - int error; - - if ((ifp = ifunit(a->ifname)) == NULL || a->altq_disc == NULL) - return (EINVAL); - crit_enter(); - error = altq_attach(&ifp->if_snd, ALTQT_PRIQ, a->altq_disc, + return altq_attach(ifq, ALTQT_PRIQ, a->altq_disc, priq_enqueue, priq_dequeue, priq_request, NULL, NULL); - crit_exit(); - return (error); } int @@ -130,20 +122,14 @@ priq_remove_altq(struct pf_altq *a) return (0); } -int -priq_add_queue(struct pf_altq *a) +static int +priq_add_queue_locked(struct pf_altq *a, struct priq_if *pif) { - struct priq_if *pif; struct priq_class *cl; - if ((pif = a->altq_disc) == NULL) - return (EINVAL); + KKASSERT(a->priority < PRIQ_MAXPRI); + KKASSERT(a->qid != 0); - /* check parameters */ - if (a->priority >= PRIQ_MAXPRI) - return (EINVAL); - if (a->qid == 0) - return (EINVAL); if (pif->pif_classes[a->priority] != NULL) return (EBUSY); if (clh_to_clp(pif, a->qid) != NULL) @@ -158,13 +144,34 @@ priq_add_queue(struct pf_altq *a) } int -priq_remove_queue(struct pf_altq *a) +priq_add_queue(struct pf_altq *a) { struct priq_if *pif; - struct priq_class *cl; + struct ifaltq *ifq; + int error; + + /* check parameters */ + if (a->priority >= PRIQ_MAXPRI) + return (EINVAL); + if (a->qid == 0) + return (EINVAL); + /* XXX not MP safe */ if ((pif = a->altq_disc) == NULL) return (EINVAL); + ifq = pif->pif_ifq; + + ALTQ_LOCK(ifq); + error = priq_add_queue_locked(a, pif); + ALTQ_UNLOCK(ifq); + + return error; +} + +static int +priq_remove_queue_locked(struct pf_altq *a, struct priq_if *pif) +{ + struct priq_class *cl; if ((cl = clh_to_clp(pif, a->qid)) == NULL) return (EINVAL); @@ -172,25 +179,53 @@ priq_remove_queue(struct pf_altq *a) return (priq_class_destroy(cl)); } +int +priq_remove_queue(struct pf_altq *a) +{ + struct priq_if *pif; + struct ifaltq *ifq; + int error; + + /* XXX not MF safe */ + if ((pif = a->altq_disc) == NULL) + return (EINVAL); + ifq = pif->pif_ifq; + + ALTQ_LOCK(ifq); + error = priq_remove_queue_locked(a, pif); + ALTQ_UNLOCK(ifq); + + return error; +} + int priq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) { struct priq_if *pif; struct priq_class *cl; struct priq_classstats stats; + struct ifaltq *ifq; int error = 0; + if (*nbytes < sizeof(stats)) + return (EINVAL); + + /* XXX not MP safe */ if ((pif = altq_lookup(a->ifname, ALTQT_PRIQ)) == NULL) return (EBADF); + ifq = pif->pif_ifq; - if ((cl = clh_to_clp(pif, a->qid)) == NULL) - return (EINVAL); + ALTQ_LOCK(ifq); - if (*nbytes < sizeof(stats)) + if ((cl = clh_to_clp(pif, a->qid)) == NULL) { + ALTQ_UNLOCK(ifq); return (EINVAL); + } get_class_stats(&stats, cl); + ALTQ_UNLOCK(ifq); + if ((error = copyout((caddr_t)&stats, ubuf, sizeof(stats))) != 0) return (error); *nbytes = sizeof(stats); diff --git a/sys/net/altq/altq_subr.c b/sys/net/altq/altq_subr.c index 7de22fb9af..c96d21c499 100644 --- a/sys/net/altq/altq_subr.c +++ b/sys/net/altq/altq_subr.c @@ -1,5 +1,5 @@ /* $KAME: altq_subr.c,v 1.23 2004/04/20 16:10:06 itojun Exp $ */ -/* $DragonFly: src/sys/net/altq/altq_subr.c,v 1.11 2008/05/10 17:24:06 dillon Exp $ */ +/* $DragonFly: src/sys/net/altq/altq_subr.c,v 1.12 2008/05/14 11:59:23 sephe Exp $ */ /* * Copyright (C) 1997-2003 @@ -74,6 +74,11 @@ * internal function prototypes */ static void tbr_timeout(void *); +static int altq_enable_locked(struct ifaltq *); +static int altq_disable_locked(struct ifaltq *); +static int altq_detach_locked(struct ifaltq *); +static int tbr_set_locked(struct ifaltq *, struct tb_profile *); + int (*altq_input)(struct mbuf *, int) = NULL; static int tbr_timer = 0; /* token bucket regulator timer */ static struct callout tbr_callout; @@ -123,8 +128,8 @@ altq_attach(struct ifaltq *ifq, int type, void *discipline, return 0; } -int -altq_detach(struct ifaltq *ifq) +static int +altq_detach_locked(struct ifaltq *ifq) { if (!ifq_is_ready(ifq)) return ENXIO; @@ -143,38 +148,67 @@ altq_detach(struct ifaltq *ifq) } int -altq_enable(struct ifaltq *ifq) +altq_detach(struct ifaltq *ifq) +{ + int error; + + ALTQ_LOCK(ifq); + error = altq_detach_locked(ifq); + ALTQ_UNLOCK(ifq); + return error; +} + +static int +altq_enable_locked(struct ifaltq *ifq) { if (!ifq_is_ready(ifq)) return ENXIO; if (ifq_is_enabled(ifq)) return 0; - crit_enter(); - ifq_purge(ifq); + ifq_purge_locked(ifq); KKASSERT(ifq->ifq_len == 0); + ifq->altq_flags |= ALTQF_ENABLED; if (ifq->altq_clfier != NULL) ifq->altq_flags |= ALTQF_CLASSIFY; - crit_exit(); - return 0; } int -altq_disable(struct ifaltq *ifq) +altq_enable(struct ifaltq *ifq) +{ + int error; + + ALTQ_LOCK(ifq); + error = altq_enable_locked(ifq); + ALTQ_UNLOCK(ifq); + return error; +} + +static int +altq_disable_locked(struct ifaltq *ifq) { if (!ifq_is_enabled(ifq)) return 0; - crit_enter(); - ifq_purge(ifq); + ifq_purge_locked(ifq); KKASSERT(ifq->ifq_len == 0); ifq->altq_flags &= ~(ALTQF_ENABLED|ALTQF_CLASSIFY); - crit_exit(); return 0; } +int +altq_disable(struct ifaltq *ifq) +{ + int error; + + ALTQ_LOCK(ifq); + error = altq_disable_locked(ifq); + ALTQ_UNLOCK(ifq); + return error; +} + /* * internal representation of token bucket parameters * rate: byte_per_unittime << 32 @@ -239,15 +273,15 @@ tbr_dequeue(struct ifaltq *ifq, struct mbuf *mpolled, int op) * set a token bucket regulator. * if the specified rate is zero, the token bucket regulator is deleted. */ -int -tbr_set(struct ifaltq *ifq, struct tb_profile *profile) +static int +tbr_set_locked(struct ifaltq *ifq, struct tb_profile *profile) { struct tb_regulator *tbr, *otbr; if (machclk_freq == 0) init_machclk(); if (machclk_freq == 0) { - kprintf("tbr_set: no cpu clock available!\n"); + kprintf("%s: no cpu clock available!\n", __func__); return (ENXIO); } @@ -283,6 +317,17 @@ tbr_set(struct ifaltq *ifq, struct tb_profile *profile) return (0); } +int +tbr_set(struct ifaltq *ifq, struct tb_profile *profile) +{ + int error; + + ALTQ_LOCK(ifq); + error = tbr_set_locked(ifq, profile); + ALTQ_UNLOCK(ifq); + return error; +} + /* * tbr_timeout goes through the interface list, and kicks the drivers * if necessary. @@ -338,54 +383,64 @@ tbr_get(struct ifaltq *ifq, struct tb_profile *profile) int altq_pfattach(struct pf_altq *a) { + struct ifaltq *ifq; struct ifnet *ifp; - struct tb_profile tb; - int error = 0; + int error; + + if (a->scheduler == ALTQT_NONE) + return 0; + + if (a->altq_disc == NULL) + return EINVAL; + + ifp = ifunit(a->ifname); + if (ifp == NULL) + return EINVAL; + ifq = &ifp->if_snd; + + ALTQ_LOCK(ifq); switch (a->scheduler) { - case ALTQT_NONE: - break; #ifdef ALTQ_CBQ case ALTQT_CBQ: - error = cbq_pfattach(a); + error = cbq_pfattach(a, ifq); break; #endif #ifdef ALTQ_PRIQ case ALTQT_PRIQ: - error = priq_pfattach(a); + error = priq_pfattach(a, ifq); break; #endif #ifdef ALTQ_HFSC case ALTQT_HFSC: - error = hfsc_pfattach(a); + error = hfsc_pfattach(a, ifq); break; #endif #ifdef ALTQ_FAIRQ case ALTQT_FAIRQ: - error = fairq_pfattach(a); + error = fairq_pfattach(a, ifq); break; #endif default: error = ENXIO; + goto back; } - ifp = ifunit(a->ifname); - /* if the state is running, enable altq */ - if (error == 0 && pfaltq_running && - ifp != NULL && ifp->if_snd.altq_type != ALTQT_NONE && - !ifq_is_enabled(&ifp->if_snd)) - error = altq_enable(&ifp->if_snd); + if (error == 0 && pfaltq_running && ifq->altq_type != ALTQT_NONE && + !ifq_is_enabled(ifq)) + error = altq_enable_locked(ifq); /* if altq is already enabled, reset set tokenbucket regulator */ - if (error == 0 && ifp != NULL && ifq_is_enabled(&ifp->if_snd)) { + if (error == 0 && ifq_is_enabled(ifq)) { + struct tb_profile tb; + tb.rate = a->ifbandwidth; tb.depth = a->tbrsize; - crit_enter(); - error = tbr_set(&ifp->if_snd, &tb); - crit_exit(); + error = tbr_set_locked(ifq, &tb); } - +back: + ALTQ_UNLOCK(ifq); return (error); } @@ -398,22 +453,30 @@ int altq_pfdetach(struct pf_altq *a) { struct ifnet *ifp; + struct ifaltq *ifq; int error = 0; - if ((ifp = ifunit(a->ifname)) == NULL) + ifp = ifunit(a->ifname); + if (ifp == NULL) return (EINVAL); + ifq = &ifp->if_snd; /* if this discipline is no longer referenced, just return */ - if (a->altq_disc == NULL || a->altq_disc != ifp->if_snd.altq_disc) + if (a->altq_disc == NULL) return (0); - crit_enter(); - if (ifq_is_enabled(&ifp->if_snd)) - error = altq_disable(&ifp->if_snd); + ALTQ_LOCK(ifq); + + if (a->altq_disc != ifq->altq_disc) + goto back; + + if (ifq_is_enabled(ifq)) + error = altq_disable_locked(ifq); if (error == 0) - error = altq_detach(&ifp->if_snd); - crit_exit(); + error = altq_detach_locked(ifq); +back: + ALTQ_UNLOCK(ifq); return (error); } diff --git a/sys/net/altq/altq_var.h b/sys/net/altq/altq_var.h index d39e16fc56..e5a54bad51 100644 --- a/sys/net/altq/altq_var.h +++ b/sys/net/altq/altq_var.h @@ -1,5 +1,5 @@ /* $KAME: altq_var.h,v 1.17 2004/04/20 05:09:08 kjc Exp $ */ -/* $DragonFly: src/sys/net/altq/altq_var.h,v 1.4 2008/05/10 17:24:06 dillon Exp $ */ +/* $DragonFly: src/sys/net/altq/altq_var.h,v 1.5 2008/05/14 11:59:23 sephe Exp $ */ /* * Copyright (C) 1998-2003 @@ -79,28 +79,28 @@ int altq_add_queue(struct pf_altq *); int altq_remove_queue(struct pf_altq *); int altq_getqstats(struct pf_altq *, void *, int *); -int cbq_pfattach(struct pf_altq *); +int cbq_pfattach(struct pf_altq *, struct ifaltq *); int cbq_add_altq(struct pf_altq *); int cbq_remove_altq(struct pf_altq *); int cbq_add_queue(struct pf_altq *); int cbq_remove_queue(struct pf_altq *); int cbq_getqstats(struct pf_altq *, void *, int *); -int priq_pfattach(struct pf_altq *); +int priq_pfattach(struct pf_altq *, struct ifaltq *); int priq_add_altq(struct pf_altq *); int priq_remove_altq(struct pf_altq *); int priq_add_queue(struct pf_altq *); int priq_remove_queue(struct pf_altq *); int priq_getqstats(struct pf_altq *, void *, int *); -int hfsc_pfattach(struct pf_altq *); +int hfsc_pfattach(struct pf_altq *, struct ifaltq *); int hfsc_add_altq(struct pf_altq *); int hfsc_remove_altq(struct pf_altq *); int hfsc_add_queue(struct pf_altq *); int hfsc_remove_queue(struct pf_altq *); int hfsc_getqstats(struct pf_altq *, void *, int *); -int fairq_pfattach(struct pf_altq *); +int fairq_pfattach(struct pf_altq *, struct ifaltq *); int fairq_add_altq(struct pf_altq *); int fairq_remove_altq(struct pf_altq *); int fairq_add_queue(struct pf_altq *); diff --git a/sys/net/altq/if_altq.h b/sys/net/altq/if_altq.h index 40c0027438..9b0da4b5d6 100644 --- a/sys/net/altq/if_altq.h +++ b/sys/net/altq/if_altq.h @@ -1,5 +1,5 @@ /* $KAME: if_altq.h,v 1.11 2003/07/10 12:07:50 kjc Exp $ */ -/* $DragonFly: src/sys/net/altq/if_altq.h,v 1.3 2006/05/20 02:42:09 dillon Exp $ */ +/* $DragonFly: src/sys/net/altq/if_altq.h,v 1.4 2008/05/14 11:59:23 sephe Exp $ */ /* * Copyright (C) 1997-2003 @@ -29,6 +29,10 @@ #ifndef _NET_ALTQ_IF_ALTQ_H_ #define _NET_ALTQ_IF_ALTQ_H_ +#ifndef _SYS_SERIALIZE_H_ +#include +#endif + struct altq_pktattr; /* @@ -60,8 +64,23 @@ struct ifaltq { /* token bucket regulator */ struct tb_regulator *altq_tbr; + + struct lwkt_serialize altq_lock; + struct mbuf *altq_prepended; /* mbuf dequeued, but not yet xmit */ + int altq_started; /* ifnet.if_start interlock */ }; +#ifdef SMP +#define ALTQ_ASSERT_LOCKED(ifq) ASSERT_SERIALIZED(&(ifq)->altq_lock) +#define ALTQ_LOCK_INIT(ifq) lwkt_serialize_init(&(ifq)->altq_lock) +#define ALTQ_LOCK(ifq) lwkt_serialize_adaptive_enter(&(ifq)->altq_lock) +#define ALTQ_UNLOCK(ifq) lwkt_serialize_exit(&(ifq)->altq_lock) +#else +#define ALTQ_ASSERT_LOCKED(ifq) ((void)0) /* XXX */ +#define ALTQ_LOCK_INIT(ifq) ((void)0) +#define ALTQ_LOCK(ifq) crit_enter() +#define ALTQ_UNLOCK(ifq) crit_exit() +#endif #ifdef _KERNEL diff --git a/sys/net/bpf.c b/sys/net/bpf.c index 326397eed5..cd9862e2d1 100644 --- a/sys/net/bpf.c +++ b/sys/net/bpf.c @@ -38,7 +38,7 @@ * @(#)bpf.c 8.2 (Berkeley) 3/28/94 * * $FreeBSD: src/sys/net/bpf.c,v 1.59.2.12 2002/04/14 21:41:48 luigi Exp $ - * $DragonFly: src/sys/net/bpf.c,v 1.44 2008/03/14 09:52:10 matthias Exp $ + * $DragonFly: src/sys/net/bpf.c,v 1.45 2008/05/14 11:59:23 sephe Exp $ */ #include "use_bpf.h" @@ -530,9 +530,7 @@ bpfwrite(struct dev_write_args *ap) dst.sa_family = pseudo_AF_HDRCMPLT; crit_enter(); - lwkt_serialize_enter(ifp->if_serializer); - error = (*ifp->if_output)(ifp, m, &dst, (struct rtentry *)NULL); - lwkt_serialize_exit(ifp->if_serializer); + error = ifp->if_output(ifp, m, &dst, NULL); crit_exit(); /* * The driver frees the mbuf. diff --git a/sys/net/gif/if_gif.c b/sys/net/gif/if_gif.c index 77225df756..f400b52c7d 100644 --- a/sys/net/gif/if_gif.c +++ b/sys/net/gif/if_gif.c @@ -27,7 +27,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/net/if_gif.c,v 1.4.2.15 2002/11/08 16:57:13 ume Exp $ - * $DragonFly: src/sys/net/gif/if_gif.c,v 1.20 2008/01/11 11:59:41 sephe Exp $ + * $DragonFly: src/sys/net/gif/if_gif.c,v 1.21 2008/05/14 11:59:23 sephe Exp $ * $KAME: if_gif.c,v 1.87 2001/10/19 08:50:27 itojun Exp $ */ @@ -290,9 +290,9 @@ gif_encapcheck(const struct mbuf *m, int off, int proto, void *arg) * Parameters: * rt: added in net2 */ -int -gif_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, - struct rtentry *rt) +static int +gif_output_serialized(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, + struct rtentry *rt) { struct gif_softc *sc = (struct gif_softc*)ifp; int error = 0; @@ -363,6 +363,18 @@ gif_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, return error; } +int +gif_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, + struct rtentry *rt) +{ + int error; + + lwkt_serialize_enter(ifp->if_serializer); + error = gif_output_serialized(ifp, m, dst, rt); + lwkt_serialize_exit(ifp->if_serializer); + return error; +} + void gif_input(struct mbuf *m, int af, struct ifnet *ifp) { diff --git a/sys/net/gre/if_gre.c b/sys/net/gre/if_gre.c index 2405d2eefc..14fecd2911 100644 --- a/sys/net/gre/if_gre.c +++ b/sys/net/gre/if_gre.c @@ -1,6 +1,6 @@ /* $NetBSD: if_gre.c,v 1.42 2002/08/14 00:23:27 itojun Exp $ */ /* $FreeBSD: src/sys/net/if_gre.c,v 1.9.2.3 2003/01/23 21:06:44 sam Exp $ */ -/* $DragonFly: src/sys/net/gre/if_gre.c,v 1.20 2008/01/11 11:59:41 sephe Exp $ */ +/* $DragonFly: src/sys/net/gre/if_gre.c,v 1.21 2008/05/14 11:59:23 sephe Exp $ */ /* * Copyright (c) 1998 The NetBSD Foundation, Inc. @@ -208,8 +208,8 @@ gre_clone_destroy(struct ifnet *ifp) * given by sc->g_proto. See also RFC 1701 and RFC 2004 */ static int -gre_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, - struct rtentry *rt) +gre_output_serialized(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, + struct rtentry *rt) { int error = 0; struct gre_softc *sc = ifp->if_softc; @@ -388,6 +388,19 @@ gre_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, return (error); } +static int +gre_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, + struct rtentry *rt) +{ + int error; + + lwkt_serialize_enter(ifp->if_serializer); + error = gre_output_serialized(ifp, m, dst, rt); + lwkt_serialize_exit(ifp->if_serializer); + + return error; +} + static int gre_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cr) { diff --git a/sys/net/i4b/driver/i4b_ipr.c b/sys/net/i4b/driver/i4b_ipr.c index a306ba6625..be2f31ee9b 100644 --- a/sys/net/i4b/driver/i4b_ipr.c +++ b/sys/net/i4b/driver/i4b_ipr.c @@ -28,7 +28,7 @@ * --------------------------------------------------------- * * $FreeBSD: src/sys/i4b/driver/i4b_ipr.c,v 1.8.2.3 2001/10/27 15:48:17 hm Exp $ - * $DragonFly: src/sys/net/i4b/driver/i4b_ipr.c,v 1.21 2008/01/06 16:55:52 swildner Exp $ + * $DragonFly: src/sys/net/i4b/driver/i4b_ipr.c,v 1.22 2008/05/14 11:59:23 sephe Exp $ * * last edit-date: [Fri Oct 26 19:32:38 2001] * @@ -292,8 +292,8 @@ i4biprattach(void *dummy) * output a packet to the ISDN B-channel *---------------------------------------------------------------------------*/ static int -i4biproutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, - struct rtentry *rtp) +i4biproutput_serialized(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, + struct rtentry *rtp) { struct ipr_softc *sc; int unit; @@ -412,6 +412,19 @@ i4biproutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, return (0); } +static int +i4biproutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, + struct rtentry *rtp) +{ + int error; + + lwkt_serialize_enter(ifp->if_serializer); + error = i4biproutput_serialized(ifp, m, dst, rtp); + lwkt_serialize_exit(ifp->if_serializer); + + return error; +} + /*---------------------------------------------------------------------------* * process ioctl *---------------------------------------------------------------------------*/ diff --git a/sys/net/if.c b/sys/net/if.c index 4e83deedb5..a23d6684e6 100644 --- a/sys/net/if.c +++ b/sys/net/if.c @@ -32,7 +32,7 @@ * * @(#)if.c 8.3 (Berkeley) 1/4/94 * $FreeBSD: src/sys/net/if.c,v 1.185 2004/03/13 02:35:03 brooks Exp $ - * $DragonFly: src/sys/net/if.c,v 1.63 2008/03/29 03:38:53 sephe Exp $ + * $DragonFly: src/sys/net/if.c,v 1.64 2008/05/14 11:59:23 sephe Exp $ */ #include "opt_compat.h" @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -69,6 +70,7 @@ #include #include #include +#include #include #include @@ -96,15 +98,6 @@ struct netmsg_ifaddr { int tail; }; -/* - * Support for non-ALTQ interfaces. - */ -static int ifq_classic_enqueue(struct ifaltq *, struct mbuf *, - struct altq_pktattr *); -static struct mbuf * - ifq_classic_dequeue(struct ifaltq *, struct mbuf *, int); -static int ifq_classic_request(struct ifaltq *, int, void *); - /* * System initialization */ @@ -138,12 +131,67 @@ MALLOC_DEFINE(M_IFMADDR, "ether_multi", "link-level multicast address"); int ifqmaxlen = IFQ_MAXLEN; struct ifnethead ifnet = TAILQ_HEAD_INITIALIZER(ifnet); +/* In ifq_dispatch(), try to do direct ifnet.if_start first */ +static int ifq_dispatch_schedonly = 0; +SYSCTL_INT(_net_link_generic, OID_AUTO, ifq_dispatch_schedonly, CTLFLAG_RW, + &ifq_dispatch_schedonly, 0, ""); + +/* In ifq_dispatch(), schedule ifnet.if_start without checking ifnet.if_snd */ +static int ifq_dispatch_schednochk = 0; +SYSCTL_INT(_net_link_generic, OID_AUTO, ifq_dispatch_schednochk, CTLFLAG_RW, + &ifq_dispatch_schednochk, 0, ""); + +/* In if_devstart(), try to do direct ifnet.if_start first */ +static int if_devstart_schedonly = 0; +SYSCTL_INT(_net_link_generic, OID_AUTO, if_devstart_schedonly, CTLFLAG_RW, + &if_devstart_schedonly, 0, ""); + +/* In if_devstart(), schedule ifnet.if_start without checking ifnet.if_snd */ +static int if_devstart_schednochk = 0; +SYSCTL_INT(_net_link_generic, OID_AUTO, if_devstart_schednochk, CTLFLAG_RW, + &if_devstart_schednochk, 0, ""); + +#ifdef SMP +/* Schedule ifnet.if_start on the current CPU */ +static int if_start_oncpu_sched = 0; +SYSCTL_INT(_net_link_generic, OID_AUTO, if_start_oncpu_sched, CTLFLAG_RW, + &if_start_oncpu_sched, 0, ""); +#endif + struct callout if_slowtimo_timer; int if_index = 0; struct ifnet **ifindex2ifnet = NULL; static struct thread ifaddr_threads[MAXCPU]; +#define IFQ_KTR_STRING "ifq=%p" +#define IFQ_KTR_ARG_SIZE (sizeof(void *)) +#ifndef KTR_IFQ +#define KTR_IFQ KTR_ALL +#endif +KTR_INFO_MASTER(ifq); +KTR_INFO(KTR_IFQ, ifq, enqueue, 0, IFQ_KTR_STRING, IFQ_KTR_ARG_SIZE); +KTR_INFO(KTR_IFQ, ifq, dequeue, 1, IFQ_KTR_STRING, IFQ_KTR_ARG_SIZE); +#define logifq(name, arg) KTR_LOG(ifq_ ## name, arg) + +#define IF_START_KTR_STRING "ifp=%p" +#define IF_START_KTR_ARG_SIZE (sizeof(void *)) +#ifndef KTR_IF_START +#define KTR_IF_START KTR_ALL +#endif +KTR_INFO_MASTER(if_start); +KTR_INFO(KTR_IF_START, if_start, run, 0, + IF_START_KTR_STRING, IF_START_KTR_ARG_SIZE); +KTR_INFO(KTR_IF_START, if_start, sched, 1, + IF_START_KTR_STRING, IF_START_KTR_ARG_SIZE); +KTR_INFO(KTR_IF_START, if_start, avoid, 2, + IF_START_KTR_STRING, IF_START_KTR_ARG_SIZE); +KTR_INFO(KTR_IF_START, if_start, contend_sched, 3, + IF_START_KTR_STRING, IF_START_KTR_ARG_SIZE); +KTR_INFO(KTR_IF_START, if_start, chase_sched, 4, + IF_START_KTR_STRING, IF_START_KTR_ARG_SIZE); +#define logifstart(name, arg) KTR_LOG(if_start_ ## name, arg) + /* * Network interface utility routines. * @@ -170,6 +218,192 @@ ifinit(void *dummy) if_slowtimo(0); } +static int +if_start_cpuid(struct ifnet *ifp) +{ + return ifp->if_cpuid; +} + +#ifdef DEVICE_POLLING +static int +if_start_cpuid_poll(struct ifnet *ifp) +{ + int poll_cpuid = ifp->if_poll_cpuid; + + if (poll_cpuid >= 0) + return poll_cpuid; + else + return ifp->if_cpuid; +} +#endif + +static void +if_start_ipifunc(void *arg) +{ + struct ifnet *ifp = arg; + struct lwkt_msg *lmsg = &ifp->if_start_nmsg[mycpuid].nm_lmsg; + + crit_enter(); + if (lmsg->ms_flags & MSGF_DONE) + lwkt_sendmsg(ifa_portfn(mycpuid), lmsg); + crit_exit(); +} + +/* + * Schedule ifnet.if_start on ifnet's CPU + */ +static void +if_start_schedule(struct ifnet *ifp) +{ +#ifdef SMP + int cpu; + + if (if_start_oncpu_sched) + cpu = mycpuid; + else + cpu = ifp->if_start_cpuid(ifp); + + if (cpu != mycpuid) + lwkt_send_ipiq(globaldata_find(cpu), if_start_ipifunc, ifp); + else +#endif + if_start_ipifunc(ifp); +} + +/* + * NOTE: + * This function will release ifnet.if_start interlock, + * if ifnet.if_start does not need to be scheduled + */ +static __inline int +if_start_need_schedule(struct ifaltq *ifq, int running) +{ + if (!running || ifq_is_empty(ifq) +#ifdef ALTQ + || ifq->altq_tbr != NULL +#endif + ) { + ALTQ_LOCK(ifq); + /* + * ifnet.if_start interlock is released, if: + * 1) Hardware can not take any packets, due to + * o interface is marked down + * o hardware queue is full (IFF_OACTIVE) + * Under the second situation, hardware interrupt + * or polling(4) will call/schedule ifnet.if_start + * when hardware queue is ready + * 2) There is not packet in the ifnet.if_snd. + * Further ifq_dispatch or ifq_handoff will call/ + * schedule ifnet.if_start + * 3) TBR is used and it does not allow further + * dequeueing. + * TBR callout will call ifnet.if_start + */ + if (!running || !ifq_data_ready(ifq)) { + ifq->altq_started = 0; + ALTQ_UNLOCK(ifq); + return 0; + } + ALTQ_UNLOCK(ifq); + } + return 1; +} + +static void +if_start_dispatch(struct netmsg *nmsg) +{ + struct lwkt_msg *lmsg = &nmsg->nm_lmsg; + struct ifnet *ifp = lmsg->u.ms_resultp; + struct ifaltq *ifq = &ifp->if_snd; + int running = 0; + + crit_enter(); + lwkt_replymsg(lmsg, 0); /* reply ASAP */ + crit_exit(); + +#ifdef SMP + if (!if_start_oncpu_sched && mycpuid != ifp->if_start_cpuid(ifp)) { + /* + * If the ifnet is still up, we need to + * chase its CPU change. + */ + if (ifp->if_flags & IFF_UP) { + logifstart(chase_sched, ifp); + if_start_schedule(ifp); + return; + } else { + goto check; + } + } +#endif + + if (ifp->if_flags & IFF_UP) { + lwkt_serialize_enter(ifp->if_serializer); /* XXX try? */ + if ((ifp->if_flags & IFF_OACTIVE) == 0) { + logifstart(run, ifp); + ifp->if_start(ifp); + if ((ifp->if_flags & + (IFF_OACTIVE | IFF_RUNNING)) == IFF_RUNNING) + running = 1; + } + lwkt_serialize_exit(ifp->if_serializer); + } +check: + if (if_start_need_schedule(ifq, running)) { + crit_enter(); + if (lmsg->ms_flags & MSGF_DONE) { /* XXX necessary? */ + logifstart(sched, ifp); + lwkt_sendmsg(ifa_portfn(mycpuid), lmsg); + } + crit_exit(); + } +} + +/* Device driver ifnet.if_start helper function */ +void +if_devstart(struct ifnet *ifp) +{ + struct ifaltq *ifq = &ifp->if_snd; + int running = 0; + + ASSERT_SERIALIZED(ifp->if_serializer); + + ALTQ_LOCK(ifq); + if (ifq->altq_started || !ifq_data_ready(ifq)) { + logifstart(avoid, ifp); + ALTQ_UNLOCK(ifq); + return; + } + ifq->altq_started = 1; + ALTQ_UNLOCK(ifq); + + if (if_devstart_schedonly) { + /* + * Always schedule ifnet.if_start on ifnet's CPU, + * short circuit the rest of this function. + */ + logifstart(sched, ifp); + if_start_schedule(ifp); + return; + } + + logifstart(run, ifp); + ifp->if_start(ifp); + + if ((ifp->if_flags & (IFF_OACTIVE | IFF_RUNNING)) == IFF_RUNNING) + running = 1; + + if (if_devstart_schednochk || if_start_need_schedule(ifq, running)) { + /* + * More data need to be transmitted, ifnet.if_start is + * scheduled on ifnet's CPU, and we keep going. + * NOTE: ifnet.if_start interlock is not released. + */ + logifstart(sched, ifp); + if_start_schedule(ifp); + } +} + /* * Attach an interface to the list of "active" interfaces. * @@ -200,11 +434,24 @@ if_attach(struct ifnet *ifp, lwkt_serialize_t serializer) } ifp->if_serializer = serializer; + ifp->if_start_cpuid = if_start_cpuid; + ifp->if_cpuid = 0; + #ifdef DEVICE_POLLING /* Device is not in polling mode by default */ ifp->if_poll_cpuid = -1; + if (ifp->if_poll != NULL) + ifp->if_start_cpuid = if_start_cpuid_poll; #endif + ifp->if_start_nmsg = kmalloc(ncpus * sizeof(struct netmsg), + M_IFADDR /* XXX */, M_WAITOK); + for (i = 0; i < ncpus; ++i) { + netmsg_init(&ifp->if_start_nmsg[i], &netisr_adone_rport, 0, + if_start_dispatch); + ifp->if_start_nmsg[i].nm_lmsg.u.ms_resultp = ifp; + } + TAILQ_INSERT_TAIL(&ifnet, ifp, if_link); ifp->if_index = ++if_index; @@ -280,6 +527,9 @@ if_attach(struct ifnet *ifp, lwkt_serialize_t serializer) ifq->altq_flags &= ALTQF_CANTCHANGE; ifq->altq_tbr = NULL; ifq->altq_ifp = ifp; + ifq->altq_started = 0; + ifq->altq_prepended = NULL; + ALTQ_LOCK_INIT(ifq); ifq_set_classic(ifq); if (!SLIST_EMPTY(&domains)) @@ -475,6 +725,7 @@ if_detach(struct ifnet *ifp) TAILQ_REMOVE(&ifnet, ifp, if_link); kfree(ifp->if_addrheads, M_IFADDR); + kfree(ifp->if_start_nmsg, M_IFADDR); crit_exit(); } @@ -1771,47 +2022,43 @@ ifq_set_classic(struct ifaltq *ifq) ifq->altq_request = ifq_classic_request; } -static int +int ifq_classic_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pa __unused) { - crit_enter(); + logifq(enqueue, ifq); if (IF_QFULL(ifq)) { m_freem(m); - crit_exit(); return(ENOBUFS); } else { IF_ENQUEUE(ifq, m); - crit_exit(); return(0); } } -static struct mbuf * +struct mbuf * ifq_classic_dequeue(struct ifaltq *ifq, struct mbuf *mpolled, int op) { struct mbuf *m; - crit_enter(); switch (op) { case ALTDQ_POLL: IF_POLL(ifq, m); break; case ALTDQ_REMOVE: + logifq(dequeue, ifq); IF_DEQUEUE(ifq, m); break; default: panic("unsupported ALTQ dequeue op: %d", op); } - crit_exit(); KKASSERT(mpolled == NULL || mpolled == m); return(m); } -static int +int ifq_classic_request(struct ifaltq *ifq, int req, void *arg) { - crit_enter(); switch (req) { case ALTRQ_PURGE: IF_DRAIN(ifq); @@ -1819,10 +2066,92 @@ ifq_classic_request(struct ifaltq *ifq, int req, void *arg) default: panic("unsupported ALTQ request: %d", req); } - crit_exit(); return(0); } +int +ifq_dispatch(struct ifnet *ifp, struct mbuf *m, struct altq_pktattr *pa) +{ + struct ifaltq *ifq = &ifp->if_snd; + int not_serialized, running = 0; + int error, start = 0; + + ALTQ_LOCK(ifq); + error = ifq_enqueue_locked(ifq, m, pa); + if (error) { + ALTQ_UNLOCK(ifq); + return error; + } + if (!ifq->altq_started) { + /* + * Hold the interlock of ifnet.if_start + */ + ifq->altq_started = 1; + start = 1; + } + ALTQ_UNLOCK(ifq); + + ifp->if_obytes += m->m_pkthdr.len; + if (m->m_flags & M_MCAST) + ifp->if_omcasts++; + + if (!start) { + logifstart(avoid, ifp); + return 0; + } + + if (ifq_dispatch_schedonly) { + /* + * Always schedule ifnet.if_start on ifnet's CPU, + * short circuit the rest of this function. + */ + logifstart(sched, ifp); + if_start_schedule(ifp); + return 0; + } + + /* + * Try to do direct ifnet.if_start first, if there is + * contention on ifnet's serializer, ifnet.if_start will + * be scheduled on ifnet's CPU. + */ + not_serialized = !IS_SERIALIZED(ifp->if_serializer); + if (not_serialized) { + if (!lwkt_serialize_try(ifp->if_serializer)) { + /* + * ifnet serializer contention happened, + * ifnet.if_start is scheduled on ifnet's + * CPU, and we keep going. + */ + logifstart(contend_sched, ifp); + if_start_schedule(ifp); + return 0; + } + } + + if ((ifp->if_flags & IFF_OACTIVE) == 0) { + logifstart(run, ifp); + ifp->if_start(ifp); + if ((ifp->if_flags & + (IFF_OACTIVE | IFF_RUNNING)) == IFF_RUNNING) + running = 1; + } + + if (not_serialized) + lwkt_serialize_exit(ifp->if_serializer); + + if (ifq_dispatch_schednochk || if_start_need_schedule(ifq, running)) { + /* + * More data need to be transmitted, ifnet.if_start is + * scheduled on ifnet's CPU, and we keep going. + * NOTE: ifnet.if_start interlock is not released. + */ + logifstart(sched, ifp); + if_start_schedule(ifp); + } + return 0; +} + void * ifa_create(int size, int flags) { diff --git a/sys/net/if_atmsubr.c b/sys/net/if_atmsubr.c index 2280343158..488d4dfdca 100644 --- a/sys/net/if_atmsubr.c +++ b/sys/net/if_atmsubr.c @@ -32,7 +32,7 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/net/if_atmsubr.c,v 1.10.2.1 2001/03/06 00:29:26 obrien Exp $ - * $DragonFly: src/sys/net/if_atmsubr.c,v 1.18 2008/03/07 11:34:19 sephe Exp $ + * $DragonFly: src/sys/net/if_atmsubr.c,v 1.19 2008/05/14 11:59:23 sephe Exp $ */ /* @@ -214,7 +214,10 @@ atm_output(struct ifnet *ifp, struct mbuf *m0, struct sockaddr *dst, /* * Dispatch message to the interface. */ - return (ifq_handoff(ifp, m, &pktattr)); + lwkt_serialize_enter(ifp->if_serializer); + error = ifq_handoff(ifp, m, &pktattr); + lwkt_serialize_exit(ifp->if_serializer); + return error; bad: if (m != NULL) diff --git a/sys/net/if_ethersubr.c b/sys/net/if_ethersubr.c index 0c6cb6733a..0a9dd5a2ac 100644 --- a/sys/net/if_ethersubr.c +++ b/sys/net/if_ethersubr.c @@ -32,7 +32,7 @@ * * @(#)if_ethersubr.c 8.1 (Berkeley) 6/10/93 * $FreeBSD: src/sys/net/if_ethersubr.c,v 1.70.2.33 2003/04/28 15:45:53 archie Exp $ - * $DragonFly: src/sys/net/if_ethersubr.c,v 1.58 2008/05/02 07:40:32 sephe Exp $ + * $DragonFly: src/sys/net/if_ethersubr.c,v 1.59 2008/05/14 11:59:23 sephe Exp $ */ #include "opt_atalk.h" @@ -181,8 +181,6 @@ ether_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, struct arpcom *ac = IFP2AC(ifp); int error; - ASSERT_SERIALIZED(ifp->if_serializer); - if (ifp->if_flags & IFF_MONITOR) gotoerr(ENETDOWN); if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) != (IFF_UP | IFF_RUNNING)) @@ -328,7 +326,9 @@ ether_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, if (ifp->if_bridge) { KASSERT(bridge_output_p != NULL, ("%s: if_bridge not loaded!", __func__)); - return ((*bridge_output_p)(ifp, m, NULL, NULL)); + lwkt_serialize_enter(ifp->if_serializer); + error = bridge_output_p(ifp, m, NULL, NULL); + lwkt_serialize_exit(ifp->if_serializer); } /* @@ -403,8 +403,6 @@ ether_output_frame(struct ifnet *ifp, struct mbuf *m) struct altq_pktattr pktattr; struct m_tag *mtag; - ASSERT_SERIALIZED(ifp->if_serializer); - /* Extract info from dummynet tag */ mtag = m_tag_find(m, PACKET_TAG_DUMMYNET, NULL); if (mtag != NULL) { @@ -445,7 +443,7 @@ ether_output_frame(struct ifnet *ifp, struct mbuf *m) * Queue message on interface, update output statistics if * successful, and start output if interface not yet active. */ - error = ifq_handoff(ifp, m, &pktattr); + error = ifq_dispatch(ifp, m, &pktattr); return (error); } diff --git a/sys/net/if_loop.c b/sys/net/if_loop.c index c1bb3d65a4..e0f3f51729 100644 --- a/sys/net/if_loop.c +++ b/sys/net/if_loop.c @@ -32,7 +32,7 @@ * * @(#)if_loop.c 8.1 (Berkeley) 6/10/93 * $FreeBSD: src/sys/net/if_loop.c,v 1.47.2.8 2003/06/01 01:46:11 silby Exp $ - * $DragonFly: src/sys/net/if_loop.c,v 1.20 2006/12/22 23:44:54 swildner Exp $ + * $DragonFly: src/sys/net/if_loop.c,v 1.21 2008/05/14 11:59:23 sephe Exp $ */ /* @@ -262,7 +262,9 @@ if_simloop(struct ifnet *ifp, struct mbuf *m, int af, int hlen) */ crit_enter(); error = ifq_enqueue(&ifp->if_snd, m, &pktattr); - (*ifp->if_start)(ifp); + lwkt_serialize_enter(ifp->if_serializer); + ifp->if_start(ifp); + lwkt_serialize_exit(ifp->if_serializer); crit_exit(); return (error); } diff --git a/sys/net/if_var.h b/sys/net/if_var.h index 330cd1db6e..f7d87dd979 100644 --- a/sys/net/if_var.h +++ b/sys/net/if_var.h @@ -32,7 +32,7 @@ * * From: @(#)if.h 8.1 (Berkeley) 6/10/93 * $FreeBSD: src/sys/net/if_var.h,v 1.18.2.16 2003/04/15 18:11:19 fjoe Exp $ - * $DragonFly: src/sys/net/if_var.h,v 1.49 2008/05/02 07:40:32 sephe Exp $ + * $DragonFly: src/sys/net/if_var.h,v 1.50 2008/05/14 11:59:23 sephe Exp $ */ #ifndef _NET_IF_VAR_H_ @@ -88,6 +88,7 @@ struct lwkt_serialize; struct ifaddr_container; struct ifaddr; struct lwkt_port; +struct netmsg; #include /* get TAILQ macros */ @@ -159,14 +160,15 @@ enum poll_cmd { POLL_ONLY, POLL_AND_CHECK_STATUS, POLL_DEREGISTER, * * MPSAFE NOTES: * - * ifnet and its related packet queues are protected by if_serializer. - * Callers of if_output, if_ioctl, if_start, if_watchdog, if_init, + * ifnet is protected by if_serializer. ifnet.if_snd is protected by its + * own spinlock. Callers of if_ioctl, if_start, if_watchdog, if_init, * if_resolvemulti, and if_poll hold if_serializer. Device drivers usually * use the same serializer for their interrupt but this is not required. - * However, the device driver must be holding if_serializer when it - * calls if_input. Note that the serializer may be temporarily released - * within if_input to avoid a deadlock (e.g. when fast-forwarding or - * bridging packets between interfaces). + * Caller of if_output must not hold if_serializer; if_output should hold + * if_serializer based on its own needs. However, the device driver must + * be holding if_serializer when it calls if_input. Note that the serializer + * may be temporarily released within if_input to avoid a deadlock (e.g. when + * fast-forwarding or bridging packets between interfaces). * * If a device driver installs the same serializer for its interrupt * as for ifnet, then the driver only really needs to worry about further @@ -212,6 +214,8 @@ struct ifnet { (void *); int (*if_resolvemulti) /* validate/resolve multicast */ (struct ifnet *, struct sockaddr **, struct sockaddr *); + int (*if_start_cpuid) /* cpuid to run if_start */ + (struct ifnet *); #ifdef DEVICE_POLLING void (*if_poll) /* IFF_POLLING support */ (struct ifnet *, enum poll_cmd, int); @@ -229,6 +233,8 @@ struct ifnet { struct ifaddr *if_lladdr; struct lwkt_serialize *if_serializer; /* serializer or MP lock */ struct lwkt_serialize if_default_serializer; /* if not supplied */ + int if_cpuid; + struct netmsg *if_start_nmsg; /* percpu messages to schedule if_start */ }; typedef void if_init_f_t (void *); @@ -536,6 +542,7 @@ struct lwkt_port *ifa_portfn(int); struct ifmultiaddr *ifmaof_ifpforaddr(struct sockaddr *, struct ifnet *); int if_simloop(struct ifnet *ifp, struct mbuf *m, int af, int hlen); +void if_devstart(struct ifnet *ifp); #define IF_LLSOCKADDR(ifp) \ ((struct sockaddr_dl *)(ifp)->if_lladdr->ifa_addr) diff --git a/sys/net/ifq_var.h b/sys/net/ifq_var.h index cdbe3e5baf..b5cec8b3f5 100644 --- a/sys/net/ifq_var.h +++ b/sys/net/ifq_var.h @@ -28,7 +28,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sys/net/ifq_var.h,v 1.9 2007/08/21 19:21:54 corecode Exp $ + * $DragonFly: src/sys/net/ifq_var.h,v 1.10 2008/05/14 11:59:23 sephe Exp $ */ /* * NOTE ON MPSAFE access. Routines which manipulate the packet queue must @@ -68,6 +68,18 @@ struct ifaltq; +/* + * Support for non-ALTQ interfaces. + */ +int ifq_classic_enqueue(struct ifaltq *, struct mbuf *, + struct altq_pktattr *); +struct mbuf *ifq_classic_dequeue(struct ifaltq *, struct mbuf *, int); +int ifq_classic_request(struct ifaltq *, int, void *); +void ifq_set_classic(struct ifaltq *); + +int ifq_dispatch(struct ifnet *, struct mbuf *, + struct altq_pktattr *); + #ifdef ALTQ static __inline int ifq_is_enabled(struct ifaltq *_ifq) @@ -115,10 +127,27 @@ ifq_set_ready(struct ifaltq *_ifq) /* * WARNING: Should only be called in an MPSAFE manner. */ +static __inline int +ifq_enqueue_locked(struct ifaltq *_ifq, struct mbuf *_m, + struct altq_pktattr *_pa) +{ +#ifdef ALTQ + if (!ifq_is_enabled(_ifq)) + return ifq_classic_enqueue(_ifq, _m, _pa); + else +#endif + return _ifq->altq_enqueue(_ifq, _m, _pa); +} + static __inline int ifq_enqueue(struct ifaltq *_ifq, struct mbuf *_m, struct altq_pktattr *_pa) { - return((*_ifq->altq_enqueue)(_ifq, _m, _pa)); + int _error; + + ALTQ_LOCK(_ifq); + _error = ifq_enqueue_locked(_ifq, _m, _pa); + ALTQ_UNLOCK(_ifq); + return _error; } /* @@ -127,36 +156,87 @@ ifq_enqueue(struct ifaltq *_ifq, struct mbuf *_m, struct altq_pktattr *_pa) static __inline struct mbuf * ifq_dequeue(struct ifaltq *_ifq, struct mbuf *_mpolled) { + struct mbuf *_m; + + ALTQ_LOCK(_ifq); + if (_ifq->altq_prepended != NULL) { + _m = _ifq->altq_prepended; + _ifq->altq_prepended = NULL; + KKASSERT(_ifq->ifq_len > 0); + _ifq->ifq_len--; + ALTQ_UNLOCK(_ifq); + return _m; + } + #ifdef ALTQ if (_ifq->altq_tbr != NULL) - return(tbr_dequeue(_ifq, _mpolled, ALTDQ_REMOVE)); + _m = tbr_dequeue(_ifq, _mpolled, ALTDQ_REMOVE); + else if (!ifq_is_enabled(_ifq)) + _m = ifq_classic_dequeue(_ifq, _mpolled, ALTDQ_REMOVE); + else #endif - return((*_ifq->altq_dequeue)(_ifq, _mpolled, ALTDQ_REMOVE)); + _m = _ifq->altq_dequeue(_ifq, _mpolled, ALTDQ_REMOVE); + ALTQ_UNLOCK(_ifq); + return _m; } /* * WARNING: Should only be called in an MPSAFE manner. */ static __inline struct mbuf * -ifq_poll(struct ifaltq *_ifq) +ifq_poll_locked(struct ifaltq *_ifq) { + if (_ifq->altq_prepended != NULL) + return _ifq->altq_prepended; + #ifdef ALTQ if (_ifq->altq_tbr != NULL) - return(tbr_dequeue(_ifq, NULL, ALTDQ_POLL)); + return tbr_dequeue(_ifq, NULL, ALTDQ_POLL); + else if (!ifq_is_enabled(_ifq)) + return ifq_classic_dequeue(_ifq, NULL, ALTDQ_POLL); + else #endif - return((*_ifq->altq_dequeue)(_ifq, NULL, ALTDQ_POLL)); + return _ifq->altq_dequeue(_ifq, NULL, ALTDQ_POLL); +} + +static __inline struct mbuf * +ifq_poll(struct ifaltq *_ifq) +{ + struct mbuf *_m; + + ALTQ_LOCK(_ifq); + _m = ifq_poll_locked(_ifq); + ALTQ_UNLOCK(_ifq); + return _m; } /* * WARNING: Should only be called in an MPSAFE manner. */ static __inline void -ifq_purge(struct ifaltq *_ifq) +ifq_purge_locked(struct ifaltq *_ifq) { - if (ifq_is_enabled(_ifq)) - (*_ifq->altq_request)(_ifq, ALTRQ_PURGE, NULL); + if (_ifq->altq_prepended != NULL) { + m_freem(_ifq->altq_prepended); + _ifq->altq_prepended = NULL; + KKASSERT(_ifq->ifq_len > 0); + _ifq->ifq_len--; + } + +#ifdef ALTQ + if (!ifq_is_enabled(_ifq)) + ifq_classic_request(_ifq, ALTRQ_PURGE, NULL); else - IF_DRAIN(_ifq); +#endif + _ifq->altq_request(_ifq, ALTRQ_PURGE, NULL); +} + +static __inline void +ifq_purge(struct ifaltq *_ifq) +{ + ALTQ_LOCK(_ifq); + ifq_purge_locked(_ifq); + ALTQ_UNLOCK(_ifq); } /* @@ -166,12 +246,26 @@ static __inline void ifq_classify(struct ifaltq *_ifq, struct mbuf *_m, uint8_t _af, struct altq_pktattr *_pa) { - if (!ifq_is_enabled(_ifq)) - return; - _pa->pattr_af = _af; - _pa->pattr_hdr = mtod(_m, caddr_t); - if (_ifq->altq_flags & ALTQF_CLASSIFY) - (*_ifq->altq_classify)(_ifq, _m, _pa); +#ifdef ALTQ + ALTQ_LOCK(_ifq); + if (ifq_is_enabled(_ifq)) { + _pa->pattr_af = _af; + _pa->pattr_hdr = mtod(_m, caddr_t); + if (_ifq->altq_flags & ALTQF_CLASSIFY) + _ifq->altq_classify(_ifq, _m, _pa); + } + ALTQ_UNLOCK(_ifq); +#endif +} + +static __inline void +ifq_prepend(struct ifaltq *_ifq, struct mbuf *_m) +{ + ALTQ_LOCK(_ifq); + KASSERT(_ifq->altq_prepended == NULL, ("pending prepended mbuf\n")); + _ifq->altq_prepended = _m; + _ifq->ifq_len++; + ALTQ_UNLOCK(_ifq); } /* @@ -212,7 +306,16 @@ ifq_set_maxlen(struct ifaltq *_ifq, int _len) _ifq->ifq_maxlen = _len; } -void ifq_set_classic(struct ifaltq *); +static __inline int +ifq_data_ready(struct ifaltq *_ifq) +{ +#ifdef ALTQ + if (_ifq->altq_tbr != NULL) + return (ifq_poll_locked(_ifq) != NULL); + else +#endif + return !ifq_is_empty(_ifq); +} #endif /* _KERNEL */ #endif /* _NET_IFQ_VAR_H_ */ diff --git a/sys/net/pf/pf.c b/sys/net/pf/pf.c index 3f4fa4ee48..c1cce9b3fe 100644 --- a/sys/net/pf/pf.c +++ b/sys/net/pf/pf.c @@ -1,7 +1,7 @@ /* $FreeBSD: src/sys/contrib/pf/net/pf.c,v 1.19 2004/09/11 11:18:25 mlaier Exp $ */ /* $OpenBSD: pf.c,v 1.433.2.2 2004/07/17 03:22:34 brad Exp $ */ /* add $OpenBSD: pf.c,v 1.448 2004/05/11 07:34:11 dhartmei Exp $ */ -/* $DragonFly: src/sys/net/pf/pf.c,v 1.18 2008/04/11 18:21:48 dillon Exp $ */ +/* $DragonFly: src/sys/net/pf/pf.c,v 1.19 2008/05/14 11:59:23 sephe Exp $ */ /* * Copyright (c) 2004 The DragonFly Project. All rights reserved. @@ -5059,9 +5059,7 @@ pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, } } - lwkt_serialize_enter(ifp->if_serializer); - error = (*ifp->if_output)(ifp, m0, sintosa(dst), ro->ro_rt); - lwkt_serialize_exit(ifp->if_serializer); + error = ifp->if_output(ifp, m0, sintosa(dst), ro->ro_rt); goto done; } @@ -5088,10 +5086,7 @@ pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, m1 = m0->m_nextpkt; m0->m_nextpkt = 0; if (error == 0) { - lwkt_serialize_enter(ifp->if_serializer); - error = (*ifp->if_output)(ifp, m0, sintosa(dst), - NULL); - lwkt_serialize_exit(ifp->if_serializer); + error = ifp->if_output(ifp, m0, sintosa(dst), NULL); } else { m_freem(m0); } diff --git a/sys/net/ppp/if_ppp.c b/sys/net/ppp/if_ppp.c index 812d35ab45..90c2b89251 100644 --- a/sys/net/ppp/if_ppp.c +++ b/sys/net/ppp/if_ppp.c @@ -70,7 +70,7 @@ */ /* $FreeBSD: src/sys/net/if_ppp.c,v 1.67.2.4 2002/04/14 21:41:48 luigi Exp $ */ -/* $DragonFly: src/sys/net/ppp/if_ppp.c,v 1.37 2008/01/06 16:55:52 swildner Exp $ */ +/* $DragonFly: src/sys/net/ppp/if_ppp.c,v 1.38 2008/05/14 11:59:23 sephe Exp $ */ /* from if_sl.c,v 1.11 84/10/04 12:54:47 rick Exp */ /* from NetBSD: if_ppp.c,v 1.15.2.2 1994/07/28 05:17:58 cgd Exp */ @@ -704,9 +704,9 @@ pppsioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cr) * Called at splnet as the if->if_output handler. * Called at splnet from pppwrite(). */ -int -pppoutput(struct ifnet *ifp, struct mbuf *m0, struct sockaddr *dst, - struct rtentry *rtp) +static int +pppoutput_serialized(struct ifnet *ifp, struct mbuf *m0, struct sockaddr *dst, + struct rtentry *rtp) { struct ppp_softc *sc = &ppp_softc[ifp->if_dunit]; int protocol, address, control; @@ -892,6 +892,19 @@ bad: return (error); } +int +pppoutput(struct ifnet *ifp, struct mbuf *m0, struct sockaddr *dst, + struct rtentry *rtp) +{ + int error; + + lwkt_serialize_enter(ifp->if_serializer); + error = pppoutput_serialized(ifp, m0, dst, rtp); + lwkt_serialize_exit(ifp->if_serializer); + + return error; +} + /* * After a change in the NPmode for some NP, move packets from the * npqueue to the send queue or the fast queue as appropriate. diff --git a/sys/net/ppp_layer/ppp_tty.c b/sys/net/ppp_layer/ppp_tty.c index f6beccfea0..b101e8edcb 100644 --- a/sys/net/ppp_layer/ppp_tty.c +++ b/sys/net/ppp_layer/ppp_tty.c @@ -71,7 +71,7 @@ */ /* $FreeBSD: src/sys/net/ppp_tty.c,v 1.43.2.1 2002/02/13 00:43:11 dillon Exp $ */ -/* $DragonFly: src/sys/net/ppp_layer/ppp_tty.c,v 1.24 2007/08/24 16:06:37 dillon Exp $ */ +/* $DragonFly: src/sys/net/ppp_layer/ppp_tty.c,v 1.25 2008/05/14 11:59:23 sephe Exp $ */ #include "opt_ppp.h" /* XXX for ppp_defs.h */ @@ -411,9 +411,7 @@ pppwrite(struct tty *tp, struct uio *uio, int flag) m0->m_len -= PPP_HDRLEN; /* call the upper layer to "transmit" it... */ - lwkt_serialize_enter(sc->sc_if.if_serializer); error = pppoutput(&sc->sc_if, m0, &dst, (struct rtentry *)0); - lwkt_serialize_exit(sc->sc_if.if_serializer); crit_exit(); return (error); } diff --git a/sys/net/sl/if_sl.c b/sys/net/sl/if_sl.c index 2d130a0505..aa65c0298b 100644 --- a/sys/net/sl/if_sl.c +++ b/sys/net/sl/if_sl.c @@ -32,7 +32,7 @@ * * @(#)if_sl.c 8.6 (Berkeley) 2/1/94 * $FreeBSD: src/sys/net/if_sl.c,v 1.84.2.2 2002/02/13 00:43:10 dillon Exp $ - * $DragonFly: src/sys/net/sl/if_sl.c,v 1.31 2006/12/22 23:44:57 swildner Exp $ + * $DragonFly: src/sys/net/sl/if_sl.c,v 1.32 2008/05/14 11:59:23 sephe Exp $ */ /* @@ -439,8 +439,8 @@ sltioctl(struct tty *tp, u_long cmd, caddr_t data, int flag, struct ucred *cred) * ordering gets trashed. It can be done for all packets in slstart. */ static int -sloutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, - struct rtentry *rtp) +sloutput_serialized(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, + struct rtentry *rtp) { struct sl_softc *sc = &sl_softc[ifp->if_dunit]; struct ip *ip; @@ -500,6 +500,19 @@ sloutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, return (0); } +static int +sloutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, + struct rtentry *rtp) +{ + int error; + + lwkt_serialize_enter(ifp->if_serializer); + error = sloutput_serialized(ifp, m, dst, rtp); + lwkt_serialize_exit(ifp->if_serializer); + + return error; +} + /* * Start output on interface. Get another datagram * to send from the interface queue and map it to diff --git a/sys/net/sppp/if_spppsubr.c b/sys/net/sppp/if_spppsubr.c index bd7f18fe18..18f37f648a 100644 --- a/sys/net/sppp/if_spppsubr.c +++ b/sys/net/sppp/if_spppsubr.c @@ -18,7 +18,7 @@ * From: Version 2.4, Thu Apr 30 17:17:21 MSD 1997 * * $FreeBSD: src/sys/net/if_spppsubr.c,v 1.59.2.13 2002/07/03 15:44:41 joerg Exp $ - * $DragonFly: src/sys/net/sppp/if_spppsubr.c,v 1.33 2008/04/20 13:44:25 swildner Exp $ + * $DragonFly: src/sys/net/sppp/if_spppsubr.c,v 1.34 2008/05/14 11:59:24 sephe Exp $ */ #include @@ -741,8 +741,8 @@ sppp_input(struct ifnet *ifp, struct mbuf *m) * Enqueue transmit packet. */ static int -sppp_output(struct ifnet *ifp, struct mbuf *m, - struct sockaddr *dst, struct rtentry *rt) +sppp_output_serialized(struct ifnet *ifp, struct mbuf *m, + struct sockaddr *dst, struct rtentry *rt) { struct sppp *sp = (struct sppp*) ifp; struct ppp_header *h; @@ -997,6 +997,19 @@ sppp_output(struct ifnet *ifp, struct mbuf *m, return (0); } +static int +sppp_output(struct ifnet *ifp, struct mbuf *m, + struct sockaddr *dst, struct rtentry *rt) +{ + int error; + + lwkt_serialize_enter(ifp->if_serializer); + error = sppp_output_serialized(ifp, m, dst, rt); + lwkt_serialize_exit(ifp->if_serializer); + + return error; +} + void sppp_attach(struct ifnet *ifp) { diff --git a/sys/net/stf/if_stf.c b/sys/net/stf/if_stf.c index 7616811951..1419b5797a 100644 --- a/sys/net/stf/if_stf.c +++ b/sys/net/stf/if_stf.c @@ -1,5 +1,5 @@ /* $FreeBSD: src/sys/net/if_stf.c,v 1.1.2.11 2003/01/23 21:06:44 sam Exp $ */ -/* $DragonFly: src/sys/net/stf/if_stf.c,v 1.21 2008/03/07 11:34:20 sephe Exp $ */ +/* $DragonFly: src/sys/net/stf/if_stf.c,v 1.22 2008/05/14 11:59:24 sephe Exp $ */ /* $KAME: if_stf.c,v 1.73 2001/12/03 11:08:30 keiichi Exp $ */ /* @@ -303,8 +303,8 @@ stf_getsrcifa6(struct ifnet *ifp) } static int -stf_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, - struct rtentry *rt) +stf_output_serialized(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, + struct rtentry *rt) { struct stf_softc *sc; struct sockaddr_in6 *dst6; @@ -404,6 +404,19 @@ stf_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, return ip_output(m, NULL, &sc->sc_ro, 0, NULL, NULL); } +static int +stf_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, + struct rtentry *rt) +{ + int error; + + lwkt_serialize_enter(ifp->if_serializer); + error = stf_output_serialized(ifp, m, dst, rt); + lwkt_serialize_exit(ifp->if_serializer); + + return error; +} + /* * Parameters: * inifp: incoming interface diff --git a/sys/net/tun/if_tun.c b/sys/net/tun/if_tun.c index e2f67feb26..015f0eaed3 100644 --- a/sys/net/tun/if_tun.c +++ b/sys/net/tun/if_tun.c @@ -14,7 +14,7 @@ * operation though. * * $FreeBSD: src/sys/net/if_tun.c,v 1.74.2.8 2002/02/13 00:43:11 dillon Exp $ - * $DragonFly: src/sys/net/tun/if_tun.c,v 1.35 2008/04/05 06:28:35 sephe Exp $ + * $DragonFly: src/sys/net/tun/if_tun.c,v 1.36 2008/05/14 11:59:24 sephe Exp $ */ #include "opt_atalk.h" @@ -273,8 +273,8 @@ tunifioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cr) * MPSAFE */ static int -tunoutput(struct ifnet *ifp, struct mbuf *m0, struct sockaddr *dst, - struct rtentry *rt) +tunoutput_serialized(struct ifnet *ifp, struct mbuf *m0, struct sockaddr *dst, + struct rtentry *rt) { struct tun_softc *tp = ifp->if_softc; int error; @@ -366,6 +366,19 @@ tunoutput(struct ifnet *ifp, struct mbuf *m0, struct sockaddr *dst, return (error); } +static int +tunoutput(struct ifnet *ifp, struct mbuf *m0, struct sockaddr *dst, + struct rtentry *rt) +{ + int error; + + lwkt_serialize_enter(ifp->if_serializer); + error = tunoutput_serialized(ifp, m0, dst, rt); + lwkt_serialize_exit(ifp->if_serializer); + + return error; +} + /* * the ops interface is now pretty minimal. */ diff --git a/sys/netgraph/fec/ng_fec.c b/sys/netgraph/fec/ng_fec.c index c2df965c16..3eeb76ac13 100644 --- a/sys/netgraph/fec/ng_fec.c +++ b/sys/netgraph/fec/ng_fec.c @@ -33,7 +33,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/netgraph/ng_fec.c,v 1.1.2.1 2002/11/01 21:39:31 julian Exp $ - * $DragonFly: src/sys/netgraph/fec/ng_fec.c,v 1.24 2008/03/19 14:46:03 sephe Exp $ + * $DragonFly: src/sys/netgraph/fec/ng_fec.c,v 1.25 2008/05/14 11:59:24 sephe Exp $ */ /* * Copyright (c) 1996-1999 Whistle Communications, Inc. @@ -772,8 +772,8 @@ ng_fec_input(struct ifnet *ifp, struct mbuf **m0) */ static int -ng_fec_output(struct ifnet *ifp, struct mbuf *m, - struct sockaddr *dst, struct rtentry *rt0) +ng_fec_output_serialized(struct ifnet *ifp, struct mbuf *m, + struct sockaddr *dst, struct rtentry *rt0) { const priv_p priv = (priv_p) ifp->if_softc; struct ng_fec_bundle *b; @@ -833,6 +833,19 @@ ng_fec_output(struct ifnet *ifp, struct mbuf *m, return(error); } +static int +ng_fec_output(struct ifnet *ifp, struct mbuf *m, + struct sockaddr *dst, struct rtentry *rt0) +{ + int error; + + lwkt_serialize_enter(ifp->if_serializer); + error = ng_fec_output_serialized(ifp, m, dst, rt0); + lwkt_serialize_exit(ifp->if_serializer); + + return error; +} + /* * Apply a hash to the source and destination addresses in the packet * in order to select an interface. Also check link status and handle diff --git a/sys/netgraph/iface/ng_iface.c b/sys/netgraph/iface/ng_iface.c index 8d34693ff6..66dc7b8667 100644 --- a/sys/netgraph/iface/ng_iface.c +++ b/sys/netgraph/iface/ng_iface.c @@ -37,7 +37,7 @@ * Author: Archie Cobbs * * $FreeBSD: src/sys/netgraph/ng_iface.c,v 1.7.2.5 2002/07/02 23:44:02 archie Exp $ - * $DragonFly: src/sys/netgraph/iface/ng_iface.c,v 1.15 2008/03/07 11:34:20 sephe Exp $ + * $DragonFly: src/sys/netgraph/iface/ng_iface.c,v 1.16 2008/05/14 11:59:24 sephe Exp $ * $Whistle: ng_iface.c,v 1.33 1999/11/01 09:24:51 julian Exp $ */ @@ -409,8 +409,8 @@ ng_iface_ioctl(struct ifnet *ifp, u_long command, caddr_t data, */ static int -ng_iface_output(struct ifnet *ifp, struct mbuf *m, - struct sockaddr *dst, struct rtentry *rt0) +ng_iface_output_serialized(struct ifnet *ifp, struct mbuf *m, + struct sockaddr *dst, struct rtentry *rt0) { const priv_p priv = (priv_p) ifp->if_softc; const iffam_p iffam = get_iffam_from_af(dst->sa_family); @@ -458,6 +458,19 @@ ng_iface_output(struct ifnet *ifp, struct mbuf *m, return (error); } +static int +ng_iface_output(struct ifnet *ifp, struct mbuf *m, + struct sockaddr *dst, struct rtentry *rt0) +{ + int error; + + lwkt_serialize_enter(ifp->if_serializer); + error = ng_iface_output_serialized(ifp, m, dst, rt0); + lwkt_serialize_exit(ifp->if_serializer); + + return error; +} + /* * This routine should never be called */ diff --git a/sys/netinet/if_ether.c b/sys/netinet/if_ether.c index 94e96e15ec..b5c5225c02 100644 --- a/sys/netinet/if_ether.c +++ b/sys/netinet/if_ether.c @@ -64,7 +64,7 @@ * * @(#)if_ether.c 8.1 (Berkeley) 6/10/93 * $FreeBSD: src/sys/netinet/if_ether.c,v 1.64.2.23 2003/04/11 07:23:15 fjoe Exp $ - * $DragonFly: src/sys/netinet/if_ether.c,v 1.45 2008/03/07 11:34:20 sephe Exp $ + * $DragonFly: src/sys/netinet/if_ether.c,v 1.46 2008/05/14 11:59:24 sephe Exp $ */ /* @@ -224,12 +224,10 @@ arp_rtrequest(int req, struct rtentry *rt, struct rt_addrinfo *info) } /* Announce a new entry if requested. */ if (rt->rt_flags & RTF_ANNOUNCE) { - lwkt_serialize_enter(rt->rt_ifp->if_serializer); arprequest(rt->rt_ifp, &SIN(rt_key(rt))->sin_addr, &SIN(rt_key(rt))->sin_addr, LLADDR(SDL(gate))); - lwkt_serialize_exit(rt->rt_ifp->if_serializer); } /*FALLTHROUGH*/ case RTM_RESOLVE: @@ -327,8 +325,6 @@ arprequest(struct ifnet *ifp, struct in_addr *sip, struct in_addr *tip, struct sockaddr sa; u_short ar_hrd; - ASSERT_SERIALIZED(ifp->if_serializer); - if ((m = m_gethdr(MB_DONTWAIT, MT_DATA)) == NULL) return; m->m_pkthdr.rcvif = (struct ifnet *)NULL; @@ -367,7 +363,7 @@ arprequest(struct ifnet *ifp, struct in_addr *sip, struct in_addr *tip, sa.sa_family = AF_UNSPEC; sa.sa_len = sizeof sa; - (*ifp->if_output)(ifp, m, &sa, (struct rtentry *)NULL); + ifp->if_output(ifp, m, &sa, (struct rtentry *)NULL); } /* @@ -634,9 +630,7 @@ arp_update_oncpu(struct mbuf *m, in_addr_t saddr, boolean_t create, */ if (la->la_hold != NULL) { m_adj(la->la_hold, sizeof(struct ether_header)); - lwkt_serialize_enter(ifp->if_serializer); - (*ifp->if_output)(ifp, la->la_hold, rt_key(rt), rt); - lwkt_serialize_exit(ifp->if_serializer); + ifp->if_output(ifp, la->la_hold, rt_key(rt), rt); la->la_hold = NULL; } } @@ -859,9 +853,7 @@ reply: } sa.sa_family = AF_UNSPEC; sa.sa_len = sizeof sa; - lwkt_serialize_enter(ifp->if_serializer); - (*ifp->if_output)(ifp, m, &sa, (struct rtentry *)0); - lwkt_serialize_exit(ifp->if_serializer); + ifp->if_output(ifp, m, &sa, (struct rtentry *)0); return; } diff --git a/sys/netinet/ip_flow.c b/sys/netinet/ip_flow.c index 57977cfbb2..cd71e6bc14 100644 --- a/sys/netinet/ip_flow.c +++ b/sys/netinet/ip_flow.c @@ -34,7 +34,7 @@ * POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: src/sys/netinet/ip_flow.c,v 1.9.2.2 2001/11/04 17:35:31 luigi Exp $ - * $DragonFly: src/sys/netinet/ip_flow.c,v 1.13 2008/04/03 16:02:32 sephe Exp $ + * $DragonFly: src/sys/netinet/ip_flow.c,v 1.14 2008/05/14 11:59:24 sephe Exp $ */ #include @@ -183,7 +183,6 @@ ipflow_fastforward(struct mbuf *m, lwkt_serialize_t serializer) if (serializer) lwkt_serialize_exit(serializer); - lwkt_serialize_enter(ifp->if_serializer); error = ifp->if_output(ifp, m, dst, rt); if (error) { if (error == ENOBUFS) @@ -191,7 +190,6 @@ ipflow_fastforward(struct mbuf *m, lwkt_serialize_t serializer) else ipf->ipf_errors++; } - lwkt_serialize_exit(ifp->if_serializer); if (serializer) lwkt_serialize_enter(serializer); diff --git a/sys/netinet/ip_output.c b/sys/netinet/ip_output.c index 9278ffa853..34e754e29e 100644 --- a/sys/netinet/ip_output.c +++ b/sys/netinet/ip_output.c @@ -28,7 +28,7 @@ * * @(#)ip_output.c 8.3 (Berkeley) 1/21/94 * $FreeBSD: src/sys/netinet/ip_output.c,v 1.99.2.37 2003/04/15 06:44:45 silby Exp $ - * $DragonFly: src/sys/netinet/ip_output.c,v 1.41 2008/01/06 16:55:52 swildner Exp $ + * $DragonFly: src/sys/netinet/ip_output.c,v 1.42 2008/05/14 11:59:24 sephe Exp $ */ #define _IP_VHL @@ -1017,10 +1017,8 @@ pass: m->m_pkthdr.len = tmp; } #endif - lwkt_serialize_enter(ifp->if_serializer); - error = (*ifp->if_output)(ifp, m, (struct sockaddr *)dst, - ro->ro_rt); - lwkt_serialize_exit(ifp->if_serializer); + error = ifp->if_output(ifp, m, (struct sockaddr *)dst, + ro->ro_rt); goto done; } @@ -1062,11 +1060,8 @@ pass: ia->ia_ifa.if_opackets++; ia->ia_ifa.if_obytes += m->m_pkthdr.len; } - lwkt_serialize_enter(ifp->if_serializer); - error = (*ifp->if_output)(ifp, m, - (struct sockaddr *)dst, - ro->ro_rt); - lwkt_serialize_exit(ifp->if_serializer); + error = ifp->if_output(ifp, m, (struct sockaddr *)dst, + ro->ro_rt); } else { m_freem(m); } diff --git a/sys/netinet6/ip6_mroute.c b/sys/netinet6/ip6_mroute.c index d6f0261531..2285dd5ac1 100644 --- a/sys/netinet6/ip6_mroute.c +++ b/sys/netinet6/ip6_mroute.c @@ -1,5 +1,5 @@ /* $FreeBSD: src/sys/netinet6/ip6_mroute.c,v 1.2.2.9 2003/01/23 21:06:47 sam Exp $ */ -/* $DragonFly: src/sys/netinet6/ip6_mroute.c,v 1.15 2007/04/22 01:13:14 dillon Exp $ */ +/* $DragonFly: src/sys/netinet6/ip6_mroute.c,v 1.16 2008/05/14 11:59:24 sephe Exp $ */ /* $KAME: ip6_mroute.c,v 1.58 2001/12/18 02:36:31 itojun Exp $ */ /* @@ -1446,10 +1446,8 @@ phyint_send(struct ip6_hdr *ip6, struct mif6 *mifp, struct mbuf *m) * We just call if_output instead of nd6_output here, since * we need no ND for a multicast forwarded packet...right? */ - lwkt_serialize_enter(ifp->if_serializer); - error = (*ifp->if_output)(ifp, mb_copy, + error = ifp->if_output(ifp, mb_copy, (struct sockaddr *)&ro.ro_dst, NULL); - lwkt_serialize_exit(ifp->if_serializer); #ifdef MRT6DEBUG if (mrt6debug & DEBUG_XMIT) log(LOG_DEBUG, "phyint_send on mif %d err %d\n", diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c index 0a78994abc..5de319aefb 100644 --- a/sys/netinet6/nd6.c +++ b/sys/netinet6/nd6.c @@ -1,5 +1,5 @@ /* $FreeBSD: src/sys/netinet6/nd6.c,v 1.2.2.15 2003/05/06 06:46:58 suz Exp $ */ -/* $DragonFly: src/sys/netinet6/nd6.c,v 1.28 2008/04/05 07:40:28 sephe Exp $ */ +/* $DragonFly: src/sys/netinet6/nd6.c,v 1.29 2008/05/14 11:59:24 sephe Exp $ */ /* $KAME: nd6.c,v 1.144 2001/05/24 07:44:00 itojun Exp $ */ /* @@ -1953,15 +1953,10 @@ nd6_output(struct ifnet *ifp, struct ifnet *origifp, struct mbuf *m, return (0); sendpkt: - if (ifp->if_flags & IFF_LOOPBACK) { - lwkt_serialize_enter(origifp->if_serializer); + if (ifp->if_flags & IFF_LOOPBACK) error = ifp->if_output(origifp, m, (struct sockaddr *)dst, rt); - lwkt_serialize_exit(origifp->if_serializer); - } else { - lwkt_serialize_enter(ifp->if_serializer); + else error = ifp->if_output(ifp, m, (struct sockaddr *)dst, rt); - lwkt_serialize_exit(ifp->if_serializer); - } return (error); bad: diff --git a/sys/netproto/atalk/aarp.c b/sys/netproto/atalk/aarp.c index e0a90de71d..86f3632492 100644 --- a/sys/netproto/atalk/aarp.c +++ b/sys/netproto/atalk/aarp.c @@ -3,7 +3,7 @@ * All Rights Reserved. * * $FreeBSD: src/sys/netatalk/aarp.c,v 1.12.2.2 2001/06/23 20:43:09 iedowse Exp $ - * $DragonFly: src/sys/netproto/atalk/aarp.c,v 1.23 2008/04/05 07:57:22 sephe Exp $ + * $DragonFly: src/sys/netproto/atalk/aarp.c,v 1.24 2008/05/14 11:59:24 sephe Exp $ */ #include "opt_atalk.h" @@ -190,7 +190,7 @@ aarpwhohas(struct arpcom *ac, struct sockaddr_at *sat) sa.sa_len = sizeof( struct sockaddr ); sa.sa_family = AF_UNSPEC; - (*ac->ac_if.if_output)(&ac->ac_if, + ac->ac_if.if_output(&ac->ac_if, m, &sa, NULL); /* XXX NULL should be routing information */ } @@ -397,10 +397,8 @@ at_aarpinput( struct arpcom *ac, struct mbuf *m) sat.sat_len = sizeof(struct sockaddr_at); sat.sat_family = AF_APPLETALK; sat.sat_addr = spa; - lwkt_serialize_enter(ac->ac_if.if_serializer); - (*ac->ac_if.if_output)( &ac->ac_if, mhold, + ac->ac_if.if_output(&ac->ac_if, mhold, (struct sockaddr *)&sat, NULL); /* XXX */ - lwkt_serialize_exit(ac->ac_if.if_serializer); } } else if ((tpa.s_net == ma.s_net) && (tpa.s_node == ma.s_node) @@ -456,9 +454,7 @@ at_aarpinput( struct arpcom *ac, struct mbuf *m) sa.sa_len = sizeof( struct sockaddr ); sa.sa_family = AF_UNSPEC; - lwkt_serialize_enter(ac->ac_if.if_serializer); - (*ac->ac_if.if_output)( &ac->ac_if, m, &sa, NULL); /* XXX */ - lwkt_serialize_exit(ac->ac_if.if_serializer); + ac->ac_if.if_output(&ac->ac_if, m, &sa, NULL); /* XXX */ return; } @@ -601,9 +597,7 @@ aarpprobe(void *arg) sa.sa_len = sizeof( struct sockaddr ); sa.sa_family = AF_UNSPEC; - lwkt_serialize_enter(ac->ac_if.if_serializer); - (*ac->ac_if.if_output)(&ac->ac_if, m, &sa, NULL); /* XXX */ - lwkt_serialize_exit(ac->ac_if.if_serializer); + ac->ac_if.if_output(&ac->ac_if, m, &sa, NULL); /* XXX */ aa->aa_probcnt--; } diff --git a/sys/netproto/atalk/ddp_output.c b/sys/netproto/atalk/ddp_output.c index 98dca87c90..900b0de297 100644 --- a/sys/netproto/atalk/ddp_output.c +++ b/sys/netproto/atalk/ddp_output.c @@ -22,7 +22,7 @@ */ /* $FreeBSD: src/sys/netatalk/ddp_output.c,v 1.13.6.1 2000/06/02 22:39:07 archie Exp $ */ -/* $DragonFly: src/sys/netproto/atalk/ddp_output.c,v 1.9 2006/12/22 23:57:53 swildner Exp $ */ +/* $DragonFly: src/sys/netproto/atalk/ddp_output.c,v 1.10 2008/05/14 11:59:24 sephe Exp $ */ #include #include @@ -230,8 +230,6 @@ ddp_route( struct mbuf *m, struct route *ro) } /* XXX */ - lwkt_serialize_enter(ifp->if_serializer); - error = (*ifp->if_output)( ifp, m, (struct sockaddr *)&gate, NULL); - lwkt_serialize_exit(ifp->if_serializer); + error = ifp->if_output(ifp, m, (struct sockaddr *)&gate, NULL); return (error); } diff --git a/sys/netproto/atm/atm_if.c b/sys/netproto/atm/atm_if.c index 1272bc183f..bedb637603 100644 --- a/sys/netproto/atm/atm_if.c +++ b/sys/netproto/atm/atm_if.c @@ -24,7 +24,7 @@ * notice must be reproduced on all copies. * * @(#) $FreeBSD: src/sys/netatm/atm_if.c,v 1.5 1999/08/28 00:48:35 peter Exp $ - * @(#) $DragonFly: src/sys/netproto/atm/atm_if.c,v 1.16 2008/03/07 11:34:21 sephe Exp $ + * @(#) $DragonFly: src/sys/netproto/atm/atm_if.c,v 1.17 2008/05/14 11:59:24 sephe Exp $ */ /* @@ -942,9 +942,9 @@ atm_nif_setaddr(struct atm_nif *nip, struct ifaddr *ifa) * errno output failed - reason indicated * */ -int -atm_ifoutput(struct ifnet *ifp, KBuffer *m, struct sockaddr *dst, - struct rtentry *rt) +static int +atm_ifoutput_serialized(struct ifnet *ifp, KBuffer *m, struct sockaddr *dst, + struct rtentry *rt) { u_short fam = dst->sa_family; int (*func)(struct ifnet *, KBuffer *, @@ -969,6 +969,18 @@ atm_ifoutput(struct ifnet *ifp, KBuffer *m, struct sockaddr *dst, return ((*func)(ifp, m, dst)); } +int +atm_ifoutput(struct ifnet *ifp, KBuffer *m, struct sockaddr *dst, + struct rtentry *rt) +{ + int error; + + lwkt_serialize_enter(ifp->if_serializer); + error = atm_ifoutput_serialized(ifp, m, dst, rt); + lwkt_serialize_exit(ifp->if_serializer); + + return error; +} /* * Handle interface ioctl requests. diff --git a/sys/netproto/ipx/ipx_ip.c b/sys/netproto/ipx/ipx_ip.c index 4f56f29aa4..83be382148 100644 --- a/sys/netproto/ipx/ipx_ip.c +++ b/sys/netproto/ipx/ipx_ip.c @@ -34,7 +34,7 @@ * @(#)ipx_ip.c * * $FreeBSD: src/sys/netipx/ipx_ip.c,v 1.24.2.2 2003/01/23 21:06:48 sam Exp $ - * $DragonFly: src/sys/netproto/ipx/ipx_ip.c,v 1.16 2006/01/14 13:36:40 swildner Exp $ + * $DragonFly: src/sys/netproto/ipx/ipx_ip.c,v 1.17 2008/05/14 11:59:24 sephe Exp $ */ /* @@ -222,8 +222,8 @@ ipxip_input(struct mbuf *m, ...) } static int -ipxipoutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, - struct rtentry *rt) +ipxipoutput_serialized(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, + struct rtentry *rt) { struct ifnet_en *ifn = (struct ifnet_en *)ifp; struct ip *ip; @@ -285,6 +285,19 @@ ipxipoutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, return (ENETUNREACH); } +static int +ipxipoutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, + struct rtentry *rt) +{ + int error; + + lwkt_serialize_enter(ifp->if_serializer); + error = ipxipoutput_serialized(ifp, m, dst, rt); + lwkt_serialize_exit(ifp->if_serializer); + + return error; +} + static void ipxipstart(struct ifnet *ifp) { diff --git a/sys/netproto/ipx/ipx_outputfl.c b/sys/netproto/ipx/ipx_outputfl.c index 9e77d6c56e..c9071aa8b7 100644 --- a/sys/netproto/ipx/ipx_outputfl.c +++ b/sys/netproto/ipx/ipx_outputfl.c @@ -34,7 +34,7 @@ * @(#)ipx_outputfl.c * * $FreeBSD: src/sys/netipx/ipx_outputfl.c,v 1.14.2.1 2000/05/01 01:10:24 bp Exp $ - * $DragonFly: src/sys/netproto/ipx/ipx_outputfl.c,v 1.8 2006/01/14 13:36:40 swildner Exp $ + * $DragonFly: src/sys/netproto/ipx/ipx_outputfl.c,v 1.9 2008/05/14 11:59:24 sephe Exp $ */ #include @@ -128,10 +128,8 @@ gotif: if (ipx_copy_output) { ipx_watch_output(m0, ifp); } - lwkt_serialize_enter(ifp->if_serializer); - error = (*ifp->if_output)(ifp, m0, - (struct sockaddr *)dst, ro->ro_rt); - lwkt_serialize_exit(ifp->if_serializer); + error = ifp->if_output(ifp, m0, (struct sockaddr *)dst, + ro->ro_rt); goto done; } else { ipxstat.ipxs_mtutoosmall++; @@ -245,10 +243,8 @@ ipx_output_type20(struct mbuf *m) m1 = m_copym(m, 0, M_COPYALL, MB_DONTWAIT); if(m1) { - lwkt_serialize_enter(ifp->if_serializer); - error = (*ifp->if_output)(ifp, m1, + error = ifp->if_output(ifp, m1, (struct sockaddr *)&dst, NULL); - lwkt_serialize_exit(ifp->if_serializer); /* XXX ipxstat.ipxs_localout++; */ } skip_this: ; diff --git a/sys/netproto/natm/natm.c b/sys/netproto/natm/natm.c index c65bcbc3ba..e25dcff654 100644 --- a/sys/netproto/natm/natm.c +++ b/sys/netproto/natm/natm.c @@ -1,6 +1,6 @@ /* $NetBSD: natm.c,v 1.5 1996/11/09 03:26:26 chuck Exp $ */ /* $FreeBSD: src/sys/netnatm/natm.c,v 1.12 2000/02/13 03:32:03 peter Exp $ */ -/* $DragonFly: src/sys/netproto/natm/natm.c,v 1.28 2008/04/05 05:30:29 sephe Exp $ */ +/* $DragonFly: src/sys/netproto/natm/natm.c,v 1.29 2008/05/14 11:59:24 sephe Exp $ */ /* * @@ -320,9 +320,7 @@ natm_usr_send(struct socket *so, int flags, struct mbuf *m, ATM_PH_SETVCI(aph, npcb->npcb_vci); ATM_PH_FLAGS(aph) = (proto == PROTO_NATMAAL5) ? ATM_PH_AAL5 : 0; - lwkt_serialize_enter(npcb->npcb_ifp->if_serializer); error = atm_output(npcb->npcb_ifp, m, NULL, NULL); - lwkt_serialize_exit(npcb->npcb_ifp->if_serializer); out: crit_exit(); @@ -632,9 +630,7 @@ natm_usrreq(struct socket *so, int req, struct mbuf *m, struct mbuf *nam, ATM_PH_SETVCI(aph, npcb->npcb_vci); ATM_PH_FLAGS(aph) = (proto == PROTO_NATMAAL5) ? ATM_PH_AAL5 : 0; - lwkt_serialize_enter(npcb->npcb_ifp->if_serializer); error = atm_output(npcb->npcb_ifp, m, NULL, NULL); - lwkt_serialize_exit(npcb->npcb_ifp->if_serializer); break; diff --git a/sys/netproto/ns/ns_ip.c b/sys/netproto/ns/ns_ip.c index c6ed12526c..440426f69b 100644 --- a/sys/netproto/ns/ns_ip.c +++ b/sys/netproto/ns/ns_ip.c @@ -32,7 +32,7 @@ * * @(#)ns_ip.c 8.1 (Berkeley) 6/10/93 * $FreeBSD: src/sys/netns/ns_ip.c,v 1.9 1999/08/28 00:49:50 peter Exp $ - * $DragonFly: src/sys/netproto/ns/ns_ip.c,v 1.14 2006/01/14 13:36:40 swildner Exp $ + * $DragonFly: src/sys/netproto/ns/ns_ip.c,v 1.15 2008/05/14 11:59:24 sephe Exp $ */ /* @@ -104,6 +104,7 @@ nsipattach(void) nsip_list = m; ifp = &m->ifen_ifnet; + ifp->if_softc = m; if_initname(ifp, "nsip", nsipif_units++); ifp->if_name = "nsip"; ifp->if_mtu = LOMTU; @@ -230,8 +231,9 @@ idpip_input(struct mbuf *m, ...) } /* ARGSUSED */ -int -nsipoutput(struct ifnet_en *ifn, struct mbuf *m, struct sockaddr *dst) +static int +nsipoutput_serialized(struct ifnet_en *ifn, struct mbuf *m, + struct sockaddr *dst) { struct ip *ip; @@ -293,6 +295,19 @@ bad: return (ENETUNREACH); } +int +nsipoutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, + struct rtentry *rt __unused) +{ + int error; + + lwkt_serialize_enter(ifp->if_serializer); + error = nsipoutput_serialized(ifp->if_softc, m, dst); + lwkt_serialize_exit(ifp->if_serializer); + + return error; +} + void nsipstart(struct ifnet *ifp) { diff --git a/sys/netproto/ns/ns_output.c b/sys/netproto/ns/ns_output.c index 4f79781e68..aa9b2eae14 100644 --- a/sys/netproto/ns/ns_output.c +++ b/sys/netproto/ns/ns_output.c @@ -32,7 +32,7 @@ * * @(#)ns_output.c 8.1 (Berkeley) 6/10/93 * $FreeBSD: src/sys/netns/ns_output.c,v 1.7 1999/08/28 00:49:51 peter Exp $ - * $DragonFly: src/sys/netproto/ns/ns_output.c,v 1.8 2006/01/14 13:36:40 swildner Exp $ + * $DragonFly: src/sys/netproto/ns/ns_output.c,v 1.9 2008/05/14 11:59:24 sephe Exp $ */ #include @@ -136,10 +136,8 @@ gotif: if (ns_copy_output) { ns_watch_output(m0, ifp); } - lwkt_serialize_enter(ifp->if_serializer); - error = (*ifp->if_output)(ifp, m0, - (struct sockaddr *)dst, ro->ro_rt); - lwkt_serialize_exit(ifp->if_serializer); + error = ifp->if_output(ifp, m0, (struct sockaddr *)dst, + ro->ro_rt); goto done; } else error = EMSGSIZE; diff --git a/sys/sys/interrupt.h b/sys/sys/interrupt.h index 7e0693a584..dbe1a1cba2 100644 --- a/sys/sys/interrupt.h +++ b/sys/sys/interrupt.h @@ -57,7 +57,7 @@ * * $FreeBSD: src/sys/sys/interrupt.h,v 1.9.2.1 2001/10/14 20:05:50 luigi Exp $ * $FreeBSD: src/sys/i386/include/ipl.h,v 1.17.2.3 2002/12/17 18:04:02 sam Exp $ - * $DragonFly: src/sys/sys/interrupt.h,v 1.18 2007/01/11 23:23:57 dillon Exp $ + * $DragonFly: src/sys/sys/interrupt.h,v 1.19 2008/05/14 11:59:24 sephe Exp $ */ #ifndef _SYS_INTERRUPT_H_ @@ -124,6 +124,7 @@ void unregister_randintr(int intr); int next_registered_randintr(int intr); void sched_ithd(int intr); /* procedure called from MD */ void forward_fastint_remote(void *arg); /* MD procedure (SMP) */ +int ithread_cpuid(int intr); extern char eintrnames[]; /* end of intrnames[] */ extern char intrnames[]; /* string table containing device names */ diff --git a/sys/sys/serialize.h b/sys/sys/serialize.h index eba9f2820b..b54ba100bb 100644 --- a/sys/sys/serialize.h +++ b/sys/sys/serialize.h @@ -8,7 +8,7 @@ * required. Unlike tokens this serialization is not safe from deadlocks * nor is it recursive, and care must be taken when using it. * - * $DragonFly: src/sys/sys/serialize.h,v 1.8 2008/05/05 12:35:03 sephe Exp $ + * $DragonFly: src/sys/sys/serialize.h,v 1.9 2008/05/14 11:59:24 sephe Exp $ */ #ifndef _SYS_SERIALIZE_H_ @@ -29,12 +29,9 @@ struct lwkt_serialize { unsigned int try_cnt; }; -/* - * Note that last_td is only maintained when INVARIANTS is turned on, - * so this check is only useful as part of a [K]KASSERT. - */ -#define ASSERT_SERIALIZED(ss) KKASSERT((ss)->last_td == curthread) -#define ASSERT_NOT_SERIALIZED(ss) KKASSERT((ss)->last_td != curthread) +#define IS_SERIALIZED(ss) ((ss)->last_td == curthread) +#define ASSERT_SERIALIZED(ss) KKASSERT(IS_SERIALIZED((ss))) +#define ASSERT_NOT_SERIALIZED(ss) KKASSERT(!IS_SERIALIZED((ss))) typedef struct lwkt_serialize *lwkt_serialize_t; @@ -50,4 +47,4 @@ void lwkt_serialize_handler_enable(lwkt_serialize_t); void lwkt_serialize_handler_call(lwkt_serialize_t, void (*)(void *, void *), void *, void *); int lwkt_serialize_handler_try(lwkt_serialize_t, void (*)(void *, void *), void *, void *); -#endif +#endif /* !_SYS_SERIALIZE_H_ */ -- 2.41.0