if: Multiple TX queue support step 3 of 3; map CPUID to subqueue
[dragonfly.git] / sys / net / ifq_var.h
CommitLineData
4d723e5a
JS
1/*-
2 * Copyright (c) 2005 The DragonFly Project. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in
12 * the documentation and/or other materials provided with the
13 * distribution.
14 * 3. Neither the name of The DragonFly Project nor the names of its
15 * contributors may be used to endorse or promote products derived
16 * from this software without specific, prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
21 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
22 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
23 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
24 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
28 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
4d723e5a 30 */
78195a76 31
1bd40720
MD
32#ifndef _NET_IFQ_VAR_H_
33#define _NET_IFQ_VAR_H_
4d723e5a 34
03d6a592
MD
35#ifndef _KERNEL
36
37#error "This file should not be included by userland programs."
38
39#else
40
41#ifndef _SYS_SYSTM_H_
1bd40720
MD
42#include <sys/systm.h>
43#endif
44#ifndef _SYS_THREAD2_H_
4986965b 45#include <sys/thread2.h>
1bd40720
MD
46#endif
47#ifndef _SYS_SERIALIZE_H_
48#include <sys/serialize.h>
49#endif
50#ifndef _SYS_MBUF_H_
51#include <sys/mbuf.h>
52#endif
53#ifndef _NET_IF_VAR_H_
54#include <net/if_var.h>
55#endif
56#ifndef _NET_ALTQ_IF_ALTQ_H_
57#include <net/altq/if_altq.h>
58#endif
59
f0a26983
SZ
60#define ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq) \
61 KASSERT(ifsq_get_ifp((ifsq)) == (ifp) && \
62 ifsq_get_index((ifsq)) == ALTQ_SUBQ_INDEX_DEFAULT, \
63 ("not ifp's default subqueue"));
64
1bd40720 65struct ifaltq;
4986965b 66
9db4b353 67/*
058d1cd0 68 * Support for "classic" ALTQ interfaces.
9db4b353 69 */
f0a26983 70int ifsq_classic_enqueue(struct ifaltq_subque *, struct mbuf *,
058d1cd0 71 struct altq_pktattr *);
f0a26983
SZ
72struct mbuf *ifsq_classic_dequeue(struct ifaltq_subque *, struct mbuf *,
73 int);
74int ifsq_classic_request(struct ifaltq_subque *, int, void *);
9db4b353
SZ
75void ifq_set_classic(struct ifaltq *);
76
0faec0d1 77void ifq_set_maxlen(struct ifaltq *, int);
2cc2f639
SZ
78void ifq_set_methods(struct ifaltq *, altq_mapsubq_t,
79 ifsq_enqueue_t, ifsq_dequeue_t, ifsq_request_t);
80int ifq_mapsubq_default(struct ifaltq *, int);
f0a26983
SZ
81
82void ifsq_devstart(struct ifaltq_subque *ifsq);
83void ifsq_devstart_sched(struct ifaltq_subque *ifsq);
058d1cd0
SZ
84
85/*
86 * Dispatch a packet to an interface.
87 */
9db4b353 88int ifq_dispatch(struct ifnet *, struct mbuf *,
058d1cd0 89 struct altq_pktattr *);
9db4b353 90
4d723e5a 91#ifdef ALTQ
058d1cd0 92
4d723e5a
JS
93static __inline int
94ifq_is_enabled(struct ifaltq *_ifq)
95{
96 return(_ifq->altq_flags & ALTQF_ENABLED);
97}
98
99static __inline int
100ifq_is_attached(struct ifaltq *_ifq)
101{
102 return(_ifq->altq_disc != NULL);
103}
058d1cd0
SZ
104
105#else /* !ALTQ */
106
4d723e5a
JS
107static __inline int
108ifq_is_enabled(struct ifaltq *_ifq)
109{
110 return(0);
111}
112
113static __inline int
114ifq_is_attached(struct ifaltq *_ifq)
115{
116 return(0);
117}
4d723e5a 118
058d1cd0
SZ
119#endif /* ALTQ */
120
4d723e5a
JS
121static __inline int
122ifq_is_ready(struct ifaltq *_ifq)
123{
124 return(_ifq->altq_flags & ALTQF_READY);
125}
126
127static __inline void
128ifq_set_ready(struct ifaltq *_ifq)
129{
130 _ifq->altq_flags |= ALTQF_READY;
131}
132
78195a76 133/*
058d1cd0 134 * ALTQ lock must be held
78195a76 135 */
4d723e5a 136static __inline int
f0a26983
SZ
137ifsq_enqueue_locked(struct ifaltq_subque *_ifsq, struct mbuf *_m,
138 struct altq_pktattr *_pa)
9db4b353
SZ
139{
140#ifdef ALTQ
f0a26983
SZ
141 if (!ifq_is_enabled(_ifsq->ifsq_altq))
142 return ifsq_classic_enqueue(_ifsq, _m, _pa);
9db4b353
SZ
143 else
144#endif
f0a26983 145 return _ifsq->ifsq_enqueue(_ifsq, _m, _pa);
9db4b353
SZ
146}
147
148static __inline int
f0a26983
SZ
149ifsq_enqueue(struct ifaltq_subque *_ifsq, struct mbuf *_m,
150 struct altq_pktattr *_pa)
4d723e5a 151{
9db4b353
SZ
152 int _error;
153
f0a26983
SZ
154 ALTQ_SQ_LOCK(_ifsq);
155 _error = ifsq_enqueue_locked(_ifsq, _m, _pa);
156 ALTQ_SQ_UNLOCK(_ifsq);
9db4b353 157 return _error;
4d723e5a
JS
158}
159
160static __inline struct mbuf *
f0a26983 161ifsq_dequeue(struct ifaltq_subque *_ifsq, struct mbuf *_mpolled)
4d723e5a 162{
9db4b353
SZ
163 struct mbuf *_m;
164
f0a26983
SZ
165 ALTQ_SQ_LOCK(_ifsq);
166 if (_ifsq->ifsq_prepended != NULL) {
167 _m = _ifsq->ifsq_prepended;
168 _ifsq->ifsq_prepended = NULL;
169 KKASSERT(_ifsq->ifq_len > 0);
170 _ifsq->ifq_len--;
171 ALTQ_SQ_UNLOCK(_ifsq);
9db4b353
SZ
172 return _m;
173 }
174
4d723e5a 175#ifdef ALTQ
f0a26983
SZ
176 if (_ifsq->ifsq_altq->altq_tbr != NULL)
177 _m = tbr_dequeue(_ifsq, _mpolled, ALTDQ_REMOVE);
178 else if (!ifq_is_enabled(_ifsq->ifsq_altq))
179 _m = ifsq_classic_dequeue(_ifsq, _mpolled, ALTDQ_REMOVE);
9db4b353 180 else
4d723e5a 181#endif
f0a26983
SZ
182 _m = _ifsq->ifsq_dequeue(_ifsq, _mpolled, ALTDQ_REMOVE);
183 ALTQ_SQ_UNLOCK(_ifsq);
9db4b353 184 return _m;
4d723e5a
JS
185}
186
78195a76 187/*
058d1cd0 188 * ALTQ lock must be held
78195a76 189 */
4d723e5a 190static __inline struct mbuf *
f0a26983 191ifsq_poll_locked(struct ifaltq_subque *_ifsq)
4d723e5a 192{
f0a26983
SZ
193 if (_ifsq->ifsq_prepended != NULL)
194 return _ifsq->ifsq_prepended;
9db4b353 195
4d723e5a 196#ifdef ALTQ
f0a26983
SZ
197 if (_ifsq->ifsq_altq->altq_tbr != NULL)
198 return tbr_dequeue(_ifsq, NULL, ALTDQ_POLL);
199 else if (!ifq_is_enabled(_ifsq->ifsq_altq))
200 return ifsq_classic_dequeue(_ifsq, NULL, ALTDQ_POLL);
9db4b353 201 else
4d723e5a 202#endif
f0a26983 203 return _ifsq->ifsq_dequeue(_ifsq, NULL, ALTDQ_POLL);
9db4b353
SZ
204}
205
206static __inline struct mbuf *
f0a26983 207ifsq_poll(struct ifaltq_subque *_ifsq)
9db4b353
SZ
208{
209 struct mbuf *_m;
210
f0a26983
SZ
211 ALTQ_SQ_LOCK(_ifsq);
212 _m = ifsq_poll_locked(_ifsq);
213 ALTQ_SQ_UNLOCK(_ifsq);
9db4b353 214 return _m;
4d723e5a
JS
215}
216
78195a76 217/*
058d1cd0 218 * ALTQ lock must be held
78195a76 219 */
4d723e5a 220static __inline void
f0a26983 221ifsq_purge_locked(struct ifaltq_subque *_ifsq)
4d723e5a 222{
f0a26983
SZ
223 if (_ifsq->ifsq_prepended != NULL) {
224 m_freem(_ifsq->ifsq_prepended);
225 _ifsq->ifsq_prepended = NULL;
226 KKASSERT(_ifsq->ifq_len > 0);
227 _ifsq->ifq_len--;
9db4b353
SZ
228 }
229
230#ifdef ALTQ
f0a26983
SZ
231 if (!ifq_is_enabled(_ifsq->ifsq_altq))
232 ifsq_classic_request(_ifsq, ALTRQ_PURGE, NULL);
91148f10 233 else
9db4b353 234#endif
f0a26983 235 _ifsq->ifsq_request(_ifsq, ALTRQ_PURGE, NULL);
9db4b353
SZ
236}
237
238static __inline void
f0a26983
SZ
239ifsq_purge(struct ifaltq_subque *_ifsq)
240{
241 ALTQ_SQ_LOCK(_ifsq);
242 ifsq_purge_locked(_ifsq);
243 ALTQ_SQ_UNLOCK(_ifsq);
244}
245
246static __inline void
247ifq_lock_all(struct ifaltq *_ifq)
248{
249 int _q;
250
251 for (_q = 0; _q < _ifq->altq_subq_cnt; ++_q)
252 ALTQ_SQ_LOCK(&_ifq->altq_subq[_q]);
253}
254
255static __inline void
256ifq_unlock_all(struct ifaltq *_ifq)
9db4b353 257{
f0a26983
SZ
258 int _q;
259
260 for (_q = _ifq->altq_subq_cnt - 1; _q >= 0; --_q)
261 ALTQ_SQ_UNLOCK(&_ifq->altq_subq[_q]);
4d723e5a
JS
262}
263
78195a76 264/*
058d1cd0 265 * ALTQ lock must be held
78195a76 266 */
4d723e5a 267static __inline void
9275f515
SZ
268ifq_purge_all_locked(struct ifaltq *_ifq)
269{
f0a26983
SZ
270 int _q;
271
272 for (_q = 0; _q < _ifq->altq_subq_cnt; ++_q)
273 ifsq_purge_locked(&_ifq->altq_subq[_q]);
9275f515
SZ
274}
275
276static __inline void
277ifq_purge_all(struct ifaltq *_ifq)
278{
f0a26983 279 ifq_lock_all(_ifq);
9275f515 280 ifq_purge_all_locked(_ifq);
f0a26983 281 ifq_unlock_all(_ifq);
9275f515
SZ
282}
283
9275f515 284static __inline void
4d723e5a 285ifq_classify(struct ifaltq *_ifq, struct mbuf *_m, uint8_t _af,
f0a26983 286 struct altq_pktattr *_pa)
4d723e5a 287{
9db4b353 288#ifdef ALTQ
9db4b353
SZ
289 if (ifq_is_enabled(_ifq)) {
290 _pa->pattr_af = _af;
291 _pa->pattr_hdr = mtod(_m, caddr_t);
f0a26983
SZ
292 if (ifq_is_enabled(_ifq) &&
293 (_ifq->altq_flags & ALTQF_CLASSIFY)) {
294 /* XXX default subqueue */
295 struct ifaltq_subque *_ifsq =
296 &_ifq->altq_subq[ALTQ_SUBQ_INDEX_DEFAULT];
297
298 ALTQ_SQ_LOCK(_ifsq);
299 if (ifq_is_enabled(_ifq) &&
300 (_ifq->altq_flags & ALTQF_CLASSIFY))
301 _ifq->altq_classify(_ifq, _m, _pa);
302 ALTQ_SQ_UNLOCK(_ifsq);
303 }
9db4b353 304 }
9db4b353
SZ
305#endif
306}
307
308static __inline void
f0a26983 309ifsq_prepend(struct ifaltq_subque *_ifsq, struct mbuf *_m)
9db4b353 310{
f0a26983
SZ
311 ALTQ_SQ_LOCK(_ifsq);
312 KASSERT(_ifsq->ifsq_prepended == NULL, ("pending prepended mbuf"));
313 _ifsq->ifsq_prepended = _m;
314 _ifsq->ifq_len++;
315 ALTQ_SQ_UNLOCK(_ifsq);
4d723e5a
JS
316}
317
058d1cd0
SZ
318/*
319 * Interface TX serializer must be held
320 */
9ed293e0 321static __inline void
f0a26983 322ifsq_set_oactive(struct ifaltq_subque *_ifsq)
9ed293e0 323{
f0a26983 324 _ifsq->ifsq_hw_oactive = 1;
9ed293e0
SZ
325}
326
058d1cd0
SZ
327/*
328 * Interface TX serializer must be held
329 */
9ed293e0 330static __inline void
f0a26983 331ifsq_clr_oactive(struct ifaltq_subque *_ifsq)
9ed293e0 332{
f0a26983 333 _ifsq->ifsq_hw_oactive = 0;
9ed293e0
SZ
334}
335
058d1cd0
SZ
336/*
337 * Interface TX serializer must be held
338 */
9ed293e0 339static __inline int
f0a26983 340ifsq_is_oactive(const struct ifaltq_subque *_ifsq)
9ed293e0 341{
f0a26983 342 return _ifsq->ifsq_hw_oactive;
9ed293e0
SZ
343}
344
78195a76 345/*
058d1cd0 346 * Hand a packet to an interface.
78195a76 347 *
058d1cd0
SZ
348 * Interface TX serializer must be held. If the interface TX
349 * serializer is not held yet, ifq_dispatch() should be used
350 * to get better performance.
78195a76 351 */
4d723e5a
JS
352static __inline int
353ifq_handoff(struct ifnet *_ifp, struct mbuf *_m, struct altq_pktattr *_pa)
354{
f0a26983 355 struct ifaltq_subque *_ifsq;
4986965b 356 int _error;
f0a26983
SZ
357 int _qid = ALTQ_SUBQ_INDEX_DEFAULT; /* XXX default subqueue */
358
359 _ifsq = &_ifp->if_snd.altq_subq[_qid];
4d723e5a 360
3c4cd924 361 ASSERT_IFNET_SERIALIZED_TX(_ifp, _ifsq);
f0a26983 362 _error = ifsq_enqueue(_ifsq, _m, _pa);
4d723e5a
JS
363 if (_error == 0) {
364 _ifp->if_obytes += _m->m_pkthdr.len;
365 if (_m->m_flags & M_MCAST)
366 _ifp->if_omcasts++;
f0a26983
SZ
367 if (!ifsq_is_oactive(_ifsq))
368 (*_ifp->if_start)(_ifp, _ifsq);
4d723e5a 369 }
4d723e5a
JS
370 return(_error);
371}
372
373static __inline int
f0a26983 374ifsq_is_empty(const struct ifaltq_subque *_ifsq)
4d723e5a 375{
f0a26983 376 return(_ifsq->ifq_len == 0);
4d723e5a
JS
377}
378
058d1cd0
SZ
379/*
380 * ALTQ lock must be held
381 */
9db4b353 382static __inline int
f0a26983 383ifsq_data_ready(struct ifaltq_subque *_ifsq)
9db4b353
SZ
384{
385#ifdef ALTQ
f0a26983
SZ
386 if (_ifsq->ifsq_altq->altq_tbr != NULL)
387 return (ifsq_poll_locked(_ifsq) != NULL);
9db4b353
SZ
388 else
389#endif
f0a26983 390 return !ifsq_is_empty(_ifsq);
9db4b353 391}
b2f93efe 392
acf18b65
SZ
393/*
394 * ALTQ lock must be held
395 */
396static __inline int
f0a26983 397ifsq_is_started(const struct ifaltq_subque *_ifsq)
acf18b65 398{
f0a26983 399 return _ifsq->ifsq_started;
acf18b65
SZ
400}
401
402/*
403 * ALTQ lock must be held
404 */
405static __inline void
f0a26983 406ifsq_set_started(struct ifaltq_subque *_ifsq)
acf18b65 407{
f0a26983 408 _ifsq->ifsq_started = 1;
acf18b65
SZ
409}
410
411/*
412 * ALTQ lock must be held
413 */
414static __inline void
f0a26983 415ifsq_clr_started(struct ifaltq_subque *_ifsq)
acf18b65 416{
f0a26983 417 _ifsq->ifsq_started = 0;
acf18b65
SZ
418}
419
f0a26983
SZ
420static __inline struct ifsubq_stage *
421ifsq_get_stage(struct ifaltq_subque *_ifsq, int _cpuid)
928e2027 422{
f0a26983 423 return &_ifsq->ifsq_stage[_cpuid];
928e2027
SZ
424}
425
dfd3b18b 426static __inline int
f0a26983 427ifsq_get_cpuid(const struct ifaltq_subque *_ifsq)
dfd3b18b 428{
f0a26983 429 return _ifsq->ifsq_cpuid;
dfd3b18b
SZ
430}
431
432static __inline void
f0a26983 433ifsq_set_cpuid(struct ifaltq_subque *_ifsq, int _cpuid)
dfd3b18b 434{
f0a26983
SZ
435 KASSERT(_cpuid >= 0 && _cpuid < ncpus,
436 ("invalid ifsq_cpuid %d", _cpuid));
437 _ifsq->ifsq_cpuid = _cpuid;
dfd3b18b
SZ
438}
439
42fdf81e 440static __inline struct lwkt_msg *
f0a26983
SZ
441ifsq_get_ifstart_lmsg(struct ifaltq_subque *_ifsq, int _cpuid)
442{
443 return &_ifsq->ifsq_ifstart_nmsg[_cpuid].lmsg;
444}
445
446static __inline int
447ifsq_get_index(const struct ifaltq_subque *_ifsq)
448{
449 return _ifsq->ifsq_index;
450}
451
452static __inline void
453ifsq_set_priv(struct ifaltq_subque *_ifsq, void *_priv)
454{
455 _ifsq->ifsq_hw_priv = _priv;
456}
457
458static __inline void *
459ifsq_get_priv(const struct ifaltq_subque *_ifsq)
460{
461 return _ifsq->ifsq_hw_priv;
462}
463
464static __inline struct ifnet *
465ifsq_get_ifp(const struct ifaltq_subque *_ifsq)
466{
467 return _ifsq->ifsq_ifp;
468}
469
470static __inline struct ifaltq_subque *
471ifq_get_subq_default(const struct ifaltq *_ifq)
472{
473 return &_ifq->altq_subq[ALTQ_SUBQ_INDEX_DEFAULT];
474}
475
476static __inline struct ifaltq_subque *
477ifq_get_subq(const struct ifaltq *_ifq, int _idx)
478{
479 KASSERT(_idx >= 0 && _idx < _ifq->altq_subq_cnt,
480 ("invalid qid %d", _idx));
481 return &_ifq->altq_subq[_idx];
482}
483
2cc2f639
SZ
484static __inline struct ifaltq_subque *
485ifq_map_subq(struct ifaltq *_ifq, int _cpuid)
486{
487 int _idx = _ifq->altq_mapsubq(_ifq, _cpuid);
488 return ifq_get_subq(_ifq, _idx);
489}
490
f0a26983
SZ
491/* COMPAT */
492static __inline int
493ifq_is_oactive(const struct ifaltq *_ifq)
494{
495 return ifsq_is_oactive(ifq_get_subq_default(_ifq));
496}
497
498/* COMPAT */
499static __inline void
500ifq_set_oactive(struct ifaltq *_ifq)
501{
502 ifsq_set_oactive(ifq_get_subq_default(_ifq));
503}
504
505/* COMPAT */
506static __inline void
507ifq_clr_oactive(struct ifaltq *_ifq)
508{
509 ifsq_clr_oactive(ifq_get_subq_default(_ifq));
510}
511
512/* COMPAT */
513static __inline int
514ifq_is_empty(struct ifaltq *_ifq)
515{
516 return ifsq_is_empty(ifq_get_subq_default(_ifq));
517}
518
519/* COMPAT */
520static __inline void
521ifq_purge(struct ifaltq *_ifq)
522{
523 ifsq_purge(ifq_get_subq_default(_ifq));
524}
525
526/* COMPAT */
527static __inline struct mbuf *
528ifq_dequeue(struct ifaltq *_ifq, struct mbuf *_mpolled)
529{
530 return ifsq_dequeue(ifq_get_subq_default(_ifq), _mpolled);
531}
532
533/* COMPAT */
534static __inline void
535ifq_prepend(struct ifaltq *_ifq, struct mbuf *_m)
536{
537 ifsq_prepend(ifq_get_subq_default(_ifq), _m);
538}
539
540/* COMPAT */
541static __inline void
542ifq_set_cpuid(struct ifaltq *_ifq, int _cpuid)
42fdf81e 543{
f0a26983
SZ
544 KASSERT(_ifq->altq_subq_cnt == 1,
545 ("invalid subqueue count %d", _ifq->altq_subq_cnt));
546 ifsq_set_cpuid(ifq_get_subq_default(_ifq), _cpuid);
42fdf81e
SZ
547}
548
03d6a592
MD
549#endif /* _KERNEL */
550#endif /* _NET_IFQ_VAR_H_ */