netmap: change header includes
[dragonfly.git] / sys / net / netmap / netmap_kern.h
CommitLineData
fb578518
FF
1/*
2 * Copyright (C) 2011-2013 Matteo Landi, Luigi Rizzo. All rights reserved.
3 * Copyright (C) 2013 Universita` di Pisa. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27/*
28 * $FreeBSD: head/sys/dev/netmap/netmap_kern.h 238985 2012-08-02 11:59:43Z luigi $
29 *
30 * The header contains the definitions of constants and function
31 * prototypes used only in kernelspace.
32 */
33
34#ifndef _NET_NETMAP_KERN_H_
35#define _NET_NETMAP_KERN_H_
36
37#define WITH_VALE // comment out to disable VALE support
38
fb578518
FF
39#define likely(x) __builtin_expect((long)!!(x), 1L)
40#define unlikely(x) __builtin_expect((long)!!(x), 0L)
41
ed9bd855
FF
42#define NM_LOCK_T struct lock
43#define NMG_LOCK_T struct lock
44#define NMG_LOCK_INIT() lockinit(&netmap_global_lock, \
bf9f7c16 45 "netmap global lock", 0, LK_CANRECURSE)
ed9bd855
FF
46#define NMG_LOCK_DESTROY() lockuninit(&netmap_global_lock)
47#define NMG_LOCK() lockmgr(&netmap_global_lock, LK_EXCLUSIVE)
48#define NMG_UNLOCK() lockmgr(&netmap_global_lock, LK_RELEASE)
bf9f7c16 49#define NMG_LOCK_ASSERT() KKASSERT(lockstatus(&netmap_global_lock, NULL) != 0)
fb578518 50
bf9f7c16 51#define NM_SELINFO_T struct kqinfo
fb578518
FF
52#define MBUF_LEN(m) ((m)->m_pkthdr.len)
53#define MBUF_IFP(m) ((m)->m_pkthdr.rcvif)
54#define NM_SEND_UP(ifp, m) ((ifp)->if_input)(ifp, m)
55
56#define NM_ATOMIC_T volatile int // XXX ?
57/* atomic operations */
58#include <machine/atomic.h>
59#define NM_ATOMIC_TEST_AND_SET(p) (!atomic_cmpset_acq_int((p), 0, 1))
60#define NM_ATOMIC_CLEAR(p) atomic_store_rel_int((p), 0)
61
62#define prefetch(x) __builtin_prefetch(x)
63
bf9f7c16
FF
64#define mb() cpu_mfence()
65#define rmb() cpu_lfence()
66#define wmb() cpu_sfence()
67
fb578518
FF
68MALLOC_DECLARE(M_NETMAP);
69
70// XXX linux struct, not used in FreeBSD
71struct net_device_ops {
72};
73struct hrtimer {
74};
75
ed9bd855 76#define IFCAP_NETMAP 0x8000 /* XXX move to <net/if.h> */
fb578518
FF
77
78#define ND(format, ...)
79#define D(format, ...) \
80 do { \
81 struct timeval __xxts; \
82 microtime(&__xxts); \
ed9bd855 83 kprintf("%03d.%06d %s [%d] " format "\n", \
fb578518
FF
84 (int)__xxts.tv_sec % 1000, (int)__xxts.tv_usec, \
85 __FUNCTION__, __LINE__, ##__VA_ARGS__); \
86 } while (0)
87
88/* rate limited, lps indicates how many per second */
89#define RD(lps, format, ...) \
90 do { \
91 static int t0, __cnt; \
92 if (t0 != time_second) { \
93 t0 = time_second; \
94 __cnt = 0; \
95 } \
96 if (__cnt++ < lps) \
97 D(format, ##__VA_ARGS__); \
98 } while (0)
99
100struct netmap_adapter;
101struct nm_bdg_fwd;
102struct nm_bridge;
103struct netmap_priv_d;
104
105const char *nm_dump_buf(char *p, int len, int lim, char *dst);
106
b3f97fad 107#include <net/netmap/netmap_mbq.h>
fb578518
FF
108
109extern NMG_LOCK_T netmap_global_lock;
110
111/*
112 * private, kernel view of a ring. Keeps track of the status of
113 * a ring across system calls.
114 *
115 * nr_hwcur index of the next buffer to refill.
116 * It corresponds to ring->cur - ring->reserved
117 *
118 * nr_hwavail the number of slots "owned" by userspace.
119 * nr_hwavail =:= ring->avail + ring->reserved
120 *
121 * The indexes in the NIC and netmap rings are offset by nkr_hwofs slots.
122 * This is so that, on a reset, buffers owned by userspace are not
123 * modified by the kernel. In particular:
124 * RX rings: the next empty buffer (hwcur + hwavail + hwofs) coincides with
125 * the next empty buffer as known by the hardware (next_to_check or so).
126 * TX rings: hwcur + hwofs coincides with next_to_send
127 *
128 * Clients cannot issue concurrent syscall on a ring. The system
129 * detects this and reports an error using two flags,
130 * NKR_WBUSY and NKR_RBUSY
131 * For received packets, slot->flags is set to nkr_slot_flags
132 * so we can provide a proper initial value (e.g. set NS_FORWARD
133 * when operating in 'transparent' mode).
134 *
135 * The following fields are used to implement lock-free copy of packets
136 * from input to output ports in VALE switch:
137 * nkr_hwlease buffer after the last one being copied.
138 * A writer in nm_bdg_flush reserves N buffers
139 * from nr_hwlease, advances it, then does the
140 * copy outside the lock.
141 * In RX rings (used for VALE ports),
142 * nkr_hwcur + nkr_hwavail <= nkr_hwlease < nkr_hwcur+N-1
143 * In TX rings (used for NIC or host stack ports)
144 * nkr_hwcur <= nkr_hwlease < nkr_hwcur+ nkr_hwavail
145 * nkr_leases array of nkr_num_slots where writers can report
146 * completion of their block. NR_NOSLOT (~0) indicates
147 * that the writer has not finished yet
148 * nkr_lease_idx index of next free slot in nr_leases, to be assigned
149 *
150 * The kring is manipulated by txsync/rxsync and generic netmap function.
151 * q_lock is used to arbitrate access to the kring from within the netmap
152 * code, and this and other protections guarantee that there is never
153 * more than 1 concurrent call to txsync or rxsync. So we are free
154 * to manipulate the kring from within txsync/rxsync without any extra
155 * locks.
156 */
157struct netmap_kring {
158 struct netmap_ring *ring;
159 uint32_t nr_hwcur;
160 uint32_t nr_hwavail;
161 uint32_t nr_kflags; /* private driver flags */
162 int32_t nr_hwreserved;
163#define NKR_PENDINTR 0x1 // Pending interrupt.
164 uint32_t nkr_num_slots;
165 int32_t nkr_hwofs; /* offset between NIC and netmap ring */
166
167 uint16_t nkr_slot_flags; /* initial value for flags */
168 struct netmap_adapter *na;
169 struct nm_bdg_fwd *nkr_ft;
170 uint32_t *nkr_leases;
171#define NR_NOSLOT ((uint32_t)~0)
172 uint32_t nkr_hwlease;
173 uint32_t nkr_lease_idx;
174
175 NM_SELINFO_T si; /* poll/select wait queue */
176 NM_LOCK_T q_lock; /* protects kring and ring. */
177 NM_ATOMIC_T nr_busy; /* prevent concurrent syscalls */
178
179 volatile int nkr_stopped;
180
181 /* support for adapters without native netmap support.
182 * On tx rings we preallocate an array of tx buffers
183 * (same size as the netmap ring), on rx rings we
184 * store incoming packets in a queue.
185 * XXX who writes to the rx queue ?
186 */
187 struct mbuf **tx_pool;
188 u_int nr_ntc; /* Emulation of a next-to-clean RX ring pointer. */
189 struct mbq rx_queue; /* A queue for intercepted rx mbufs. */
190
191} __attribute__((__aligned__(64)));
192
193
194/* return the next index, with wraparound */
195static inline uint32_t
196nm_next(uint32_t i, uint32_t lim)
197{
198 return unlikely (i == lim) ? 0 : i + 1;
199}
200
201/*
202 *
203 * Here is the layout for the Rx and Tx rings.
204
205 RxRING TxRING
206
207 +-----------------+ +-----------------+
208 | | | |
209 |XXX free slot XXX| |XXX free slot XXX|
210 +-----------------+ +-----------------+
211 | |<-hwcur | |<-hwcur
212 | reserved h | | (ready |
213 +----------- w -+ | to be |
214 cur->| a | | sent) h |
215 | v | +---------- w |
216 | a | cur->| (being a |
217 | i | | prepared) v |
218 | avail l | | a |
219 +-----------------+ + a ------ i +
220 | | ... | v l |<-hwlease
221 | (being | ... | a | ...
222 | prepared) | ... | i | ...
223 +-----------------+ ... | l | ...
224 | |<-hwlease +-----------------+
225 | | | |
226 | | | |
227 | | | |
228 | | | |
229 +-----------------+ +-----------------+
230
231 * The cur/avail (user view) and hwcur/hwavail (kernel view)
232 * are used in the normal operation of the card.
233 *
234 * When a ring is the output of a switch port (Rx ring for
235 * a VALE port, Tx ring for the host stack or NIC), slots
236 * are reserved in blocks through 'hwlease' which points
237 * to the next unused slot.
238 * On an Rx ring, hwlease is always after hwavail,
239 * and completions cause avail to advance.
240 * On a Tx ring, hwlease is always between cur and hwavail,
241 * and completions cause cur to advance.
242 *
243 * nm_kr_space() returns the maximum number of slots that
244 * can be assigned.
245 * nm_kr_lease() reserves the required number of buffers,
246 * advances nkr_hwlease and also returns an entry in
247 * a circular array where completions should be reported.
248 */
249
250
251
252
253enum txrx { NR_RX = 0, NR_TX = 1 };
254
255/*
256 * The "struct netmap_adapter" extends the "struct adapter"
257 * (or equivalent) device descriptor.
258 * It contains all base fields needed to support netmap operation.
259 * There are in fact different types of netmap adapters
260 * (native, generic, VALE switch...) so a netmap_adapter is
261 * just the first field in the derived type.
262 */
263struct netmap_adapter {
264 /*
265 * On linux we do not have a good way to tell if an interface
266 * is netmap-capable. So we use the following trick:
267 * NA(ifp) points here, and the first entry (which hopefully
268 * always exists and is at least 32 bits) contains a magic
269 * value which we can use to detect that the interface is good.
270 */
271 uint32_t magic;
272 uint32_t na_flags; /* future place for IFCAP_NETMAP */
273#define NAF_SKIP_INTR 1 /* use the regular interrupt handler.
274 * useful during initialization
275 */
276#define NAF_SW_ONLY 2 /* forward packets only to sw adapter */
277#define NAF_BDG_MAYSLEEP 4 /* the bridge is allowed to sleep when
278 * forwarding packets coming from this
279 * interface
280 */
281#define NAF_MEM_OWNER 8 /* the adapter is responsible for the
282 * deallocation of the memory allocator
283 */
284#define NAF_NATIVE_ON 16 /* the adapter is native and the attached
285 * interface is in netmap mode
286 */
287 int active_fds; /* number of user-space descriptors using this
288 interface, which is equal to the number of
289 struct netmap_if objs in the mapped region. */
290
291 u_int num_rx_rings; /* number of adapter receive rings */
292 u_int num_tx_rings; /* number of adapter transmit rings */
293
294 u_int num_tx_desc; /* number of descriptor in each queue */
295 u_int num_rx_desc;
296
297 /* tx_rings and rx_rings are private but allocated
298 * as a contiguous chunk of memory. Each array has
299 * N+1 entries, for the adapter queues and for the host queue.
300 */
301 struct netmap_kring *tx_rings; /* array of TX rings. */
302 struct netmap_kring *rx_rings; /* array of RX rings. */
303 void *tailroom; /* space below the rings array */
304 /* (used for leases) */
305
13431b3e 306 NM_SELINFO_T tx_si, rx_si; /* global wait queues */
fb578518
FF
307
308 /* copy of if_qflush and if_transmit pointers, to intercept
309 * packets from the network stack when netmap is active.
310 */
311 int (*if_transmit)(struct ifnet *, struct mbuf *);
312
313 /* references to the ifnet and device routines, used by
314 * the generic netmap functions.
315 */
316 struct ifnet *ifp; /* adapter is ifp->if_softc */
317
318 /* private cleanup */
319 void (*nm_dtor)(struct netmap_adapter *);
320
321 int (*nm_register)(struct netmap_adapter *, int onoff);
322
323 int (*nm_txsync)(struct netmap_adapter *, u_int ring, int flags);
324 int (*nm_rxsync)(struct netmap_adapter *, u_int ring, int flags);
325#define NAF_FORCE_READ 1
326#define NAF_FORCE_RECLAIM 2
327 /* return configuration information */
328 int (*nm_config)(struct netmap_adapter *,
329 u_int *txr, u_int *txd, u_int *rxr, u_int *rxd);
330 int (*nm_krings_create)(struct netmap_adapter *);
331 void (*nm_krings_delete)(struct netmap_adapter *);
332 int (*nm_notify)(struct netmap_adapter *,
333 u_int ring, enum txrx, int flags);
334#define NAF_GLOBAL_NOTIFY 4
335#define NAF_DISABLE_NOTIFY 8
336
337 /* standard refcount to control the lifetime of the adapter
338 * (it should be equal to the lifetime of the corresponding ifp)
339 */
340 int na_refcount;
341
342 /* memory allocator (opaque)
343 * We also cache a pointer to the lut_entry for translating
344 * buffer addresses, and the total number of buffers.
345 */
346 struct netmap_mem_d *nm_mem;
347 struct lut_entry *na_lut;
348 uint32_t na_lut_objtotal; /* max buffer index */
349
350 /* used internally. If non-null, the interface cannot be bound
351 * from userspace
352 */
353 void *na_private;
354};
355
356/*
357 * If the NIC is owned by the kernel
358 * (i.e., bridge), neither another bridge nor user can use it;
359 * if the NIC is owned by a user, only users can share it.
360 * Evaluation must be done under NMG_LOCK().
361 */
362#define NETMAP_OWNED_BY_KERN(na) (na->na_private)
363#define NETMAP_OWNED_BY_ANY(na) \
364 (NETMAP_OWNED_BY_KERN(na) || (na->active_fds > 0))
365
366
367/*
368 * derived netmap adapters for various types of ports
369 */
370struct netmap_vp_adapter { /* VALE software port */
371 struct netmap_adapter up;
372
373 /*
374 * Bridge support:
375 *
376 * bdg_port is the port number used in the bridge;
377 * na_bdg points to the bridge this NA is attached to.
378 */
379 int bdg_port;
380 struct nm_bridge *na_bdg;
381 int retry;
382};
383
384struct netmap_hw_adapter { /* physical device */
385 struct netmap_adapter up;
386
387 struct net_device_ops nm_ndo; // XXX linux only
388};
389
390struct netmap_generic_adapter { /* non-native device */
391 struct netmap_hw_adapter up;
392
393 /* Pointer to a previously used netmap adapter. */
394 struct netmap_adapter *prev;
395
396 /* generic netmap adapters support:
397 * a net_device_ops struct overrides ndo_select_queue(),
398 * save_if_input saves the if_input hook (FreeBSD),
399 * mit_timer and mit_pending implement rx interrupt mitigation,
400 */
401 struct net_device_ops generic_ndo;
402 void (*save_if_input)(struct ifnet *, struct mbuf *);
403
404 struct hrtimer mit_timer;
405 int mit_pending;
406};
407
408#ifdef WITH_VALE
409
410/* bridge wrapper for non VALE ports. It is used to connect real devices to the bridge.
411 *
412 * The real device must already have its own netmap adapter (hwna). The
413 * bridge wrapper and the hwna adapter share the same set of netmap rings and
414 * buffers, but they have two separate sets of krings descriptors, with tx/rx
415 * meanings swapped:
416 *
417 * netmap
418 * bwrap krings rings krings hwna
419 * +------+ +------+ +-----+ +------+ +------+
420 * |tx_rings->| |\ /| |----| |<-tx_rings|
421 * | | +------+ \ / +-----+ +------+ | |
422 * | | X | |
423 * | | / \ | |
424 * | | +------+/ \+-----+ +------+ | |
425 * |rx_rings->| | | |----| |<-rx_rings|
426 * | | +------+ +-----+ +------+ | |
427 * +------+ +------+
428 *
429 * - packets coming from the bridge go to the brwap rx rings, which are also the
430 * hwna tx rings. The bwrap notify callback will then complete the hwna tx
431 * (see netmap_bwrap_notify).
432 * - packets coming from the outside go to the hwna rx rings, which are also the
433 * bwrap tx rings. The (overwritten) hwna notify method will then complete
434 * the bridge tx (see netmap_bwrap_intr_notify).
435 *
436 * The bridge wrapper may optionally connect the hwna 'host' rings to the
437 * bridge. This is done by using a second port in the bridge and connecting it
438 * to the 'host' netmap_vp_adapter contained in the netmap_bwrap_adapter.
439 * The brwap host adapter cross-links the hwna host rings in the same way as shown above.
440 *
441 * - packets coming from the bridge and directed to host stack are handled by the
442 * bwrap host notify callback (see netmap_bwrap_host_notify)
443 * - packets coming from the host stack are still handled by the overwritten
444 * hwna notify callback (netmap_bwrap_intr_notify), but are diverted to the
445 * host adapter depending on the ring number.
446 *
447 */
448struct netmap_bwrap_adapter {
449 struct netmap_vp_adapter up;
450 struct netmap_vp_adapter host; /* for host rings */
451 struct netmap_adapter *hwna; /* the underlying device */
452
453 /* backup of the hwna notify callback */
454 int (*save_notify)(struct netmap_adapter *,
455 u_int ring, enum txrx, int flags);
456 /* When we attach a physical interface to the bridge, we
457 * allow the controlling process to terminate, so we need
458 * a place to store the netmap_priv_d data structure.
459 * This is only done when physical interfaces are attached to a bridge.
460 */
461 struct netmap_priv_d *na_kpriv;
462};
463
464
465/*
466 * Available space in the ring. Only used in VALE code
467 */
468static inline uint32_t
469nm_kr_space(struct netmap_kring *k, int is_rx)
470{
471 int space;
472
473 if (is_rx) {
474 int busy = k->nkr_hwlease - k->nr_hwcur + k->nr_hwreserved;
475 if (busy < 0)
476 busy += k->nkr_num_slots;
477 space = k->nkr_num_slots - 1 - busy;
478 } else {
479 space = k->nr_hwcur + k->nr_hwavail - k->nkr_hwlease;
480 if (space < 0)
481 space += k->nkr_num_slots;
482 }
483#if 0
484 // sanity check
485 if (k->nkr_hwlease >= k->nkr_num_slots ||
486 k->nr_hwcur >= k->nkr_num_slots ||
487 k->nr_hwavail >= k->nkr_num_slots ||
488 busy < 0 ||
489 busy >= k->nkr_num_slots) {
490 D("invalid kring, cur %d avail %d lease %d lease_idx %d lim %d", k->nr_hwcur, k->nr_hwavail, k->nkr_hwlease,
491 k->nkr_lease_idx, k->nkr_num_slots);
492 }
493#endif
494 return space;
495}
496
497
498
499
500/* make a lease on the kring for N positions. return the
501 * lease index
502 */
503static inline uint32_t
504nm_kr_lease(struct netmap_kring *k, u_int n, int is_rx)
505{
506 uint32_t lim = k->nkr_num_slots - 1;
507 uint32_t lease_idx = k->nkr_lease_idx;
508
509 k->nkr_leases[lease_idx] = NR_NOSLOT;
510 k->nkr_lease_idx = nm_next(lease_idx, lim);
511
512 if (n > nm_kr_space(k, is_rx)) {
513 D("invalid request for %d slots", n);
514 panic("x");
515 }
516 /* XXX verify that there are n slots */
517 k->nkr_hwlease += n;
518 if (k->nkr_hwlease > lim)
519 k->nkr_hwlease -= lim + 1;
520
521 if (k->nkr_hwlease >= k->nkr_num_slots ||
522 k->nr_hwcur >= k->nkr_num_slots ||
523 k->nr_hwavail >= k->nkr_num_slots ||
524 k->nkr_lease_idx >= k->nkr_num_slots) {
525 D("invalid kring %s, cur %d avail %d lease %d lease_idx %d lim %d",
526 k->na->ifp->if_xname,
527 k->nr_hwcur, k->nr_hwavail, k->nkr_hwlease,
528 k->nkr_lease_idx, k->nkr_num_slots);
529 }
530 return lease_idx;
531}
532
533#endif /* WITH_VALE */
534
535/* return update position */
536static inline uint32_t
537nm_kr_rxpos(struct netmap_kring *k)
538{
539 uint32_t pos = k->nr_hwcur + k->nr_hwavail;
540 if (pos >= k->nkr_num_slots)
541 pos -= k->nkr_num_slots;
542#if 0
543 if (pos >= k->nkr_num_slots ||
544 k->nkr_hwlease >= k->nkr_num_slots ||
545 k->nr_hwcur >= k->nkr_num_slots ||
546 k->nr_hwavail >= k->nkr_num_slots ||
547 k->nkr_lease_idx >= k->nkr_num_slots) {
548 D("invalid kring, cur %d avail %d lease %d lease_idx %d lim %d", k->nr_hwcur, k->nr_hwavail, k->nkr_hwlease,
549 k->nkr_lease_idx, k->nkr_num_slots);
550 }
551#endif
552 return pos;
553}
554
555
556/*
557 * protect against multiple threads using the same ring.
558 * also check that the ring has not been stopped.
559 * We only care for 0 or !=0 as a return code.
560 */
561#define NM_KR_BUSY 1
562#define NM_KR_STOPPED 2
563
564static __inline void nm_kr_put(struct netmap_kring *kr)
565{
566 NM_ATOMIC_CLEAR(&kr->nr_busy);
567}
568
569static __inline int nm_kr_tryget(struct netmap_kring *kr)
570{
571 /* check a first time without taking the lock
572 * to avoid starvation for nm_kr_get()
573 */
574 if (unlikely(kr->nkr_stopped)) {
575 ND("ring %p stopped (%d)", kr, kr->nkr_stopped);
576 return NM_KR_STOPPED;
577 }
578 if (unlikely(NM_ATOMIC_TEST_AND_SET(&kr->nr_busy)))
579 return NM_KR_BUSY;
580 /* check a second time with lock held */
581 if (unlikely(kr->nkr_stopped)) {
582 ND("ring %p stopped (%d)", kr, kr->nkr_stopped);
583 nm_kr_put(kr);
584 return NM_KR_STOPPED;
585 }
586 return 0;
587}
588
589
590/*
591 * The following are support routines used by individual drivers to
592 * support netmap operation.
593 *
594 * netmap_attach() initializes a struct netmap_adapter, allocating the
595 * struct netmap_ring's and the struct selinfo.
596 *
597 * netmap_detach() frees the memory allocated by netmap_attach().
598 *
599 * netmap_transmit() replaces the if_transmit routine of the interface,
600 * and is used to intercept packets coming from the stack.
601 *
602 * netmap_load_map/netmap_reload_map are helper routines to set/reset
603 * the dmamap for a packet buffer
604 *
605 * netmap_reset() is a helper routine to be called in the driver
606 * when reinitializing a ring.
607 */
608int netmap_attach(struct netmap_adapter *);
609int netmap_attach_common(struct netmap_adapter *);
610void netmap_detach_common(struct netmap_adapter *na);
611void netmap_detach(struct ifnet *);
612int netmap_transmit(struct ifnet *, struct mbuf *);
613struct netmap_slot *netmap_reset(struct netmap_adapter *na,
614 enum txrx tx, u_int n, u_int new_cur);
615int netmap_ring_reinit(struct netmap_kring *);
616
617
618/*
619 * Support routines to be used with the VALE switch
620 */
621int netmap_update_config(struct netmap_adapter *na);
622int netmap_krings_create(struct netmap_adapter *na, u_int ntx, u_int nrx, u_int tailroom);
623void netmap_krings_delete(struct netmap_adapter *na);
624
625struct netmap_if *
626netmap_do_regif(struct netmap_priv_d *priv, struct netmap_adapter *na,
627 uint16_t ringid, int *err);
628
629
630
631u_int nm_bound_var(u_int *v, u_int dflt, u_int lo, u_int hi, const char *msg);
632int netmap_get_na(struct nmreq *nmr, struct netmap_adapter **na, int create);
633int netmap_get_hw_na(struct ifnet *ifp, struct netmap_adapter **na);
634
635#ifdef WITH_VALE
636/*
637 * The following bridge-related interfaces are used by other kernel modules
638 * In the version that only supports unicast or broadcast, the lookup
639 * function can return 0 .. NM_BDG_MAXPORTS-1 for regular ports,
640 * NM_BDG_MAXPORTS for broadcast, NM_BDG_MAXPORTS+1 for unknown.
641 * XXX in practice "unknown" might be handled same as broadcast.
642 */
643typedef u_int (*bdg_lookup_fn_t)(char *buf, u_int len,
644 uint8_t *ring_nr, struct netmap_vp_adapter *);
645u_int netmap_bdg_learning(char *, u_int, uint8_t *,
646 struct netmap_vp_adapter *);
647
648#define NM_BDG_MAXPORTS 254 /* up to 254 */
649#define NM_BDG_BROADCAST NM_BDG_MAXPORTS
650#define NM_BDG_NOPORT (NM_BDG_MAXPORTS+1)
651
652#define NM_NAME "vale" /* prefix for bridge port name */
653
654
655/* these are redefined in case of no VALE support */
656int netmap_get_bdg_na(struct nmreq *nmr, struct netmap_adapter **na, int create);
657void netmap_init_bridges(void);
658int netmap_bdg_ctl(struct nmreq *nmr, bdg_lookup_fn_t func);
659
660#else /* !WITH_VALE */
661#define netmap_get_bdg_na(_1, _2, _3) 0
662#define netmap_init_bridges(_1)
663#define netmap_bdg_ctl(_1, _2) EINVAL
664#endif /* !WITH_VALE */
665
666/* Various prototypes */
b3f97fad 667struct dev_kqfilter_args; /* XXX this shouldn't be here */
f27ed164 668int netmap_kqfilter(struct dev_kqfilter_args *ap);
fb578518
FF
669
670
671int netmap_init(void);
672void netmap_fini(void);
673int netmap_get_memory(struct netmap_priv_d* p);
674void netmap_dtor(void *data);
675int netmap_dtor_locked(struct netmap_priv_d *priv);
676
13431b3e
FF
677struct dev_ioctl_args; /* XXX this shouldn't be here */
678int netmap_ioctl(struct dev_ioctl_args *ap);
fb578518
FF
679
680/* netmap_adapter creation/destruction */
681#define NM_IFPNAME(ifp) ((ifp) ? (ifp)->if_xname : "zombie")
682#define NM_DEBUG_PUTGET 1
683
684#ifdef NM_DEBUG_PUTGET
685
686#define NM_DBG(f) __##f
687
688void __netmap_adapter_get(struct netmap_adapter *na);
689
690#define netmap_adapter_get(na) \
691 do { \
692 struct netmap_adapter *__na = na; \
693 D("getting %p:%s (%d)", __na, NM_IFPNAME(__na->ifp), __na->na_refcount); \
694 __netmap_adapter_get(__na); \
695 } while (0)
696
697int __netmap_adapter_put(struct netmap_adapter *na);
698
699#define netmap_adapter_put(na) \
700 do { \
701 struct netmap_adapter *__na = na; \
702 D("putting %p:%s (%d)", __na, NM_IFPNAME(__na->ifp), __na->na_refcount); \
703 __netmap_adapter_put(__na); \
704 } while (0)
705
706#else /* !NM_DEBUG_PUTGET */
707
708#define NM_DBG(f) f
709void netmap_adapter_get(struct netmap_adapter *na);
710int netmap_adapter_put(struct netmap_adapter *na);
711
712#endif /* !NM_DEBUG_PUTGET */
713
714
715
716extern u_int netmap_buf_size;
717#define NETMAP_BUF_SIZE netmap_buf_size // XXX remove
718extern int netmap_mitigate;
719extern int netmap_no_pendintr;
720extern u_int netmap_total_buffers;
721extern char *netmap_buffer_base;
722extern int netmap_verbose; // XXX debugging
723enum { /* verbose flags */
724 NM_VERB_ON = 1, /* generic verbose */
725 NM_VERB_HOST = 0x2, /* verbose host stack */
726 NM_VERB_RXSYNC = 0x10, /* verbose on rxsync/txsync */
727 NM_VERB_TXSYNC = 0x20,
728 NM_VERB_RXINTR = 0x100, /* verbose on rx/tx intr (driver) */
729 NM_VERB_TXINTR = 0x200,
730 NM_VERB_NIC_RXSYNC = 0x1000, /* verbose on rx/tx intr (driver) */
731 NM_VERB_NIC_TXSYNC = 0x2000,
732};
733
734extern int netmap_txsync_retry;
735extern int netmap_generic_mit;
736extern int netmap_generic_ringsize;
737
738/*
739 * NA returns a pointer to the struct netmap adapter from the ifp,
740 * WNA is used to write it.
741 */
742#ifndef WNA
ed9bd855 743#define WNA(_ifp) (_ifp)->if_unused7 /* XXX better name ;) */
fb578518
FF
744#endif
745#define NA(_ifp) ((struct netmap_adapter *)WNA(_ifp))
746
747/*
748 * Macros to determine if an interface is netmap capable or netmap enabled.
749 * See the magic field in struct netmap_adapter.
750 */
fb578518
FF
751/*
752 * on FreeBSD just use if_capabilities and if_capenable.
753 */
754#define NETMAP_CAPABLE(ifp) (NA(ifp) && \
755 (ifp)->if_capabilities & IFCAP_NETMAP )
756
757#define NETMAP_SET_CAPABLE(ifp) \
758 (ifp)->if_capabilities |= IFCAP_NETMAP
759
fb578518
FF
760/* Callback invoked by the dma machinery after a successfull dmamap_load */
761static void netmap_dmamap_cb(__unused void *arg,
762 __unused bus_dma_segment_t * segs, __unused int nseg, __unused int error)
763{
764}
765
766/* bus_dmamap_load wrapper: call aforementioned function if map != NULL.
767 * XXX can we do it without a callback ?
768 */
769static inline void
770netmap_load_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf)
771{
772 if (map)
773 bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE,
774 netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT);
775}
776
777/* update the map when a buffer changes. */
778static inline void
779netmap_reload_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf)
780{
781 if (map) {
782 bus_dmamap_unload(tag, map);
783 bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE,
784 netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT);
785 }
786}
787
fb578518
FF
788/*
789 * functions to map NIC to KRING indexes (n2k) and vice versa (k2n)
790 */
791static inline int
792netmap_idx_n2k(struct netmap_kring *kr, int idx)
793{
794 int n = kr->nkr_num_slots;
795 idx += kr->nkr_hwofs;
796 if (idx < 0)
797 return idx + n;
798 else if (idx < n)
799 return idx;
800 else
801 return idx - n;
802}
803
804
805static inline int
806netmap_idx_k2n(struct netmap_kring *kr, int idx)
807{
808 int n = kr->nkr_num_slots;
809 idx -= kr->nkr_hwofs;
810 if (idx < 0)
811 return idx + n;
812 else if (idx < n)
813 return idx;
814 else
815 return idx - n;
816}
817
818
819/* Entries of the look-up table. */
820struct lut_entry {
821 void *vaddr; /* virtual address. */
822 vm_paddr_t paddr; /* physical address. */
823};
824
825struct netmap_obj_pool;
826extern struct lut_entry *netmap_buffer_lut;
827#define NMB_VA(i) (netmap_buffer_lut[i].vaddr)
828#define NMB_PA(i) (netmap_buffer_lut[i].paddr)
829
830/*
831 * NMB return the virtual address of a buffer (buffer 0 on bad index)
832 * PNMB also fills the physical address
833 */
834static inline void *
835NMB(struct netmap_slot *slot)
836{
837 uint32_t i = slot->buf_idx;
838 return (unlikely(i >= netmap_total_buffers)) ? NMB_VA(0) : NMB_VA(i);
839}
840
841static inline void *
842PNMB(struct netmap_slot *slot, uint64_t *pp)
843{
844 uint32_t i = slot->buf_idx;
845 void *ret = (i >= netmap_total_buffers) ? NMB_VA(0) : NMB_VA(i);
846
847 *pp = (i >= netmap_total_buffers) ? NMB_PA(0) : NMB_PA(i);
848 return ret;
849}
850
851/* Generic version of NMB, which uses device-specific memory. */
852static inline void *
853BDG_NMB(struct netmap_adapter *na, struct netmap_slot *slot)
854{
855 struct lut_entry *lut = na->na_lut;
856 uint32_t i = slot->buf_idx;
857 return (unlikely(i >= na->na_lut_objtotal)) ?
858 lut[0].vaddr : lut[i].vaddr;
859}
860
861/* default functions to handle rx/tx interrupts */
862int netmap_rx_irq(struct ifnet *, u_int, u_int *);
863#define netmap_tx_irq(_n, _q) netmap_rx_irq(_n, _q, NULL)
864int netmap_common_irq(struct ifnet *, u_int, u_int *work_done);
865
866
867void netmap_txsync_to_host(struct netmap_adapter *na);
868void netmap_disable_all_rings(struct ifnet *);
869void netmap_enable_all_rings(struct ifnet *);
870void netmap_disable_ring(struct netmap_kring *kr);
871
872
873/* Structure associated to each thread which registered an interface.
874 *
875 * The first 4 fields of this structure are written by NIOCREGIF and
876 * read by poll() and NIOC?XSYNC.
877 * There is low contention among writers (actually, a correct user program
878 * should have no contention among writers) and among writers and readers,
879 * so we use a single global lock to protect the structure initialization.
880 * Since initialization involves the allocation of memory, we reuse the memory
881 * allocator lock.
882 * Read access to the structure is lock free. Readers must check that
883 * np_nifp is not NULL before using the other fields.
884 * If np_nifp is NULL initialization has not been performed, so they should
885 * return an error to userlevel.
886 *
887 * The ref_done field is used to regulate access to the refcount in the
888 * memory allocator. The refcount must be incremented at most once for
889 * each open("/dev/netmap"). The increment is performed by the first
890 * function that calls netmap_get_memory() (currently called by
891 * mmap(), NIOCGINFO and NIOCREGIF).
892 * If the refcount is incremented, it is then decremented when the
893 * private structure is destroyed.
894 */
895struct netmap_priv_d {
896 struct netmap_if * volatile np_nifp; /* netmap if descriptor. */
897
898 struct netmap_adapter *np_na;
899 int np_ringid; /* from the ioctl */
900 u_int np_qfirst, np_qlast; /* range of rings to scan */
901 uint16_t np_txpoll;
902
903 struct netmap_mem_d *np_mref; /* use with NMG_LOCK held */
904 /* np_refcount is only used on FreeBSD */
905 int np_refcount; /* use with NMG_LOCK held */
906};
907
908
909/*
910 * generic netmap emulation for devices that do not have
911 * native netmap support.
912 * XXX generic_netmap_register() is only exported to implement
913 * nma_is_generic().
914 */
915int generic_netmap_register(struct netmap_adapter *na, int enable);
916int generic_netmap_attach(struct ifnet *ifp);
917
918int netmap_catch_rx(struct netmap_adapter *na, int intercept);
919void generic_rx_handler(struct ifnet *ifp, struct mbuf *m);;
920void netmap_catch_packet_steering(struct netmap_generic_adapter *na, int enable);
921int generic_xmit_frame(struct ifnet *ifp, struct mbuf *m, void *addr, u_int len, u_int ring_nr);
922int generic_find_num_desc(struct ifnet *ifp, u_int *tx, u_int *rx);
923void generic_find_num_queues(struct ifnet *ifp, u_int *txq, u_int *rxq);
924
925static __inline int
926nma_is_generic(struct netmap_adapter *na)
927{
928 return na->nm_register == generic_netmap_register;
929}
930
931/*
932 * netmap_mitigation API. This is used by the generic adapter
933 * to reduce the number of interrupt requests/selwakeup
934 * to clients on incoming packets.
935 */
936void netmap_mitigation_init(struct netmap_generic_adapter *na);
937void netmap_mitigation_start(struct netmap_generic_adapter *na);
938void netmap_mitigation_restart(struct netmap_generic_adapter *na);
939int netmap_mitigation_active(struct netmap_generic_adapter *na);
940void netmap_mitigation_cleanup(struct netmap_generic_adapter *na);
941
942// int generic_timer_handler(struct hrtimer *t);
943
944#endif /* _NET_NETMAP_KERN_H_ */