rtsock: Fix NET_RT_FLAGS support
[dragonfly.git] / tools / tools / netrate / pktgen / pktgen.c
CommitLineData
4fa8e46e 1/*
391741ec 2 * Copyright (c) 2012 The DragonFly Project. All rights reserved.
4fa8e46e
SZ
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Sepherosa Ziehau <sepherosa@gmail.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
4fa8e46e
SZ
33 */
34
35#define _IP_VHL
36
37#include <sys/param.h>
38#include <sys/conf.h>
39#include <sys/device.h>
4fa8e46e
SZ
40#include <sys/in_cksum.h>
41#include <sys/kernel.h>
42#include <sys/malloc.h>
43#include <sys/mbuf.h>
44#include <sys/proc.h>
9eaf083f 45#include <sys/priv.h>
391741ec 46#include <sys/queue.h>
4fa8e46e
SZ
47#include <sys/socket.h>
48#include <sys/systm.h>
49#include <sys/serialize.h>
50
51#include <net/if.h>
52#include <net/if_dl.h>
53#include <net/if_var.h>
54#include <net/ifq_var.h>
55#include <net/ethernet.h>
391741ec 56#include <net/netmsg2.h>
5337421c 57#include <net/netisr2.h>
4fa8e46e
SZ
58
59#include <netinet/in.h>
9064712f 60#include <netinet/in_pcb.h>
4fa8e46e
SZ
61#include <netinet/ip.h>
62#include <netinet/udp_var.h>
63
64#include "pktgen.h"
65
66#define CDEV_NAME "pktg"
4fa8e46e 67
391741ec
SZ
68#define PKTGEN_BUFSZ 2048
69
390d8a05
SZ
70#ifndef PKTGEN_DEVCNT
71#define PKTGEN_DEVCNT 4
72#endif
73
391741ec
SZ
74struct pktgen;
75
b0cd2b4b
SZ
76struct netmsg_pktgen {
77 struct netmsg_base np_base;
78 struct pktgen *np_pktg;
79 struct ifaltq_subque *np_ifsq;
80};
81
391741ec
SZ
82struct pktgen_buf {
83 struct netmsg_base pb_nmsg; /* MUST BE THE FIRST */
84 void *pb_buf;
85 volatile int pb_done;
4c9906cb 86 int pb_inuse;
391741ec 87 struct ifnet *pb_ifp;
2fb36fab 88 struct ifaltq_subque *pb_ifsq;
391741ec
SZ
89 int pb_len;
90 int pb_cpuid;
91 struct pktgen *pb_pktg;
92 LIST_ENTRY(pktgen_buf) pb_link;
93};
94
1bbaa2ad
SZ
95struct pktgen_pcpu {
96 struct callout pktg_stop;
97 LIST_HEAD(, pktgen_buf) pktg_buflist;
98};
99
4fa8e46e
SZ
100struct pktgen {
101 uint32_t pktg_flags; /* PKTG_F_ */
102 int pktg_refcnt;
103
4fa8e46e 104 int pktg_duration;
4fa8e46e
SZ
105
106 int pktg_datalen;
4fa8e46e 107 struct ifnet *pktg_ifp;
c95ebcd6 108
793f802f
SZ
109 int pktg_pktenq;
110
391741ec
SZ
111 struct sockaddr_in pktg_src;
112 int pktg_ndst;
113 struct sockaddr_in *pktg_dst;
4fa8e46e 114 uint8_t pktg_dst_lladdr[ETHER_ADDR_LEN];
391741ec 115
1bbaa2ad 116 struct pktgen_pcpu pktg_pcpu[MAXCPU];
4fa8e46e
SZ
117};
118
eec66bd4
SZ
119#define PKTG_F_CONFIG 0x1
120#define PKTG_F_RUNNING 0x4
121#define PKTG_F_SWITCH_SRCDST 0x8
4fa8e46e
SZ
122
123static int pktgen_modevent(module_t, int, void *);
391741ec
SZ
124
125static void pktgen_buf_free(void *);
126static void pktgen_buf_ref(void *);
127static void pktgen_buf_send(netmsg_t);
128
4fa8e46e 129static int pktgen_config(struct pktgen *,
391741ec 130 const struct pktgen_conf *);
b0cd2b4b 131static int pktgen_start(struct pktgen *, int);
391741ec 132static void pktgen_free(struct pktgen *);
c6830078 133static void pktgen_ref(struct pktgen *);
1bbaa2ad 134static void pktgen_pcpu_stop_cb(void *);
391741ec 135static void pktgen_mbuf(struct pktgen_buf *, struct mbuf *);
b0cd2b4b
SZ
136static void pktgen_start_ifsq(struct pktgen *,
137 struct ifaltq_subque *);
138static void pktgen_start_ifsq_handler(netmsg_t);
4fa8e46e
SZ
139
140static d_open_t pktgen_open;
141static d_close_t pktgen_close;
142static d_ioctl_t pktgen_ioctl;
143
144static struct dev_ops pktgen_ops = {
624409c2 145 { CDEV_NAME, 0, D_MPSAFE },
4fa8e46e
SZ
146 .d_open = pktgen_open,
147 .d_close = pktgen_close,
148 .d_ioctl = pktgen_ioctl,
149};
150
4c9906cb 151static volatile int pktgen_refcnt;
e3d0d8d7 152static struct lwkt_token pktgen_tok = LWKT_TOKEN_INITIALIZER(pktgen_token);
4fa8e46e
SZ
153
154MALLOC_DECLARE(M_PKTGEN);
155MALLOC_DEFINE(M_PKTGEN, CDEV_NAME, "Packet generator");
156
157DEV_MODULE(pktgen, pktgen_modevent, NULL);
158
159static int
160pktgen_modevent(module_t mod, int type, void *data)
161{
390d8a05 162 int error = 0, i;
4fa8e46e
SZ
163
164 switch (type) {
165 case MOD_LOAD:
390d8a05
SZ
166 for (i = 0; i < PKTGEN_DEVCNT; ++i) {
167 make_dev(&pktgen_ops, 0, UID_ROOT, GID_WHEEL, 0600,
168 CDEV_NAME"%d", i);
169 }
4fa8e46e
SZ
170 break;
171
172 case MOD_UNLOAD:
173 if (pktgen_refcnt > 0)
174 return EBUSY;
4b3b3198 175 dev_ops_remove_all(&pktgen_ops);
4fa8e46e
SZ
176 break;
177
178 default:
179 error = EOPNOTSUPP;
180 break;
181 }
182 return error;
183}
184
185static int
186pktgen_open(struct dev_open_args *ap)
187{
188 cdev_t dev = ap->a_head.a_dev;
189 struct pktgen *pktg;
1bbaa2ad 190 int error, i;
4fa8e46e 191
9eaf083f 192 error = priv_check_cred(ap->a_cred, PRIV_ROOT, 0);
4fa8e46e
SZ
193 if (error)
194 return error;
195
e3d0d8d7 196 lwkt_gettoken(&pktgen_tok);
4fa8e46e
SZ
197
198 if (dev->si_drv1 != NULL) {
e3d0d8d7 199 lwkt_reltoken(&pktgen_tok);
4fa8e46e
SZ
200 return EBUSY;
201 }
202
203 pktg = kmalloc(sizeof(*pktg), M_PKTGEN, M_ZERO | M_WAITOK);
1bbaa2ad
SZ
204 for (i = 0; i < ncpus; ++i) {
205 struct pktgen_pcpu *p = &pktg->pktg_pcpu[i];
206
207 callout_init_mp(&p->pktg_stop);
208 LIST_INIT(&p->pktg_buflist);
209 }
4fa8e46e 210
4fa8e46e
SZ
211 dev->si_drv1 = pktg;
212 pktg->pktg_refcnt = 1;
213
4c9906cb 214 atomic_add_int(&pktgen_refcnt, 1);
4fa8e46e 215
e3d0d8d7 216 lwkt_reltoken(&pktgen_tok);
4fa8e46e
SZ
217 return 0;
218}
219
220static int
221pktgen_close(struct dev_close_args *ap)
222{
223 cdev_t dev = ap->a_head.a_dev;
224 struct pktgen *pktg = dev->si_drv1;
225
e3d0d8d7 226 lwkt_gettoken(&pktgen_tok);
4fa8e46e 227 dev->si_drv1 = NULL;
e3d0d8d7 228 lwkt_reltoken(&pktgen_tok);
391741ec 229
4c9906cb
SZ
230 pktgen_free(pktg);
231
4fa8e46e
SZ
232 return 0;
233}
234
235static int
236pktgen_ioctl(struct dev_ioctl_args *ap __unused)
237{
238 cdev_t dev = ap->a_head.a_dev;
239 caddr_t data = ap->a_data;
240 struct pktgen *pktg = dev->si_drv1;
241 int error;
242
e3d0d8d7 243 lwkt_gettoken(&pktgen_tok);
4fa8e46e
SZ
244
245 switch (ap->a_cmd) {
246 case PKTGENSTART:
b0cd2b4b
SZ
247 error = pktgen_start(pktg, 0);
248 break;
249
250 case PKTGENMQSTART:
251 error = pktgen_start(pktg, 1);
4fa8e46e
SZ
252 break;
253
254 case PKTGENSCONF:
255 error = pktgen_config(pktg, (const struct pktgen_conf *)data);
256 break;
257
258 default:
259 error = EOPNOTSUPP;
260 break;
261 }
262
e3d0d8d7 263 lwkt_reltoken(&pktgen_tok);
4fa8e46e
SZ
264 return error;
265}
266
267static int
268pktgen_config(struct pktgen *pktg, const struct pktgen_conf *conf)
269{
270 const struct sockaddr_in *sin;
391741ec 271 struct sockaddr_in *dst = NULL;
4fa8e46e
SZ
272 const struct sockaddr *sa;
273 struct ifnet *ifp;
391741ec 274 size_t dst_size;
793f802f 275 int i, error, pktenq;
4fa8e46e 276
391741ec 277 if (pktg->pktg_flags & (PKTG_F_RUNNING | PKTG_F_CONFIG))
4fa8e46e
SZ
278 return EBUSY;
279
391741ec
SZ
280 if (conf->pc_datalen <= 0 ||
281 conf->pc_datalen > ETHERMTU - sizeof(struct udpiphdr))
4fa8e46e
SZ
282 return EINVAL;
283 if (conf->pc_duration <= 0)
284 return EINVAL;
285
4fa8e46e
SZ
286 sin = &conf->pc_src;
287 if (sin->sin_family != AF_INET)
288 return EPROTONOSUPPORT;
289 if (sin->sin_port == 0)
290 return EINVAL;
291
391741ec 292 if (conf->pc_ndst <= 0)
4fa8e46e 293 return EINVAL;
391741ec
SZ
294 dst_size = conf->pc_ndst * sizeof(struct sockaddr_in);
295
296 dst = kmalloc(dst_size, M_PKTGEN, M_WAITOK | M_NULLOK);
297 if (dst == NULL)
298 return ENOMEM;
4fa8e46e 299
391741ec
SZ
300 error = copyin(conf->pc_dst, dst, dst_size);
301 if (error)
302 goto failed;
303
304 for (i = 0; i < conf->pc_ndst; ++i) {
305 sin = &dst[i];
306 if (sin->sin_family != AF_INET) {
307 error = EPROTONOSUPPORT;
308 goto failed;
309 }
310 if (sin->sin_port == 0) {
311 error = EINVAL;
312 goto failed;
313 }
314 }
315
316 ifp = ifunit(conf->pc_ifname);
317 if (ifp == NULL) {
318 error = ENXIO;
319 goto failed;
320 }
321
793f802f 322 pktenq = conf->pc_pktenq;
2fb36fab 323 if (pktenq < 0 || pktenq > ifp->if_snd.altq_maxlen) {
793f802f
SZ
324 error = ENOBUFS;
325 goto failed;
326 } else if (pktenq == 0) {
2fb36fab 327 pktenq = (ifp->if_snd.altq_maxlen * 3) / 4;
793f802f
SZ
328 }
329
391741ec
SZ
330 sa = &conf->pc_dst_lladdr;
331 if (sa->sa_family != AF_LINK) {
332 error = EPROTONOSUPPORT;
333 goto failed;
334 }
335 if (sa->sa_len != ETHER_ADDR_LEN) {
336 error = EPROTONOSUPPORT;
337 goto failed;
338 }
339 if (ETHER_IS_MULTICAST(sa->sa_data) ||
340 bcmp(sa->sa_data, ifp->if_broadcastaddr, ifp->if_addrlen) == 0) {
341 error = EADDRNOTAVAIL;
342 goto failed;
343 }
344
345 /*
346 * Accept the config
347 */
4fa8e46e 348 pktg->pktg_flags |= PKTG_F_CONFIG;
4fa8e46e 349
eec66bd4
SZ
350 if (conf->pc_flags & PKTGEN_FLAG_SWITCH_SRCDST)
351 pktg->pktg_flags |= PKTG_F_SWITCH_SRCDST;
4fa8e46e 352 pktg->pktg_duration = conf->pc_duration;
4fa8e46e 353 pktg->pktg_datalen = conf->pc_datalen;
793f802f 354 pktg->pktg_pktenq = pktenq;
391741ec
SZ
355 pktg->pktg_ifp = ifp;
356 pktg->pktg_src = conf->pc_src;
357 pktg->pktg_ndst = conf->pc_ndst;
358 KKASSERT(pktg->pktg_dst == NULL);
359 pktg->pktg_dst = dst;
4fa8e46e 360 bcopy(sa->sa_data, pktg->pktg_dst_lladdr, ETHER_ADDR_LEN);
c95ebcd6 361
4fa8e46e 362 return 0;
391741ec
SZ
363
364failed:
365 if (dst != NULL)
366 kfree(dst, M_PKTGEN);
367 return error;
4fa8e46e
SZ
368}
369
b0cd2b4b
SZ
370static void
371pktgen_start_ifsq(struct pktgen *pktg, struct ifaltq_subque *ifsq)
372{
373 struct netmsg_pktgen *np;
374
375 np = kmalloc(sizeof(*np), M_LWKTMSG, M_WAITOK);
376 netmsg_init(&np->np_base, NULL, &netisr_afree_rport, 0,
377 pktgen_start_ifsq_handler);
378 np->np_pktg = pktg;
379 np->np_ifsq = ifsq;
380
ec7f7fc8 381 lwkt_sendmsg(netisr_cpuport(ifsq_get_cpuid(ifsq)), &np->np_base.lmsg);
b0cd2b4b
SZ
382}
383
4fa8e46e 384static int
b0cd2b4b 385pktgen_start(struct pktgen *pktg, int mq)
4fa8e46e 386{
b0cd2b4b
SZ
387 struct ifaltq *ifq;
388
389 if ((pktg->pktg_flags & PKTG_F_CONFIG) == 0)
390 return EINVAL;
391 if (pktg->pktg_flags & PKTG_F_RUNNING)
392 return EBUSY;
393 pktg->pktg_flags |= PKTG_F_RUNNING;
394
395 ifq = &pktg->pktg_ifp->if_snd;
396 if (!mq) {
397 pktgen_ref(pktg);
398 pktgen_start_ifsq(pktg, ifq_get_subq_default(ifq));
399 } else {
400 int i;
401
402 for (i = 0; i < ifq->altq_subq_cnt; ++i)
403 pktgen_ref(pktg);
404 for (i = 0; i < ifq->altq_subq_cnt; ++i)
405 pktgen_start_ifsq(pktg, ifq_get_subq(ifq, i));
406 }
407 return 0;
408}
409
410static void
411pktgen_start_ifsq_handler(netmsg_t nmsg)
412{
413 struct netmsg_pktgen *np = (struct netmsg_pktgen *)nmsg;
414 struct pktgen *pktg = np->np_pktg;
415 struct ifaltq_subque *ifsq = np->np_ifsq;
416
391741ec
SZ
417 struct mbuf *m, *head = NULL, **next;
418 struct ifnet *ifp;
1bbaa2ad 419 struct pktgen_pcpu *p;
b0cd2b4b 420 int cpuid, i, alloc_cnt, keep_cnt;
391741ec
SZ
421
422 u_short ulen, psum;
423 int len, ip_len;
424
b0cd2b4b
SZ
425 /* Reply ASAP */
426 lwkt_replymsg(&np->np_base.lmsg, 0);
4fa8e46e 427
391741ec 428 ifp = pktg->pktg_ifp;
4fa8e46e 429
2fb36fab 430 cpuid = ifsq_get_cpuid(ifsq);
b0cd2b4b 431 KKASSERT(cpuid == mycpuid);
4fa8e46e 432
1bbaa2ad
SZ
433 p = &pktg->pktg_pcpu[cpuid];
434
793f802f
SZ
435 keep_cnt = pktg->pktg_pktenq;
436 alloc_cnt = keep_cnt * 2;
4fa8e46e 437
391741ec
SZ
438 /*
439 * Prefault enough mbuf into mbuf objcache
440 */
441 next = &head;
442 for (i = 0; i < alloc_cnt; ++i) {
b5523eac 443 MGETHDR(m, M_WAITOK, MT_DATA);
391741ec
SZ
444 *next = m;
445 next = &m->m_nextpkt;
446 }
4fa8e46e 447
391741ec
SZ
448 for (i = 0; i < alloc_cnt - keep_cnt; ++i) {
449 m = head;
450 head = m->m_nextpkt;
451 m->m_nextpkt = NULL;
452 m_freem(m);
453 }
454 KKASSERT(head != NULL);
4fa8e46e 455
391741ec
SZ
456 /*
457 * Setup the packets' data
458 */
459 ip_len = pktg->pktg_datalen + sizeof(struct udpiphdr);
4fa8e46e
SZ
460 len = ip_len + ETHER_HDR_LEN;
461
391741ec
SZ
462 psum = htons((u_short)pktg->pktg_datalen + sizeof(struct udphdr) +
463 IPPROTO_UDP);
4fa8e46e
SZ
464 ulen = htons(pktg->pktg_datalen + sizeof(struct udphdr));
465
391741ec
SZ
466 m = head;
467 i = 0;
468 while (m != NULL) {
469 struct mbuf *nextm;
470 const struct sockaddr_in *dst;
471 struct pktgen_buf *pb;
472 struct ip *ip;
473 struct udpiphdr *ui;
474 struct ether_header *eh;
475
c6830078 476 pktgen_ref(pktg);
391741ec
SZ
477
478 pb = kmalloc(sizeof(*pb), M_PKTGEN, M_WAITOK | M_ZERO);
479 pb->pb_ifp = ifp;
2fb36fab 480 pb->pb_ifsq = ifsq;
391741ec
SZ
481 pb->pb_inuse = 1;
482 pb->pb_buf = kmalloc(PKTGEN_BUFSZ, M_PKTGEN, M_WAITOK);
483 pb->pb_len = len;
484 pb->pb_cpuid = cpuid;
485 pb->pb_pktg = pktg;
486 netmsg_init(&pb->pb_nmsg, NULL, &netisr_adone_rport, 0,
487 pktgen_buf_send);
1bbaa2ad 488 LIST_INSERT_HEAD(&p->pktg_buflist, pb, pb_link);
391741ec
SZ
489
490 dst = &pktg->pktg_dst[i % pktg->pktg_ndst];
491 ++i;
492
493 m->m_ext.ext_arg = pb;
494 m->m_ext.ext_buf = pb->pb_buf;
495 m->m_ext.ext_free = pktgen_buf_free;
496 m->m_ext.ext_ref = pktgen_buf_ref;
497 m->m_ext.ext_size = PKTGEN_BUFSZ;
498
499 m->m_data = m->m_ext.ext_buf;
500 m->m_flags |= M_EXT;
4fa8e46e
SZ
501 m->m_len = m->m_pkthdr.len = len;
502
391741ec
SZ
503 m->m_data += ETHER_HDR_LEN;
504 m->m_len -= ETHER_HDR_LEN;
505 m->m_pkthdr.len -= ETHER_HDR_LEN;
4fa8e46e
SZ
506
507 ui = mtod(m, struct udpiphdr *);
508 ui->ui_pr = IPPROTO_UDP;
eec66bd4
SZ
509 if (pktg->pktg_flags & PKTG_F_SWITCH_SRCDST) {
510 ui->ui_src.s_addr = dst->sin_addr.s_addr;
511 ui->ui_dst.s_addr = pktg->pktg_src.sin_addr.s_addr;
512 ui->ui_sport = dst->sin_port;
513 ui->ui_dport = pktg->pktg_src.sin_port;
514 } else {
515 ui->ui_src.s_addr = pktg->pktg_src.sin_addr.s_addr;
516 ui->ui_dst.s_addr = dst->sin_addr.s_addr;
517 ui->ui_sport = pktg->pktg_src.sin_port;
518 ui->ui_dport = dst->sin_port;
519 }
4fa8e46e 520 ui->ui_ulen = ulen;
391741ec
SZ
521 ui->ui_sum = in_pseudo(ui->ui_src.s_addr, ui->ui_dst.s_addr,
522 psum);
64c94a5f 523 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
4fa8e46e
SZ
524
525 ip = (struct ip *)ui;
526 ip->ip_len = ip_len;
527 ip->ip_ttl = 64; /* XXX */
528 ip->ip_tos = 0; /* XXX */
529 ip->ip_vhl = IP_VHL_BORING;
530 ip->ip_off = 0;
2640b9e5 531 ip->ip_sum = 0;
4fa8e46e
SZ
532 ip->ip_id = ip_newid();
533
391741ec 534 in_delayed_cksum(m);
4fa8e46e
SZ
535
536 ip->ip_len = htons(ip->ip_len);
391741ec
SZ
537 ip->ip_sum = in_cksum_hdr(ip);
538
539 m->m_data -= ETHER_HDR_LEN;
540 m->m_len += ETHER_HDR_LEN;
541 m->m_pkthdr.len += ETHER_HDR_LEN;
c95ebcd6 542
c95ebcd6
SZ
543 eh = mtod(m, struct ether_header *);
544 bcopy(pktg->pktg_dst_lladdr, eh->ether_dhost, ETHER_ADDR_LEN);
545 bcopy(IF_LLADDR(ifp), eh->ether_shost, ETHER_ADDR_LEN);
546 eh->ether_type = htons(ETHERTYPE_IP);
547
391741ec
SZ
548 nextm = m->m_nextpkt;
549 m->m_nextpkt = NULL;
c95ebcd6 550
391741ec 551 ifq_dispatch(ifp, m, NULL);
c95ebcd6 552
391741ec
SZ
553 m = nextm;
554 }
c95ebcd6 555
1bbaa2ad
SZ
556 callout_reset(&p->pktg_stop, pktg->pktg_duration * hz,
557 pktgen_pcpu_stop_cb, p);
c95ebcd6 558
b0cd2b4b 559 pktgen_free(pktg);
391741ec 560}
c95ebcd6 561
391741ec
SZ
562static void
563pktgen_mbuf(struct pktgen_buf *pb, struct mbuf *m)
564{
565 m->m_ext.ext_arg = pb;
566 m->m_ext.ext_buf = pb->pb_buf;
567 m->m_ext.ext_free = pktgen_buf_free;
568 m->m_ext.ext_ref = pktgen_buf_ref;
569 m->m_ext.ext_size = PKTGEN_BUFSZ;
570
571 m->m_data = m->m_ext.ext_buf;
572 m->m_flags |= M_EXT;
573 m->m_len = m->m_pkthdr.len = pb->pb_len;
574}
c95ebcd6 575
391741ec
SZ
576static void
577pktgen_buf_send(netmsg_t msg)
578{
579 struct pktgen_buf *pb = (struct pktgen_buf *)msg;
580 struct mbuf *m;
c95ebcd6 581
ec7f7fc8 582 KKASSERT(&curthread->td_msgport == netisr_cpuport(pb->pb_cpuid));
c95ebcd6 583
391741ec
SZ
584 crit_enter();
585 lwkt_replymsg(&pb->pb_nmsg.lmsg, 0);
586 crit_exit();
c95ebcd6 587
b5523eac 588 MGETHDR(m, M_WAITOK, MT_DATA);
391741ec
SZ
589 pktgen_mbuf(pb, m);
590 ifq_dispatch(pb->pb_ifp, m, NULL);
591}
c95ebcd6 592
391741ec
SZ
593static void
594pktgen_buf_free(void *arg)
595{
596 struct pktgen_buf *pb = arg;
597 struct mbuf *m;
c95ebcd6 598
391741ec
SZ
599 KKASSERT(pb->pb_inuse > 0);
600 if (pb->pb_done) {
601 if (atomic_fetchadd_int(&pb->pb_inuse, -1) == 1) {
602 struct pktgen *pktg;
c95ebcd6 603
391741ec 604 pktg = pb->pb_pktg;
4c9906cb 605 crit_enter();
391741ec 606 LIST_REMOVE(pb, pb_link);
4c9906cb 607 crit_exit();
391741ec
SZ
608 kfree(pb->pb_buf, M_PKTGEN);
609 kfree(pb, M_PKTGEN);
4fa8e46e 610
391741ec 611 pktgen_free(pktg);
4fa8e46e 612 }
391741ec 613 return;
4fa8e46e 614 }
db161a19 615
ec7f7fc8 616 if (&curthread->td_msgport != netisr_cpuport(pb->pb_cpuid)) {
391741ec
SZ
617 KKASSERT(pb->pb_cpuid == mycpuid);
618 crit_enter();
619 KKASSERT(pb->pb_nmsg.lmsg.ms_flags & MSGF_DONE);
ec7f7fc8 620 lwkt_sendmsg(netisr_cpuport(pb->pb_cpuid), &pb->pb_nmsg.lmsg);
391741ec
SZ
621 crit_exit();
622 return;
623 }
624
b5523eac 625 MGETHDR(m, M_WAITOK, MT_DATA);
391741ec 626 pktgen_mbuf(pb, m);
2fb36fab 627 ifsq_enqueue(pb->pb_ifsq, m, NULL);
db161a19
SZ
628}
629
630static void
391741ec 631pktgen_buf_ref(void *arg)
db161a19 632{
391741ec 633 struct pktgen_buf *pb = arg;
db161a19 634
391741ec 635 panic("%s should never be called\n", __func__);
4fa8e46e 636
391741ec
SZ
637 KKASSERT(pb->pb_inuse > 0);
638 atomic_add_int(&pb->pb_inuse, 1);
639}
4fa8e46e 640
391741ec
SZ
641static void
642pktgen_free(struct pktgen *pktg)
643{
4fa8e46e 644 KKASSERT(pktg->pktg_refcnt > 0);
4c9906cb 645 if (atomic_fetchadd_int(&pktg->pktg_refcnt, -1) == 1) {
1bbaa2ad
SZ
646 int i;
647
391741ec
SZ
648 if (pktg->pktg_dst != NULL)
649 kfree(pktg->pktg_dst, M_PKTGEN);
1bbaa2ad
SZ
650
651 for (i = 0; i < ncpus; ++i)
652 KKASSERT(LIST_EMPTY(&pktg->pktg_pcpu[i].pktg_buflist));
391741ec
SZ
653 kfree(pktg, M_PKTGEN);
654 }
4fa8e46e
SZ
655
656 KKASSERT(pktgen_refcnt > 0);
4c9906cb 657 atomic_subtract_int(&pktgen_refcnt, 1);
391741ec
SZ
658}
659
660static void
1bbaa2ad 661pktgen_pcpu_stop_cb(void *arg)
391741ec 662{
1bbaa2ad 663 struct pktgen_pcpu *p = arg;
391741ec 664 struct pktgen_buf *pb;
4fa8e46e 665
4c9906cb 666 crit_enter();
1bbaa2ad 667 LIST_FOREACH(pb, &p->pktg_buflist, pb_link)
391741ec 668 pb->pb_done = 1;
4c9906cb 669 crit_exit();
4fa8e46e 670}
c6830078
SZ
671
672static void
673pktgen_ref(struct pktgen *pktg)
674{
675 atomic_add_int(&pktg->pktg_refcnt, 1);
676 atomic_add_int(&pktgen_refcnt, 1);
677}