Make uaudio(4) compilable into the kernel.
[dragonfly.git] / tools / tools / netrate / pktgen / pktgen.c
CommitLineData
4fa8e46e 1/*
391741ec 2 * Copyright (c) 2012 The DragonFly Project. All rights reserved.
4fa8e46e
SZ
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Sepherosa Ziehau <sepherosa@gmail.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
db161a19 34 * $DragonFly: src/tools/tools/netrate/pktgen/pktgen.c,v 1.4 2008/04/02 14:18:55 sephe Exp $
4fa8e46e
SZ
35 */
36
37#define _IP_VHL
38
39#include <sys/param.h>
40#include <sys/conf.h>
41#include <sys/device.h>
4fa8e46e
SZ
42#include <sys/in_cksum.h>
43#include <sys/kernel.h>
44#include <sys/malloc.h>
45#include <sys/mbuf.h>
46#include <sys/proc.h>
9eaf083f 47#include <sys/priv.h>
391741ec 48#include <sys/queue.h>
4fa8e46e
SZ
49#include <sys/socket.h>
50#include <sys/systm.h>
51#include <sys/serialize.h>
52
53#include <net/if.h>
54#include <net/if_dl.h>
55#include <net/if_var.h>
56#include <net/ifq_var.h>
57#include <net/ethernet.h>
391741ec 58#include <net/netmsg2.h>
5337421c 59#include <net/netisr2.h>
4fa8e46e
SZ
60
61#include <netinet/in.h>
62#include <netinet/ip.h>
63#include <netinet/udp_var.h>
64
65#include "pktgen.h"
66
67#define CDEV_NAME "pktg"
4fa8e46e 68
391741ec
SZ
69#define PKTGEN_BUFSZ 2048
70
390d8a05
SZ
71#ifndef PKTGEN_DEVCNT
72#define PKTGEN_DEVCNT 4
73#endif
74
391741ec
SZ
75struct pktgen;
76
b0cd2b4b
SZ
77struct netmsg_pktgen {
78 struct netmsg_base np_base;
79 struct pktgen *np_pktg;
80 struct ifaltq_subque *np_ifsq;
81};
82
391741ec
SZ
83struct pktgen_buf {
84 struct netmsg_base pb_nmsg; /* MUST BE THE FIRST */
85 void *pb_buf;
86 volatile int pb_done;
4c9906cb 87 int pb_inuse;
391741ec 88 struct ifnet *pb_ifp;
2fb36fab 89 struct ifaltq_subque *pb_ifsq;
391741ec
SZ
90 int pb_len;
91 int pb_cpuid;
92 struct pktgen *pb_pktg;
93 LIST_ENTRY(pktgen_buf) pb_link;
94};
95
1bbaa2ad
SZ
96struct pktgen_pcpu {
97 struct callout pktg_stop;
98 LIST_HEAD(, pktgen_buf) pktg_buflist;
99};
100
4fa8e46e
SZ
101struct pktgen {
102 uint32_t pktg_flags; /* PKTG_F_ */
103 int pktg_refcnt;
104
4fa8e46e 105 int pktg_duration;
4fa8e46e
SZ
106
107 int pktg_datalen;
4fa8e46e 108 struct ifnet *pktg_ifp;
c95ebcd6 109
793f802f
SZ
110 int pktg_pktenq;
111
391741ec
SZ
112 struct sockaddr_in pktg_src;
113 int pktg_ndst;
114 struct sockaddr_in *pktg_dst;
4fa8e46e 115 uint8_t pktg_dst_lladdr[ETHER_ADDR_LEN];
391741ec 116
1bbaa2ad 117 struct pktgen_pcpu pktg_pcpu[MAXCPU];
4fa8e46e
SZ
118};
119
eec66bd4
SZ
120#define PKTG_F_CONFIG 0x1
121#define PKTG_F_RUNNING 0x4
122#define PKTG_F_SWITCH_SRCDST 0x8
4fa8e46e
SZ
123
124static int pktgen_modevent(module_t, int, void *);
391741ec
SZ
125
126static void pktgen_buf_free(void *);
127static void pktgen_buf_ref(void *);
128static void pktgen_buf_send(netmsg_t);
129
4fa8e46e 130static int pktgen_config(struct pktgen *,
391741ec 131 const struct pktgen_conf *);
b0cd2b4b 132static int pktgen_start(struct pktgen *, int);
391741ec 133static void pktgen_free(struct pktgen *);
c6830078 134static void pktgen_ref(struct pktgen *);
1bbaa2ad 135static void pktgen_pcpu_stop_cb(void *);
391741ec 136static void pktgen_mbuf(struct pktgen_buf *, struct mbuf *);
b0cd2b4b
SZ
137static void pktgen_start_ifsq(struct pktgen *,
138 struct ifaltq_subque *);
139static void pktgen_start_ifsq_handler(netmsg_t);
4fa8e46e
SZ
140
141static d_open_t pktgen_open;
142static d_close_t pktgen_close;
143static d_ioctl_t pktgen_ioctl;
144
145static struct dev_ops pktgen_ops = {
624409c2 146 { CDEV_NAME, 0, D_MPSAFE },
4fa8e46e
SZ
147 .d_open = pktgen_open,
148 .d_close = pktgen_close,
149 .d_ioctl = pktgen_ioctl,
150};
151
4c9906cb 152static volatile int pktgen_refcnt;
e3d0d8d7 153static struct lwkt_token pktgen_tok = LWKT_TOKEN_INITIALIZER(pktgen_token);
4fa8e46e
SZ
154
155MALLOC_DECLARE(M_PKTGEN);
156MALLOC_DEFINE(M_PKTGEN, CDEV_NAME, "Packet generator");
157
158DEV_MODULE(pktgen, pktgen_modevent, NULL);
159
160static int
161pktgen_modevent(module_t mod, int type, void *data)
162{
390d8a05 163 int error = 0, i;
4fa8e46e
SZ
164
165 switch (type) {
166 case MOD_LOAD:
390d8a05
SZ
167 for (i = 0; i < PKTGEN_DEVCNT; ++i) {
168 make_dev(&pktgen_ops, 0, UID_ROOT, GID_WHEEL, 0600,
169 CDEV_NAME"%d", i);
170 }
4fa8e46e
SZ
171 break;
172
173 case MOD_UNLOAD:
174 if (pktgen_refcnt > 0)
175 return EBUSY;
4b3b3198 176 dev_ops_remove_all(&pktgen_ops);
4fa8e46e
SZ
177 break;
178
179 default:
180 error = EOPNOTSUPP;
181 break;
182 }
183 return error;
184}
185
186static int
187pktgen_open(struct dev_open_args *ap)
188{
189 cdev_t dev = ap->a_head.a_dev;
190 struct pktgen *pktg;
1bbaa2ad 191 int error, i;
4fa8e46e 192
9eaf083f 193 error = priv_check_cred(ap->a_cred, PRIV_ROOT, 0);
4fa8e46e
SZ
194 if (error)
195 return error;
196
e3d0d8d7 197 lwkt_gettoken(&pktgen_tok);
4fa8e46e
SZ
198
199 if (dev->si_drv1 != NULL) {
e3d0d8d7 200 lwkt_reltoken(&pktgen_tok);
4fa8e46e
SZ
201 return EBUSY;
202 }
203
204 pktg = kmalloc(sizeof(*pktg), M_PKTGEN, M_ZERO | M_WAITOK);
1bbaa2ad
SZ
205 for (i = 0; i < ncpus; ++i) {
206 struct pktgen_pcpu *p = &pktg->pktg_pcpu[i];
207
208 callout_init_mp(&p->pktg_stop);
209 LIST_INIT(&p->pktg_buflist);
210 }
4fa8e46e 211
4fa8e46e
SZ
212 dev->si_drv1 = pktg;
213 pktg->pktg_refcnt = 1;
214
4c9906cb 215 atomic_add_int(&pktgen_refcnt, 1);
4fa8e46e 216
e3d0d8d7 217 lwkt_reltoken(&pktgen_tok);
4fa8e46e
SZ
218 return 0;
219}
220
221static int
222pktgen_close(struct dev_close_args *ap)
223{
224 cdev_t dev = ap->a_head.a_dev;
225 struct pktgen *pktg = dev->si_drv1;
226
e3d0d8d7 227 lwkt_gettoken(&pktgen_tok);
4fa8e46e 228 dev->si_drv1 = NULL;
e3d0d8d7 229 lwkt_reltoken(&pktgen_tok);
391741ec 230
4c9906cb
SZ
231 pktgen_free(pktg);
232
4fa8e46e
SZ
233 return 0;
234}
235
236static int
237pktgen_ioctl(struct dev_ioctl_args *ap __unused)
238{
239 cdev_t dev = ap->a_head.a_dev;
240 caddr_t data = ap->a_data;
241 struct pktgen *pktg = dev->si_drv1;
242 int error;
243
e3d0d8d7 244 lwkt_gettoken(&pktgen_tok);
4fa8e46e
SZ
245
246 switch (ap->a_cmd) {
247 case PKTGENSTART:
b0cd2b4b
SZ
248 error = pktgen_start(pktg, 0);
249 break;
250
251 case PKTGENMQSTART:
252 error = pktgen_start(pktg, 1);
4fa8e46e
SZ
253 break;
254
255 case PKTGENSCONF:
256 error = pktgen_config(pktg, (const struct pktgen_conf *)data);
257 break;
258
259 default:
260 error = EOPNOTSUPP;
261 break;
262 }
263
e3d0d8d7 264 lwkt_reltoken(&pktgen_tok);
4fa8e46e
SZ
265 return error;
266}
267
268static int
269pktgen_config(struct pktgen *pktg, const struct pktgen_conf *conf)
270{
271 const struct sockaddr_in *sin;
391741ec 272 struct sockaddr_in *dst = NULL;
4fa8e46e
SZ
273 const struct sockaddr *sa;
274 struct ifnet *ifp;
391741ec 275 size_t dst_size;
793f802f 276 int i, error, pktenq;
4fa8e46e 277
391741ec 278 if (pktg->pktg_flags & (PKTG_F_RUNNING | PKTG_F_CONFIG))
4fa8e46e
SZ
279 return EBUSY;
280
391741ec
SZ
281 if (conf->pc_datalen <= 0 ||
282 conf->pc_datalen > ETHERMTU - sizeof(struct udpiphdr))
4fa8e46e
SZ
283 return EINVAL;
284 if (conf->pc_duration <= 0)
285 return EINVAL;
286
4fa8e46e
SZ
287 sin = &conf->pc_src;
288 if (sin->sin_family != AF_INET)
289 return EPROTONOSUPPORT;
290 if (sin->sin_port == 0)
291 return EINVAL;
292
391741ec 293 if (conf->pc_ndst <= 0)
4fa8e46e 294 return EINVAL;
391741ec
SZ
295 dst_size = conf->pc_ndst * sizeof(struct sockaddr_in);
296
297 dst = kmalloc(dst_size, M_PKTGEN, M_WAITOK | M_NULLOK);
298 if (dst == NULL)
299 return ENOMEM;
4fa8e46e 300
391741ec
SZ
301 error = copyin(conf->pc_dst, dst, dst_size);
302 if (error)
303 goto failed;
304
305 for (i = 0; i < conf->pc_ndst; ++i) {
306 sin = &dst[i];
307 if (sin->sin_family != AF_INET) {
308 error = EPROTONOSUPPORT;
309 goto failed;
310 }
311 if (sin->sin_port == 0) {
312 error = EINVAL;
313 goto failed;
314 }
315 }
316
317 ifp = ifunit(conf->pc_ifname);
318 if (ifp == NULL) {
319 error = ENXIO;
320 goto failed;
321 }
322
793f802f 323 pktenq = conf->pc_pktenq;
2fb36fab 324 if (pktenq < 0 || pktenq > ifp->if_snd.altq_maxlen) {
793f802f
SZ
325 error = ENOBUFS;
326 goto failed;
327 } else if (pktenq == 0) {
2fb36fab 328 pktenq = (ifp->if_snd.altq_maxlen * 3) / 4;
793f802f
SZ
329 }
330
391741ec
SZ
331 sa = &conf->pc_dst_lladdr;
332 if (sa->sa_family != AF_LINK) {
333 error = EPROTONOSUPPORT;
334 goto failed;
335 }
336 if (sa->sa_len != ETHER_ADDR_LEN) {
337 error = EPROTONOSUPPORT;
338 goto failed;
339 }
340 if (ETHER_IS_MULTICAST(sa->sa_data) ||
341 bcmp(sa->sa_data, ifp->if_broadcastaddr, ifp->if_addrlen) == 0) {
342 error = EADDRNOTAVAIL;
343 goto failed;
344 }
345
346 /*
347 * Accept the config
348 */
4fa8e46e 349 pktg->pktg_flags |= PKTG_F_CONFIG;
4fa8e46e 350
eec66bd4
SZ
351 if (conf->pc_flags & PKTGEN_FLAG_SWITCH_SRCDST)
352 pktg->pktg_flags |= PKTG_F_SWITCH_SRCDST;
4fa8e46e 353 pktg->pktg_duration = conf->pc_duration;
4fa8e46e 354 pktg->pktg_datalen = conf->pc_datalen;
793f802f 355 pktg->pktg_pktenq = pktenq;
391741ec
SZ
356 pktg->pktg_ifp = ifp;
357 pktg->pktg_src = conf->pc_src;
358 pktg->pktg_ndst = conf->pc_ndst;
359 KKASSERT(pktg->pktg_dst == NULL);
360 pktg->pktg_dst = dst;
4fa8e46e 361 bcopy(sa->sa_data, pktg->pktg_dst_lladdr, ETHER_ADDR_LEN);
c95ebcd6 362
4fa8e46e 363 return 0;
391741ec
SZ
364
365failed:
366 if (dst != NULL)
367 kfree(dst, M_PKTGEN);
368 return error;
4fa8e46e
SZ
369}
370
b0cd2b4b
SZ
371static void
372pktgen_start_ifsq(struct pktgen *pktg, struct ifaltq_subque *ifsq)
373{
374 struct netmsg_pktgen *np;
375
376 np = kmalloc(sizeof(*np), M_LWKTMSG, M_WAITOK);
377 netmsg_init(&np->np_base, NULL, &netisr_afree_rport, 0,
378 pktgen_start_ifsq_handler);
379 np->np_pktg = pktg;
380 np->np_ifsq = ifsq;
381
ec7f7fc8 382 lwkt_sendmsg(netisr_cpuport(ifsq_get_cpuid(ifsq)), &np->np_base.lmsg);
b0cd2b4b
SZ
383}
384
4fa8e46e 385static int
b0cd2b4b 386pktgen_start(struct pktgen *pktg, int mq)
4fa8e46e 387{
b0cd2b4b
SZ
388 struct ifaltq *ifq;
389
390 if ((pktg->pktg_flags & PKTG_F_CONFIG) == 0)
391 return EINVAL;
392 if (pktg->pktg_flags & PKTG_F_RUNNING)
393 return EBUSY;
394 pktg->pktg_flags |= PKTG_F_RUNNING;
395
396 ifq = &pktg->pktg_ifp->if_snd;
397 if (!mq) {
398 pktgen_ref(pktg);
399 pktgen_start_ifsq(pktg, ifq_get_subq_default(ifq));
400 } else {
401 int i;
402
403 for (i = 0; i < ifq->altq_subq_cnt; ++i)
404 pktgen_ref(pktg);
405 for (i = 0; i < ifq->altq_subq_cnt; ++i)
406 pktgen_start_ifsq(pktg, ifq_get_subq(ifq, i));
407 }
408 return 0;
409}
410
411static void
412pktgen_start_ifsq_handler(netmsg_t nmsg)
413{
414 struct netmsg_pktgen *np = (struct netmsg_pktgen *)nmsg;
415 struct pktgen *pktg = np->np_pktg;
416 struct ifaltq_subque *ifsq = np->np_ifsq;
417
391741ec
SZ
418 struct mbuf *m, *head = NULL, **next;
419 struct ifnet *ifp;
1bbaa2ad 420 struct pktgen_pcpu *p;
b0cd2b4b 421 int cpuid, i, alloc_cnt, keep_cnt;
391741ec
SZ
422
423 u_short ulen, psum;
424 int len, ip_len;
425
b0cd2b4b
SZ
426 /* Reply ASAP */
427 lwkt_replymsg(&np->np_base.lmsg, 0);
4fa8e46e 428
391741ec 429 ifp = pktg->pktg_ifp;
4fa8e46e 430
2fb36fab 431 cpuid = ifsq_get_cpuid(ifsq);
b0cd2b4b 432 KKASSERT(cpuid == mycpuid);
4fa8e46e 433
1bbaa2ad
SZ
434 p = &pktg->pktg_pcpu[cpuid];
435
793f802f
SZ
436 keep_cnt = pktg->pktg_pktenq;
437 alloc_cnt = keep_cnt * 2;
4fa8e46e 438
391741ec
SZ
439 /*
440 * Prefault enough mbuf into mbuf objcache
441 */
442 next = &head;
443 for (i = 0; i < alloc_cnt; ++i) {
444 MGETHDR(m, MB_WAIT, MT_DATA);
445 *next = m;
446 next = &m->m_nextpkt;
447 }
4fa8e46e 448
391741ec
SZ
449 for (i = 0; i < alloc_cnt - keep_cnt; ++i) {
450 m = head;
451 head = m->m_nextpkt;
452 m->m_nextpkt = NULL;
453 m_freem(m);
454 }
455 KKASSERT(head != NULL);
4fa8e46e 456
391741ec
SZ
457 /*
458 * Setup the packets' data
459 */
460 ip_len = pktg->pktg_datalen + sizeof(struct udpiphdr);
4fa8e46e
SZ
461 len = ip_len + ETHER_HDR_LEN;
462
391741ec
SZ
463 psum = htons((u_short)pktg->pktg_datalen + sizeof(struct udphdr) +
464 IPPROTO_UDP);
4fa8e46e
SZ
465 ulen = htons(pktg->pktg_datalen + sizeof(struct udphdr));
466
391741ec
SZ
467 m = head;
468 i = 0;
469 while (m != NULL) {
470 struct mbuf *nextm;
471 const struct sockaddr_in *dst;
472 struct pktgen_buf *pb;
473 struct ip *ip;
474 struct udpiphdr *ui;
475 struct ether_header *eh;
476
c6830078 477 pktgen_ref(pktg);
391741ec
SZ
478
479 pb = kmalloc(sizeof(*pb), M_PKTGEN, M_WAITOK | M_ZERO);
480 pb->pb_ifp = ifp;
2fb36fab 481 pb->pb_ifsq = ifsq;
391741ec
SZ
482 pb->pb_inuse = 1;
483 pb->pb_buf = kmalloc(PKTGEN_BUFSZ, M_PKTGEN, M_WAITOK);
484 pb->pb_len = len;
485 pb->pb_cpuid = cpuid;
486 pb->pb_pktg = pktg;
487 netmsg_init(&pb->pb_nmsg, NULL, &netisr_adone_rport, 0,
488 pktgen_buf_send);
1bbaa2ad 489 LIST_INSERT_HEAD(&p->pktg_buflist, pb, pb_link);
391741ec
SZ
490
491 dst = &pktg->pktg_dst[i % pktg->pktg_ndst];
492 ++i;
493
494 m->m_ext.ext_arg = pb;
495 m->m_ext.ext_buf = pb->pb_buf;
496 m->m_ext.ext_free = pktgen_buf_free;
497 m->m_ext.ext_ref = pktgen_buf_ref;
498 m->m_ext.ext_size = PKTGEN_BUFSZ;
499
500 m->m_data = m->m_ext.ext_buf;
501 m->m_flags |= M_EXT;
4fa8e46e
SZ
502 m->m_len = m->m_pkthdr.len = len;
503
391741ec
SZ
504 m->m_data += ETHER_HDR_LEN;
505 m->m_len -= ETHER_HDR_LEN;
506 m->m_pkthdr.len -= ETHER_HDR_LEN;
4fa8e46e
SZ
507
508 ui = mtod(m, struct udpiphdr *);
509 ui->ui_pr = IPPROTO_UDP;
eec66bd4
SZ
510 if (pktg->pktg_flags & PKTG_F_SWITCH_SRCDST) {
511 ui->ui_src.s_addr = dst->sin_addr.s_addr;
512 ui->ui_dst.s_addr = pktg->pktg_src.sin_addr.s_addr;
513 ui->ui_sport = dst->sin_port;
514 ui->ui_dport = pktg->pktg_src.sin_port;
515 } else {
516 ui->ui_src.s_addr = pktg->pktg_src.sin_addr.s_addr;
517 ui->ui_dst.s_addr = dst->sin_addr.s_addr;
518 ui->ui_sport = pktg->pktg_src.sin_port;
519 ui->ui_dport = dst->sin_port;
520 }
4fa8e46e 521 ui->ui_ulen = ulen;
391741ec
SZ
522 ui->ui_sum = in_pseudo(ui->ui_src.s_addr, ui->ui_dst.s_addr,
523 psum);
64c94a5f 524 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
4fa8e46e
SZ
525
526 ip = (struct ip *)ui;
527 ip->ip_len = ip_len;
528 ip->ip_ttl = 64; /* XXX */
529 ip->ip_tos = 0; /* XXX */
530 ip->ip_vhl = IP_VHL_BORING;
531 ip->ip_off = 0;
2640b9e5 532 ip->ip_sum = 0;
4fa8e46e
SZ
533 ip->ip_id = ip_newid();
534
391741ec 535 in_delayed_cksum(m);
4fa8e46e
SZ
536
537 ip->ip_len = htons(ip->ip_len);
391741ec
SZ
538 ip->ip_sum = in_cksum_hdr(ip);
539
540 m->m_data -= ETHER_HDR_LEN;
541 m->m_len += ETHER_HDR_LEN;
542 m->m_pkthdr.len += ETHER_HDR_LEN;
c95ebcd6 543
c95ebcd6
SZ
544 eh = mtod(m, struct ether_header *);
545 bcopy(pktg->pktg_dst_lladdr, eh->ether_dhost, ETHER_ADDR_LEN);
546 bcopy(IF_LLADDR(ifp), eh->ether_shost, ETHER_ADDR_LEN);
547 eh->ether_type = htons(ETHERTYPE_IP);
548
391741ec
SZ
549 nextm = m->m_nextpkt;
550 m->m_nextpkt = NULL;
c95ebcd6 551
391741ec 552 ifq_dispatch(ifp, m, NULL);
c95ebcd6 553
391741ec
SZ
554 m = nextm;
555 }
c95ebcd6 556
1bbaa2ad
SZ
557 callout_reset(&p->pktg_stop, pktg->pktg_duration * hz,
558 pktgen_pcpu_stop_cb, p);
c95ebcd6 559
b0cd2b4b 560 pktgen_free(pktg);
391741ec 561}
c95ebcd6 562
391741ec
SZ
563static void
564pktgen_mbuf(struct pktgen_buf *pb, struct mbuf *m)
565{
566 m->m_ext.ext_arg = pb;
567 m->m_ext.ext_buf = pb->pb_buf;
568 m->m_ext.ext_free = pktgen_buf_free;
569 m->m_ext.ext_ref = pktgen_buf_ref;
570 m->m_ext.ext_size = PKTGEN_BUFSZ;
571
572 m->m_data = m->m_ext.ext_buf;
573 m->m_flags |= M_EXT;
574 m->m_len = m->m_pkthdr.len = pb->pb_len;
575}
c95ebcd6 576
391741ec
SZ
577static void
578pktgen_buf_send(netmsg_t msg)
579{
580 struct pktgen_buf *pb = (struct pktgen_buf *)msg;
581 struct mbuf *m;
c95ebcd6 582
ec7f7fc8 583 KKASSERT(&curthread->td_msgport == netisr_cpuport(pb->pb_cpuid));
c95ebcd6 584
391741ec
SZ
585 crit_enter();
586 lwkt_replymsg(&pb->pb_nmsg.lmsg, 0);
587 crit_exit();
c95ebcd6 588
391741ec
SZ
589 MGETHDR(m, MB_WAIT, MT_DATA);
590 pktgen_mbuf(pb, m);
591 ifq_dispatch(pb->pb_ifp, m, NULL);
592}
c95ebcd6 593
391741ec
SZ
594static void
595pktgen_buf_free(void *arg)
596{
597 struct pktgen_buf *pb = arg;
598 struct mbuf *m;
c95ebcd6 599
391741ec
SZ
600 KKASSERT(pb->pb_inuse > 0);
601 if (pb->pb_done) {
602 if (atomic_fetchadd_int(&pb->pb_inuse, -1) == 1) {
603 struct pktgen *pktg;
c95ebcd6 604
391741ec 605 pktg = pb->pb_pktg;
4c9906cb 606 crit_enter();
391741ec 607 LIST_REMOVE(pb, pb_link);
4c9906cb 608 crit_exit();
391741ec
SZ
609 kfree(pb->pb_buf, M_PKTGEN);
610 kfree(pb, M_PKTGEN);
4fa8e46e 611
391741ec 612 pktgen_free(pktg);
4fa8e46e 613 }
391741ec 614 return;
4fa8e46e 615 }
db161a19 616
ec7f7fc8 617 if (&curthread->td_msgport != netisr_cpuport(pb->pb_cpuid)) {
391741ec
SZ
618 KKASSERT(pb->pb_cpuid == mycpuid);
619 crit_enter();
620 KKASSERT(pb->pb_nmsg.lmsg.ms_flags & MSGF_DONE);
ec7f7fc8 621 lwkt_sendmsg(netisr_cpuport(pb->pb_cpuid), &pb->pb_nmsg.lmsg);
391741ec
SZ
622 crit_exit();
623 return;
624 }
625
626 MGETHDR(m, MB_WAIT, MT_DATA);
627 pktgen_mbuf(pb, m);
2fb36fab 628 ifsq_enqueue(pb->pb_ifsq, m, NULL);
db161a19
SZ
629}
630
631static void
391741ec 632pktgen_buf_ref(void *arg)
db161a19 633{
391741ec 634 struct pktgen_buf *pb = arg;
db161a19 635
391741ec 636 panic("%s should never be called\n", __func__);
4fa8e46e 637
391741ec
SZ
638 KKASSERT(pb->pb_inuse > 0);
639 atomic_add_int(&pb->pb_inuse, 1);
640}
4fa8e46e 641
391741ec
SZ
642static void
643pktgen_free(struct pktgen *pktg)
644{
4fa8e46e 645 KKASSERT(pktg->pktg_refcnt > 0);
4c9906cb 646 if (atomic_fetchadd_int(&pktg->pktg_refcnt, -1) == 1) {
1bbaa2ad
SZ
647 int i;
648
391741ec
SZ
649 if (pktg->pktg_dst != NULL)
650 kfree(pktg->pktg_dst, M_PKTGEN);
1bbaa2ad
SZ
651
652 for (i = 0; i < ncpus; ++i)
653 KKASSERT(LIST_EMPTY(&pktg->pktg_pcpu[i].pktg_buflist));
391741ec
SZ
654 kfree(pktg, M_PKTGEN);
655 }
4fa8e46e
SZ
656
657 KKASSERT(pktgen_refcnt > 0);
4c9906cb 658 atomic_subtract_int(&pktgen_refcnt, 1);
391741ec
SZ
659}
660
661static void
1bbaa2ad 662pktgen_pcpu_stop_cb(void *arg)
391741ec 663{
1bbaa2ad 664 struct pktgen_pcpu *p = arg;
391741ec 665 struct pktgen_buf *pb;
4fa8e46e 666
4c9906cb 667 crit_enter();
1bbaa2ad 668 LIST_FOREACH(pb, &p->pktg_buflist, pb_link)
391741ec 669 pb->pb_done = 1;
4c9906cb 670 crit_exit();
4fa8e46e 671}
c6830078
SZ
672
673static void
674pktgen_ref(struct pktgen *pktg)
675{
676 atomic_add_int(&pktg->pktg_refcnt, 1);
677 atomic_add_int(&pktgen_refcnt, 1);
678}