igmp: IGMP processing mpsafe
[dragonfly.git] / tools / tools / netrate / pktgen / pktgen.c
CommitLineData
4fa8e46e 1/*
391741ec 2 * Copyright (c) 2012 The DragonFly Project. All rights reserved.
4fa8e46e
SZ
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Sepherosa Ziehau <sepherosa@gmail.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
db161a19 34 * $DragonFly: src/tools/tools/netrate/pktgen/pktgen.c,v 1.4 2008/04/02 14:18:55 sephe Exp $
4fa8e46e
SZ
35 */
36
37#define _IP_VHL
38
39#include <sys/param.h>
40#include <sys/conf.h>
41#include <sys/device.h>
4fa8e46e
SZ
42#include <sys/in_cksum.h>
43#include <sys/kernel.h>
44#include <sys/malloc.h>
45#include <sys/mbuf.h>
46#include <sys/proc.h>
9eaf083f 47#include <sys/priv.h>
391741ec 48#include <sys/queue.h>
4fa8e46e
SZ
49#include <sys/socket.h>
50#include <sys/systm.h>
51#include <sys/serialize.h>
52
53#include <net/if.h>
54#include <net/if_dl.h>
55#include <net/if_var.h>
56#include <net/ifq_var.h>
57#include <net/ethernet.h>
391741ec 58#include <net/netmsg2.h>
5337421c 59#include <net/netisr2.h>
4fa8e46e
SZ
60
61#include <netinet/in.h>
9064712f 62#include <netinet/in_pcb.h>
4fa8e46e
SZ
63#include <netinet/ip.h>
64#include <netinet/udp_var.h>
65
66#include "pktgen.h"
67
68#define CDEV_NAME "pktg"
4fa8e46e 69
391741ec
SZ
70#define PKTGEN_BUFSZ 2048
71
390d8a05
SZ
72#ifndef PKTGEN_DEVCNT
73#define PKTGEN_DEVCNT 4
74#endif
75
391741ec
SZ
76struct pktgen;
77
b0cd2b4b
SZ
78struct netmsg_pktgen {
79 struct netmsg_base np_base;
80 struct pktgen *np_pktg;
81 struct ifaltq_subque *np_ifsq;
82};
83
391741ec
SZ
84struct pktgen_buf {
85 struct netmsg_base pb_nmsg; /* MUST BE THE FIRST */
86 void *pb_buf;
87 volatile int pb_done;
4c9906cb 88 int pb_inuse;
391741ec 89 struct ifnet *pb_ifp;
2fb36fab 90 struct ifaltq_subque *pb_ifsq;
391741ec
SZ
91 int pb_len;
92 int pb_cpuid;
93 struct pktgen *pb_pktg;
94 LIST_ENTRY(pktgen_buf) pb_link;
95};
96
1bbaa2ad
SZ
97struct pktgen_pcpu {
98 struct callout pktg_stop;
99 LIST_HEAD(, pktgen_buf) pktg_buflist;
100};
101
4fa8e46e
SZ
102struct pktgen {
103 uint32_t pktg_flags; /* PKTG_F_ */
104 int pktg_refcnt;
105
4fa8e46e 106 int pktg_duration;
4fa8e46e
SZ
107
108 int pktg_datalen;
4fa8e46e 109 struct ifnet *pktg_ifp;
c95ebcd6 110
793f802f
SZ
111 int pktg_pktenq;
112
391741ec
SZ
113 struct sockaddr_in pktg_src;
114 int pktg_ndst;
115 struct sockaddr_in *pktg_dst;
4fa8e46e 116 uint8_t pktg_dst_lladdr[ETHER_ADDR_LEN];
391741ec 117
1bbaa2ad 118 struct pktgen_pcpu pktg_pcpu[MAXCPU];
4fa8e46e
SZ
119};
120
eec66bd4
SZ
121#define PKTG_F_CONFIG 0x1
122#define PKTG_F_RUNNING 0x4
123#define PKTG_F_SWITCH_SRCDST 0x8
4fa8e46e
SZ
124
125static int pktgen_modevent(module_t, int, void *);
391741ec
SZ
126
127static void pktgen_buf_free(void *);
128static void pktgen_buf_ref(void *);
129static void pktgen_buf_send(netmsg_t);
130
4fa8e46e 131static int pktgen_config(struct pktgen *,
391741ec 132 const struct pktgen_conf *);
b0cd2b4b 133static int pktgen_start(struct pktgen *, int);
391741ec 134static void pktgen_free(struct pktgen *);
c6830078 135static void pktgen_ref(struct pktgen *);
1bbaa2ad 136static void pktgen_pcpu_stop_cb(void *);
391741ec 137static void pktgen_mbuf(struct pktgen_buf *, struct mbuf *);
b0cd2b4b
SZ
138static void pktgen_start_ifsq(struct pktgen *,
139 struct ifaltq_subque *);
140static void pktgen_start_ifsq_handler(netmsg_t);
4fa8e46e
SZ
141
142static d_open_t pktgen_open;
143static d_close_t pktgen_close;
144static d_ioctl_t pktgen_ioctl;
145
146static struct dev_ops pktgen_ops = {
624409c2 147 { CDEV_NAME, 0, D_MPSAFE },
4fa8e46e
SZ
148 .d_open = pktgen_open,
149 .d_close = pktgen_close,
150 .d_ioctl = pktgen_ioctl,
151};
152
4c9906cb 153static volatile int pktgen_refcnt;
e3d0d8d7 154static struct lwkt_token pktgen_tok = LWKT_TOKEN_INITIALIZER(pktgen_token);
4fa8e46e
SZ
155
156MALLOC_DECLARE(M_PKTGEN);
157MALLOC_DEFINE(M_PKTGEN, CDEV_NAME, "Packet generator");
158
159DEV_MODULE(pktgen, pktgen_modevent, NULL);
160
161static int
162pktgen_modevent(module_t mod, int type, void *data)
163{
390d8a05 164 int error = 0, i;
4fa8e46e
SZ
165
166 switch (type) {
167 case MOD_LOAD:
390d8a05
SZ
168 for (i = 0; i < PKTGEN_DEVCNT; ++i) {
169 make_dev(&pktgen_ops, 0, UID_ROOT, GID_WHEEL, 0600,
170 CDEV_NAME"%d", i);
171 }
4fa8e46e
SZ
172 break;
173
174 case MOD_UNLOAD:
175 if (pktgen_refcnt > 0)
176 return EBUSY;
4b3b3198 177 dev_ops_remove_all(&pktgen_ops);
4fa8e46e
SZ
178 break;
179
180 default:
181 error = EOPNOTSUPP;
182 break;
183 }
184 return error;
185}
186
187static int
188pktgen_open(struct dev_open_args *ap)
189{
190 cdev_t dev = ap->a_head.a_dev;
191 struct pktgen *pktg;
1bbaa2ad 192 int error, i;
4fa8e46e 193
9eaf083f 194 error = priv_check_cred(ap->a_cred, PRIV_ROOT, 0);
4fa8e46e
SZ
195 if (error)
196 return error;
197
e3d0d8d7 198 lwkt_gettoken(&pktgen_tok);
4fa8e46e
SZ
199
200 if (dev->si_drv1 != NULL) {
e3d0d8d7 201 lwkt_reltoken(&pktgen_tok);
4fa8e46e
SZ
202 return EBUSY;
203 }
204
205 pktg = kmalloc(sizeof(*pktg), M_PKTGEN, M_ZERO | M_WAITOK);
1bbaa2ad
SZ
206 for (i = 0; i < ncpus; ++i) {
207 struct pktgen_pcpu *p = &pktg->pktg_pcpu[i];
208
209 callout_init_mp(&p->pktg_stop);
210 LIST_INIT(&p->pktg_buflist);
211 }
4fa8e46e 212
4fa8e46e
SZ
213 dev->si_drv1 = pktg;
214 pktg->pktg_refcnt = 1;
215
4c9906cb 216 atomic_add_int(&pktgen_refcnt, 1);
4fa8e46e 217
e3d0d8d7 218 lwkt_reltoken(&pktgen_tok);
4fa8e46e
SZ
219 return 0;
220}
221
222static int
223pktgen_close(struct dev_close_args *ap)
224{
225 cdev_t dev = ap->a_head.a_dev;
226 struct pktgen *pktg = dev->si_drv1;
227
e3d0d8d7 228 lwkt_gettoken(&pktgen_tok);
4fa8e46e 229 dev->si_drv1 = NULL;
e3d0d8d7 230 lwkt_reltoken(&pktgen_tok);
391741ec 231
4c9906cb
SZ
232 pktgen_free(pktg);
233
4fa8e46e
SZ
234 return 0;
235}
236
237static int
238pktgen_ioctl(struct dev_ioctl_args *ap __unused)
239{
240 cdev_t dev = ap->a_head.a_dev;
241 caddr_t data = ap->a_data;
242 struct pktgen *pktg = dev->si_drv1;
243 int error;
244
e3d0d8d7 245 lwkt_gettoken(&pktgen_tok);
4fa8e46e
SZ
246
247 switch (ap->a_cmd) {
248 case PKTGENSTART:
b0cd2b4b
SZ
249 error = pktgen_start(pktg, 0);
250 break;
251
252 case PKTGENMQSTART:
253 error = pktgen_start(pktg, 1);
4fa8e46e
SZ
254 break;
255
256 case PKTGENSCONF:
257 error = pktgen_config(pktg, (const struct pktgen_conf *)data);
258 break;
259
260 default:
261 error = EOPNOTSUPP;
262 break;
263 }
264
e3d0d8d7 265 lwkt_reltoken(&pktgen_tok);
4fa8e46e
SZ
266 return error;
267}
268
269static int
270pktgen_config(struct pktgen *pktg, const struct pktgen_conf *conf)
271{
272 const struct sockaddr_in *sin;
391741ec 273 struct sockaddr_in *dst = NULL;
4fa8e46e
SZ
274 const struct sockaddr *sa;
275 struct ifnet *ifp;
391741ec 276 size_t dst_size;
793f802f 277 int i, error, pktenq;
4fa8e46e 278
391741ec 279 if (pktg->pktg_flags & (PKTG_F_RUNNING | PKTG_F_CONFIG))
4fa8e46e
SZ
280 return EBUSY;
281
391741ec
SZ
282 if (conf->pc_datalen <= 0 ||
283 conf->pc_datalen > ETHERMTU - sizeof(struct udpiphdr))
4fa8e46e
SZ
284 return EINVAL;
285 if (conf->pc_duration <= 0)
286 return EINVAL;
287
4fa8e46e
SZ
288 sin = &conf->pc_src;
289 if (sin->sin_family != AF_INET)
290 return EPROTONOSUPPORT;
291 if (sin->sin_port == 0)
292 return EINVAL;
293
391741ec 294 if (conf->pc_ndst <= 0)
4fa8e46e 295 return EINVAL;
391741ec
SZ
296 dst_size = conf->pc_ndst * sizeof(struct sockaddr_in);
297
298 dst = kmalloc(dst_size, M_PKTGEN, M_WAITOK | M_NULLOK);
299 if (dst == NULL)
300 return ENOMEM;
4fa8e46e 301
391741ec
SZ
302 error = copyin(conf->pc_dst, dst, dst_size);
303 if (error)
304 goto failed;
305
306 for (i = 0; i < conf->pc_ndst; ++i) {
307 sin = &dst[i];
308 if (sin->sin_family != AF_INET) {
309 error = EPROTONOSUPPORT;
310 goto failed;
311 }
312 if (sin->sin_port == 0) {
313 error = EINVAL;
314 goto failed;
315 }
316 }
317
318 ifp = ifunit(conf->pc_ifname);
319 if (ifp == NULL) {
320 error = ENXIO;
321 goto failed;
322 }
323
793f802f 324 pktenq = conf->pc_pktenq;
2fb36fab 325 if (pktenq < 0 || pktenq > ifp->if_snd.altq_maxlen) {
793f802f
SZ
326 error = ENOBUFS;
327 goto failed;
328 } else if (pktenq == 0) {
2fb36fab 329 pktenq = (ifp->if_snd.altq_maxlen * 3) / 4;
793f802f
SZ
330 }
331
391741ec
SZ
332 sa = &conf->pc_dst_lladdr;
333 if (sa->sa_family != AF_LINK) {
334 error = EPROTONOSUPPORT;
335 goto failed;
336 }
337 if (sa->sa_len != ETHER_ADDR_LEN) {
338 error = EPROTONOSUPPORT;
339 goto failed;
340 }
341 if (ETHER_IS_MULTICAST(sa->sa_data) ||
342 bcmp(sa->sa_data, ifp->if_broadcastaddr, ifp->if_addrlen) == 0) {
343 error = EADDRNOTAVAIL;
344 goto failed;
345 }
346
347 /*
348 * Accept the config
349 */
4fa8e46e 350 pktg->pktg_flags |= PKTG_F_CONFIG;
4fa8e46e 351
eec66bd4
SZ
352 if (conf->pc_flags & PKTGEN_FLAG_SWITCH_SRCDST)
353 pktg->pktg_flags |= PKTG_F_SWITCH_SRCDST;
4fa8e46e 354 pktg->pktg_duration = conf->pc_duration;
4fa8e46e 355 pktg->pktg_datalen = conf->pc_datalen;
793f802f 356 pktg->pktg_pktenq = pktenq;
391741ec
SZ
357 pktg->pktg_ifp = ifp;
358 pktg->pktg_src = conf->pc_src;
359 pktg->pktg_ndst = conf->pc_ndst;
360 KKASSERT(pktg->pktg_dst == NULL);
361 pktg->pktg_dst = dst;
4fa8e46e 362 bcopy(sa->sa_data, pktg->pktg_dst_lladdr, ETHER_ADDR_LEN);
c95ebcd6 363
4fa8e46e 364 return 0;
391741ec
SZ
365
366failed:
367 if (dst != NULL)
368 kfree(dst, M_PKTGEN);
369 return error;
4fa8e46e
SZ
370}
371
b0cd2b4b
SZ
372static void
373pktgen_start_ifsq(struct pktgen *pktg, struct ifaltq_subque *ifsq)
374{
375 struct netmsg_pktgen *np;
376
377 np = kmalloc(sizeof(*np), M_LWKTMSG, M_WAITOK);
378 netmsg_init(&np->np_base, NULL, &netisr_afree_rport, 0,
379 pktgen_start_ifsq_handler);
380 np->np_pktg = pktg;
381 np->np_ifsq = ifsq;
382
ec7f7fc8 383 lwkt_sendmsg(netisr_cpuport(ifsq_get_cpuid(ifsq)), &np->np_base.lmsg);
b0cd2b4b
SZ
384}
385
4fa8e46e 386static int
b0cd2b4b 387pktgen_start(struct pktgen *pktg, int mq)
4fa8e46e 388{
b0cd2b4b
SZ
389 struct ifaltq *ifq;
390
391 if ((pktg->pktg_flags & PKTG_F_CONFIG) == 0)
392 return EINVAL;
393 if (pktg->pktg_flags & PKTG_F_RUNNING)
394 return EBUSY;
395 pktg->pktg_flags |= PKTG_F_RUNNING;
396
397 ifq = &pktg->pktg_ifp->if_snd;
398 if (!mq) {
399 pktgen_ref(pktg);
400 pktgen_start_ifsq(pktg, ifq_get_subq_default(ifq));
401 } else {
402 int i;
403
404 for (i = 0; i < ifq->altq_subq_cnt; ++i)
405 pktgen_ref(pktg);
406 for (i = 0; i < ifq->altq_subq_cnt; ++i)
407 pktgen_start_ifsq(pktg, ifq_get_subq(ifq, i));
408 }
409 return 0;
410}
411
412static void
413pktgen_start_ifsq_handler(netmsg_t nmsg)
414{
415 struct netmsg_pktgen *np = (struct netmsg_pktgen *)nmsg;
416 struct pktgen *pktg = np->np_pktg;
417 struct ifaltq_subque *ifsq = np->np_ifsq;
418
391741ec
SZ
419 struct mbuf *m, *head = NULL, **next;
420 struct ifnet *ifp;
1bbaa2ad 421 struct pktgen_pcpu *p;
b0cd2b4b 422 int cpuid, i, alloc_cnt, keep_cnt;
391741ec
SZ
423
424 u_short ulen, psum;
425 int len, ip_len;
426
b0cd2b4b
SZ
427 /* Reply ASAP */
428 lwkt_replymsg(&np->np_base.lmsg, 0);
4fa8e46e 429
391741ec 430 ifp = pktg->pktg_ifp;
4fa8e46e 431
2fb36fab 432 cpuid = ifsq_get_cpuid(ifsq);
b0cd2b4b 433 KKASSERT(cpuid == mycpuid);
4fa8e46e 434
1bbaa2ad
SZ
435 p = &pktg->pktg_pcpu[cpuid];
436
793f802f
SZ
437 keep_cnt = pktg->pktg_pktenq;
438 alloc_cnt = keep_cnt * 2;
4fa8e46e 439
391741ec
SZ
440 /*
441 * Prefault enough mbuf into mbuf objcache
442 */
443 next = &head;
444 for (i = 0; i < alloc_cnt; ++i) {
445 MGETHDR(m, MB_WAIT, MT_DATA);
446 *next = m;
447 next = &m->m_nextpkt;
448 }
4fa8e46e 449
391741ec
SZ
450 for (i = 0; i < alloc_cnt - keep_cnt; ++i) {
451 m = head;
452 head = m->m_nextpkt;
453 m->m_nextpkt = NULL;
454 m_freem(m);
455 }
456 KKASSERT(head != NULL);
4fa8e46e 457
391741ec
SZ
458 /*
459 * Setup the packets' data
460 */
461 ip_len = pktg->pktg_datalen + sizeof(struct udpiphdr);
4fa8e46e
SZ
462 len = ip_len + ETHER_HDR_LEN;
463
391741ec
SZ
464 psum = htons((u_short)pktg->pktg_datalen + sizeof(struct udphdr) +
465 IPPROTO_UDP);
4fa8e46e
SZ
466 ulen = htons(pktg->pktg_datalen + sizeof(struct udphdr));
467
391741ec
SZ
468 m = head;
469 i = 0;
470 while (m != NULL) {
471 struct mbuf *nextm;
472 const struct sockaddr_in *dst;
473 struct pktgen_buf *pb;
474 struct ip *ip;
475 struct udpiphdr *ui;
476 struct ether_header *eh;
477
c6830078 478 pktgen_ref(pktg);
391741ec
SZ
479
480 pb = kmalloc(sizeof(*pb), M_PKTGEN, M_WAITOK | M_ZERO);
481 pb->pb_ifp = ifp;
2fb36fab 482 pb->pb_ifsq = ifsq;
391741ec
SZ
483 pb->pb_inuse = 1;
484 pb->pb_buf = kmalloc(PKTGEN_BUFSZ, M_PKTGEN, M_WAITOK);
485 pb->pb_len = len;
486 pb->pb_cpuid = cpuid;
487 pb->pb_pktg = pktg;
488 netmsg_init(&pb->pb_nmsg, NULL, &netisr_adone_rport, 0,
489 pktgen_buf_send);
1bbaa2ad 490 LIST_INSERT_HEAD(&p->pktg_buflist, pb, pb_link);
391741ec
SZ
491
492 dst = &pktg->pktg_dst[i % pktg->pktg_ndst];
493 ++i;
494
495 m->m_ext.ext_arg = pb;
496 m->m_ext.ext_buf = pb->pb_buf;
497 m->m_ext.ext_free = pktgen_buf_free;
498 m->m_ext.ext_ref = pktgen_buf_ref;
499 m->m_ext.ext_size = PKTGEN_BUFSZ;
500
501 m->m_data = m->m_ext.ext_buf;
502 m->m_flags |= M_EXT;
4fa8e46e
SZ
503 m->m_len = m->m_pkthdr.len = len;
504
391741ec
SZ
505 m->m_data += ETHER_HDR_LEN;
506 m->m_len -= ETHER_HDR_LEN;
507 m->m_pkthdr.len -= ETHER_HDR_LEN;
4fa8e46e
SZ
508
509 ui = mtod(m, struct udpiphdr *);
510 ui->ui_pr = IPPROTO_UDP;
eec66bd4
SZ
511 if (pktg->pktg_flags & PKTG_F_SWITCH_SRCDST) {
512 ui->ui_src.s_addr = dst->sin_addr.s_addr;
513 ui->ui_dst.s_addr = pktg->pktg_src.sin_addr.s_addr;
514 ui->ui_sport = dst->sin_port;
515 ui->ui_dport = pktg->pktg_src.sin_port;
516 } else {
517 ui->ui_src.s_addr = pktg->pktg_src.sin_addr.s_addr;
518 ui->ui_dst.s_addr = dst->sin_addr.s_addr;
519 ui->ui_sport = pktg->pktg_src.sin_port;
520 ui->ui_dport = dst->sin_port;
521 }
4fa8e46e 522 ui->ui_ulen = ulen;
391741ec
SZ
523 ui->ui_sum = in_pseudo(ui->ui_src.s_addr, ui->ui_dst.s_addr,
524 psum);
64c94a5f 525 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
4fa8e46e
SZ
526
527 ip = (struct ip *)ui;
528 ip->ip_len = ip_len;
529 ip->ip_ttl = 64; /* XXX */
530 ip->ip_tos = 0; /* XXX */
531 ip->ip_vhl = IP_VHL_BORING;
532 ip->ip_off = 0;
2640b9e5 533 ip->ip_sum = 0;
4fa8e46e
SZ
534 ip->ip_id = ip_newid();
535
391741ec 536 in_delayed_cksum(m);
4fa8e46e
SZ
537
538 ip->ip_len = htons(ip->ip_len);
391741ec
SZ
539 ip->ip_sum = in_cksum_hdr(ip);
540
541 m->m_data -= ETHER_HDR_LEN;
542 m->m_len += ETHER_HDR_LEN;
543 m->m_pkthdr.len += ETHER_HDR_LEN;
c95ebcd6 544
c95ebcd6
SZ
545 eh = mtod(m, struct ether_header *);
546 bcopy(pktg->pktg_dst_lladdr, eh->ether_dhost, ETHER_ADDR_LEN);
547 bcopy(IF_LLADDR(ifp), eh->ether_shost, ETHER_ADDR_LEN);
548 eh->ether_type = htons(ETHERTYPE_IP);
549
391741ec
SZ
550 nextm = m->m_nextpkt;
551 m->m_nextpkt = NULL;
c95ebcd6 552
391741ec 553 ifq_dispatch(ifp, m, NULL);
c95ebcd6 554
391741ec
SZ
555 m = nextm;
556 }
c95ebcd6 557
1bbaa2ad
SZ
558 callout_reset(&p->pktg_stop, pktg->pktg_duration * hz,
559 pktgen_pcpu_stop_cb, p);
c95ebcd6 560
b0cd2b4b 561 pktgen_free(pktg);
391741ec 562}
c95ebcd6 563
391741ec
SZ
564static void
565pktgen_mbuf(struct pktgen_buf *pb, struct mbuf *m)
566{
567 m->m_ext.ext_arg = pb;
568 m->m_ext.ext_buf = pb->pb_buf;
569 m->m_ext.ext_free = pktgen_buf_free;
570 m->m_ext.ext_ref = pktgen_buf_ref;
571 m->m_ext.ext_size = PKTGEN_BUFSZ;
572
573 m->m_data = m->m_ext.ext_buf;
574 m->m_flags |= M_EXT;
575 m->m_len = m->m_pkthdr.len = pb->pb_len;
576}
c95ebcd6 577
391741ec
SZ
578static void
579pktgen_buf_send(netmsg_t msg)
580{
581 struct pktgen_buf *pb = (struct pktgen_buf *)msg;
582 struct mbuf *m;
c95ebcd6 583
ec7f7fc8 584 KKASSERT(&curthread->td_msgport == netisr_cpuport(pb->pb_cpuid));
c95ebcd6 585
391741ec
SZ
586 crit_enter();
587 lwkt_replymsg(&pb->pb_nmsg.lmsg, 0);
588 crit_exit();
c95ebcd6 589
391741ec
SZ
590 MGETHDR(m, MB_WAIT, MT_DATA);
591 pktgen_mbuf(pb, m);
592 ifq_dispatch(pb->pb_ifp, m, NULL);
593}
c95ebcd6 594
391741ec
SZ
595static void
596pktgen_buf_free(void *arg)
597{
598 struct pktgen_buf *pb = arg;
599 struct mbuf *m;
c95ebcd6 600
391741ec
SZ
601 KKASSERT(pb->pb_inuse > 0);
602 if (pb->pb_done) {
603 if (atomic_fetchadd_int(&pb->pb_inuse, -1) == 1) {
604 struct pktgen *pktg;
c95ebcd6 605
391741ec 606 pktg = pb->pb_pktg;
4c9906cb 607 crit_enter();
391741ec 608 LIST_REMOVE(pb, pb_link);
4c9906cb 609 crit_exit();
391741ec
SZ
610 kfree(pb->pb_buf, M_PKTGEN);
611 kfree(pb, M_PKTGEN);
4fa8e46e 612
391741ec 613 pktgen_free(pktg);
4fa8e46e 614 }
391741ec 615 return;
4fa8e46e 616 }
db161a19 617
ec7f7fc8 618 if (&curthread->td_msgport != netisr_cpuport(pb->pb_cpuid)) {
391741ec
SZ
619 KKASSERT(pb->pb_cpuid == mycpuid);
620 crit_enter();
621 KKASSERT(pb->pb_nmsg.lmsg.ms_flags & MSGF_DONE);
ec7f7fc8 622 lwkt_sendmsg(netisr_cpuport(pb->pb_cpuid), &pb->pb_nmsg.lmsg);
391741ec
SZ
623 crit_exit();
624 return;
625 }
626
627 MGETHDR(m, MB_WAIT, MT_DATA);
628 pktgen_mbuf(pb, m);
2fb36fab 629 ifsq_enqueue(pb->pb_ifsq, m, NULL);
db161a19
SZ
630}
631
632static void
391741ec 633pktgen_buf_ref(void *arg)
db161a19 634{
391741ec 635 struct pktgen_buf *pb = arg;
db161a19 636
391741ec 637 panic("%s should never be called\n", __func__);
4fa8e46e 638
391741ec
SZ
639 KKASSERT(pb->pb_inuse > 0);
640 atomic_add_int(&pb->pb_inuse, 1);
641}
4fa8e46e 642
391741ec
SZ
643static void
644pktgen_free(struct pktgen *pktg)
645{
4fa8e46e 646 KKASSERT(pktg->pktg_refcnt > 0);
4c9906cb 647 if (atomic_fetchadd_int(&pktg->pktg_refcnt, -1) == 1) {
1bbaa2ad
SZ
648 int i;
649
391741ec
SZ
650 if (pktg->pktg_dst != NULL)
651 kfree(pktg->pktg_dst, M_PKTGEN);
1bbaa2ad
SZ
652
653 for (i = 0; i < ncpus; ++i)
654 KKASSERT(LIST_EMPTY(&pktg->pktg_pcpu[i].pktg_buflist));
391741ec
SZ
655 kfree(pktg, M_PKTGEN);
656 }
4fa8e46e
SZ
657
658 KKASSERT(pktgen_refcnt > 0);
4c9906cb 659 atomic_subtract_int(&pktgen_refcnt, 1);
391741ec
SZ
660}
661
662static void
1bbaa2ad 663pktgen_pcpu_stop_cb(void *arg)
391741ec 664{
1bbaa2ad 665 struct pktgen_pcpu *p = arg;
391741ec 666 struct pktgen_buf *pb;
4fa8e46e 667
4c9906cb 668 crit_enter();
1bbaa2ad 669 LIST_FOREACH(pb, &p->pktg_buflist, pb_link)
391741ec 670 pb->pb_done = 1;
4c9906cb 671 crit_exit();
4fa8e46e 672}
c6830078
SZ
673
674static void
675pktgen_ref(struct pktgen *pktg)
676{
677 atomic_add_int(&pktg->pktg_refcnt, 1);
678 atomic_add_int(&pktgen_refcnt, 1);
679}