Commit | Line | Data |
---|---|---|
4fa8e46e | 1 | /* |
391741ec | 2 | * Copyright (c) 2012 The DragonFly Project. All rights reserved. |
4fa8e46e SZ |
3 | * |
4 | * This code is derived from software contributed to The DragonFly Project | |
5 | * by Sepherosa Ziehau <sepherosa@gmail.com> | |
6 | * | |
7 | * Redistribution and use in source and binary forms, with or without | |
8 | * modification, are permitted provided that the following conditions | |
9 | * are met: | |
10 | * | |
11 | * 1. Redistributions of source code must retain the above copyright | |
12 | * notice, this list of conditions and the following disclaimer. | |
13 | * 2. Redistributions in binary form must reproduce the above copyright | |
14 | * notice, this list of conditions and the following disclaimer in | |
15 | * the documentation and/or other materials provided with the | |
16 | * distribution. | |
17 | * 3. Neither the name of The DragonFly Project nor the names of its | |
18 | * contributors may be used to endorse or promote products derived | |
19 | * from this software without specific, prior written permission. | |
20 | * | |
21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
22 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS | |
24 | * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE | |
25 | * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, | |
26 | * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, | |
27 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; | |
28 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED | |
29 | * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, | |
30 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT | |
31 | * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |
32 | * SUCH DAMAGE. | |
4fa8e46e SZ |
33 | */ |
34 | ||
35 | #define _IP_VHL | |
36 | ||
37 | #include <sys/param.h> | |
38 | #include <sys/conf.h> | |
39 | #include <sys/device.h> | |
4fa8e46e SZ |
40 | #include <sys/in_cksum.h> |
41 | #include <sys/kernel.h> | |
42 | #include <sys/malloc.h> | |
43 | #include <sys/mbuf.h> | |
44 | #include <sys/proc.h> | |
2b3f93ea | 45 | #include <sys/caps.h> |
391741ec | 46 | #include <sys/queue.h> |
4fa8e46e SZ |
47 | #include <sys/socket.h> |
48 | #include <sys/systm.h> | |
49 | #include <sys/serialize.h> | |
50 | ||
51 | #include <net/if.h> | |
52 | #include <net/if_dl.h> | |
53 | #include <net/if_var.h> | |
54 | #include <net/ifq_var.h> | |
55 | #include <net/ethernet.h> | |
391741ec | 56 | #include <net/netmsg2.h> |
5337421c | 57 | #include <net/netisr2.h> |
4fa8e46e SZ |
58 | |
59 | #include <netinet/in.h> | |
9064712f | 60 | #include <netinet/in_pcb.h> |
4fa8e46e SZ |
61 | #include <netinet/ip.h> |
62 | #include <netinet/udp_var.h> | |
63 | ||
64 | #include "pktgen.h" | |
65 | ||
66 | #define CDEV_NAME "pktg" | |
4fa8e46e | 67 | |
391741ec SZ |
68 | #define PKTGEN_BUFSZ 2048 |
69 | ||
390d8a05 SZ |
70 | #ifndef PKTGEN_DEVCNT |
71 | #define PKTGEN_DEVCNT 4 | |
72 | #endif | |
73 | ||
391741ec SZ |
74 | struct pktgen; |
75 | ||
b0cd2b4b SZ |
76 | struct netmsg_pktgen { |
77 | struct netmsg_base np_base; | |
78 | struct pktgen *np_pktg; | |
79 | struct ifaltq_subque *np_ifsq; | |
80 | }; | |
81 | ||
391741ec SZ |
82 | struct pktgen_buf { |
83 | struct netmsg_base pb_nmsg; /* MUST BE THE FIRST */ | |
84 | void *pb_buf; | |
85 | volatile int pb_done; | |
4c9906cb | 86 | int pb_inuse; |
391741ec | 87 | struct ifnet *pb_ifp; |
2fb36fab | 88 | struct ifaltq_subque *pb_ifsq; |
391741ec SZ |
89 | int pb_len; |
90 | int pb_cpuid; | |
91 | struct pktgen *pb_pktg; | |
92 | LIST_ENTRY(pktgen_buf) pb_link; | |
93 | }; | |
94 | ||
1bbaa2ad SZ |
95 | struct pktgen_pcpu { |
96 | struct callout pktg_stop; | |
97 | LIST_HEAD(, pktgen_buf) pktg_buflist; | |
98 | }; | |
99 | ||
4fa8e46e SZ |
100 | struct pktgen { |
101 | uint32_t pktg_flags; /* PKTG_F_ */ | |
102 | int pktg_refcnt; | |
103 | ||
4fa8e46e | 104 | int pktg_duration; |
4fa8e46e SZ |
105 | |
106 | int pktg_datalen; | |
4fa8e46e | 107 | struct ifnet *pktg_ifp; |
c95ebcd6 | 108 | |
793f802f SZ |
109 | int pktg_pktenq; |
110 | ||
391741ec SZ |
111 | struct sockaddr_in pktg_src; |
112 | int pktg_ndst; | |
113 | struct sockaddr_in *pktg_dst; | |
4fa8e46e | 114 | uint8_t pktg_dst_lladdr[ETHER_ADDR_LEN]; |
391741ec | 115 | |
1bbaa2ad | 116 | struct pktgen_pcpu pktg_pcpu[MAXCPU]; |
4fa8e46e SZ |
117 | }; |
118 | ||
eec66bd4 SZ |
119 | #define PKTG_F_CONFIG 0x1 |
120 | #define PKTG_F_RUNNING 0x4 | |
121 | #define PKTG_F_SWITCH_SRCDST 0x8 | |
4fa8e46e SZ |
122 | |
123 | static int pktgen_modevent(module_t, int, void *); | |
391741ec SZ |
124 | |
125 | static void pktgen_buf_free(void *); | |
126 | static void pktgen_buf_ref(void *); | |
127 | static void pktgen_buf_send(netmsg_t); | |
128 | ||
4fa8e46e | 129 | static int pktgen_config(struct pktgen *, |
391741ec | 130 | const struct pktgen_conf *); |
b0cd2b4b | 131 | static int pktgen_start(struct pktgen *, int); |
391741ec | 132 | static void pktgen_free(struct pktgen *); |
c6830078 | 133 | static void pktgen_ref(struct pktgen *); |
1bbaa2ad | 134 | static void pktgen_pcpu_stop_cb(void *); |
391741ec | 135 | static void pktgen_mbuf(struct pktgen_buf *, struct mbuf *); |
b0cd2b4b SZ |
136 | static void pktgen_start_ifsq(struct pktgen *, |
137 | struct ifaltq_subque *); | |
138 | static void pktgen_start_ifsq_handler(netmsg_t); | |
4fa8e46e SZ |
139 | |
140 | static d_open_t pktgen_open; | |
141 | static d_close_t pktgen_close; | |
142 | static d_ioctl_t pktgen_ioctl; | |
143 | ||
144 | static struct dev_ops pktgen_ops = { | |
624409c2 | 145 | { CDEV_NAME, 0, D_MPSAFE }, |
4fa8e46e SZ |
146 | .d_open = pktgen_open, |
147 | .d_close = pktgen_close, | |
148 | .d_ioctl = pktgen_ioctl, | |
149 | }; | |
150 | ||
4c9906cb | 151 | static volatile int pktgen_refcnt; |
e3d0d8d7 | 152 | static struct lwkt_token pktgen_tok = LWKT_TOKEN_INITIALIZER(pktgen_token); |
4fa8e46e SZ |
153 | |
154 | MALLOC_DECLARE(M_PKTGEN); | |
155 | MALLOC_DEFINE(M_PKTGEN, CDEV_NAME, "Packet generator"); | |
156 | ||
157 | DEV_MODULE(pktgen, pktgen_modevent, NULL); | |
158 | ||
159 | static int | |
160 | pktgen_modevent(module_t mod, int type, void *data) | |
161 | { | |
390d8a05 | 162 | int error = 0, i; |
4fa8e46e SZ |
163 | |
164 | switch (type) { | |
165 | case MOD_LOAD: | |
390d8a05 SZ |
166 | for (i = 0; i < PKTGEN_DEVCNT; ++i) { |
167 | make_dev(&pktgen_ops, 0, UID_ROOT, GID_WHEEL, 0600, | |
168 | CDEV_NAME"%d", i); | |
169 | } | |
4fa8e46e SZ |
170 | break; |
171 | ||
172 | case MOD_UNLOAD: | |
173 | if (pktgen_refcnt > 0) | |
174 | return EBUSY; | |
4b3b3198 | 175 | dev_ops_remove_all(&pktgen_ops); |
4fa8e46e SZ |
176 | break; |
177 | ||
178 | default: | |
179 | error = EOPNOTSUPP; | |
180 | break; | |
181 | } | |
182 | return error; | |
183 | } | |
184 | ||
185 | static int | |
186 | pktgen_open(struct dev_open_args *ap) | |
187 | { | |
188 | cdev_t dev = ap->a_head.a_dev; | |
189 | struct pktgen *pktg; | |
1bbaa2ad | 190 | int error, i; |
4fa8e46e | 191 | |
9eaf083f | 192 | error = priv_check_cred(ap->a_cred, PRIV_ROOT, 0); |
4fa8e46e SZ |
193 | if (error) |
194 | return error; | |
195 | ||
e3d0d8d7 | 196 | lwkt_gettoken(&pktgen_tok); |
4fa8e46e SZ |
197 | |
198 | if (dev->si_drv1 != NULL) { | |
e3d0d8d7 | 199 | lwkt_reltoken(&pktgen_tok); |
4fa8e46e SZ |
200 | return EBUSY; |
201 | } | |
202 | ||
203 | pktg = kmalloc(sizeof(*pktg), M_PKTGEN, M_ZERO | M_WAITOK); | |
1bbaa2ad SZ |
204 | for (i = 0; i < ncpus; ++i) { |
205 | struct pktgen_pcpu *p = &pktg->pktg_pcpu[i]; | |
206 | ||
207 | callout_init_mp(&p->pktg_stop); | |
208 | LIST_INIT(&p->pktg_buflist); | |
209 | } | |
4fa8e46e | 210 | |
4fa8e46e SZ |
211 | dev->si_drv1 = pktg; |
212 | pktg->pktg_refcnt = 1; | |
213 | ||
4c9906cb | 214 | atomic_add_int(&pktgen_refcnt, 1); |
4fa8e46e | 215 | |
e3d0d8d7 | 216 | lwkt_reltoken(&pktgen_tok); |
4fa8e46e SZ |
217 | return 0; |
218 | } | |
219 | ||
220 | static int | |
221 | pktgen_close(struct dev_close_args *ap) | |
222 | { | |
223 | cdev_t dev = ap->a_head.a_dev; | |
224 | struct pktgen *pktg = dev->si_drv1; | |
225 | ||
e3d0d8d7 | 226 | lwkt_gettoken(&pktgen_tok); |
4fa8e46e | 227 | dev->si_drv1 = NULL; |
e3d0d8d7 | 228 | lwkt_reltoken(&pktgen_tok); |
391741ec | 229 | |
4c9906cb SZ |
230 | pktgen_free(pktg); |
231 | ||
4fa8e46e SZ |
232 | return 0; |
233 | } | |
234 | ||
235 | static int | |
236 | pktgen_ioctl(struct dev_ioctl_args *ap __unused) | |
237 | { | |
238 | cdev_t dev = ap->a_head.a_dev; | |
239 | caddr_t data = ap->a_data; | |
240 | struct pktgen *pktg = dev->si_drv1; | |
241 | int error; | |
242 | ||
e3d0d8d7 | 243 | lwkt_gettoken(&pktgen_tok); |
4fa8e46e SZ |
244 | |
245 | switch (ap->a_cmd) { | |
246 | case PKTGENSTART: | |
b0cd2b4b SZ |
247 | error = pktgen_start(pktg, 0); |
248 | break; | |
249 | ||
250 | case PKTGENMQSTART: | |
251 | error = pktgen_start(pktg, 1); | |
4fa8e46e SZ |
252 | break; |
253 | ||
254 | case PKTGENSCONF: | |
255 | error = pktgen_config(pktg, (const struct pktgen_conf *)data); | |
256 | break; | |
257 | ||
258 | default: | |
259 | error = EOPNOTSUPP; | |
260 | break; | |
261 | } | |
262 | ||
e3d0d8d7 | 263 | lwkt_reltoken(&pktgen_tok); |
4fa8e46e SZ |
264 | return error; |
265 | } | |
266 | ||
267 | static int | |
268 | pktgen_config(struct pktgen *pktg, const struct pktgen_conf *conf) | |
269 | { | |
270 | const struct sockaddr_in *sin; | |
391741ec | 271 | struct sockaddr_in *dst = NULL; |
4fa8e46e SZ |
272 | const struct sockaddr *sa; |
273 | struct ifnet *ifp; | |
391741ec | 274 | size_t dst_size; |
793f802f | 275 | int i, error, pktenq; |
4fa8e46e | 276 | |
391741ec | 277 | if (pktg->pktg_flags & (PKTG_F_RUNNING | PKTG_F_CONFIG)) |
4fa8e46e SZ |
278 | return EBUSY; |
279 | ||
391741ec SZ |
280 | if (conf->pc_datalen <= 0 || |
281 | conf->pc_datalen > ETHERMTU - sizeof(struct udpiphdr)) | |
4fa8e46e SZ |
282 | return EINVAL; |
283 | if (conf->pc_duration <= 0) | |
284 | return EINVAL; | |
285 | ||
4fa8e46e SZ |
286 | sin = &conf->pc_src; |
287 | if (sin->sin_family != AF_INET) | |
288 | return EPROTONOSUPPORT; | |
289 | if (sin->sin_port == 0) | |
290 | return EINVAL; | |
291 | ||
391741ec | 292 | if (conf->pc_ndst <= 0) |
4fa8e46e | 293 | return EINVAL; |
391741ec SZ |
294 | dst_size = conf->pc_ndst * sizeof(struct sockaddr_in); |
295 | ||
296 | dst = kmalloc(dst_size, M_PKTGEN, M_WAITOK | M_NULLOK); | |
297 | if (dst == NULL) | |
298 | return ENOMEM; | |
4fa8e46e | 299 | |
391741ec SZ |
300 | error = copyin(conf->pc_dst, dst, dst_size); |
301 | if (error) | |
302 | goto failed; | |
303 | ||
304 | for (i = 0; i < conf->pc_ndst; ++i) { | |
305 | sin = &dst[i]; | |
306 | if (sin->sin_family != AF_INET) { | |
307 | error = EPROTONOSUPPORT; | |
308 | goto failed; | |
309 | } | |
310 | if (sin->sin_port == 0) { | |
311 | error = EINVAL; | |
312 | goto failed; | |
313 | } | |
314 | } | |
315 | ||
73d83e2b SZ |
316 | /* |
317 | * XXX NOT MPSAFE. | |
318 | * Make sure that pc_ifname is not gone. | |
319 | */ | |
320 | ifnet_lock(); | |
391741ec | 321 | ifp = ifunit(conf->pc_ifname); |
73d83e2b | 322 | ifnet_unlock(); |
391741ec SZ |
323 | if (ifp == NULL) { |
324 | error = ENXIO; | |
325 | goto failed; | |
326 | } | |
327 | ||
793f802f | 328 | pktenq = conf->pc_pktenq; |
2fb36fab | 329 | if (pktenq < 0 || pktenq > ifp->if_snd.altq_maxlen) { |
793f802f SZ |
330 | error = ENOBUFS; |
331 | goto failed; | |
332 | } else if (pktenq == 0) { | |
2fb36fab | 333 | pktenq = (ifp->if_snd.altq_maxlen * 3) / 4; |
793f802f SZ |
334 | } |
335 | ||
391741ec SZ |
336 | sa = &conf->pc_dst_lladdr; |
337 | if (sa->sa_family != AF_LINK) { | |
338 | error = EPROTONOSUPPORT; | |
339 | goto failed; | |
340 | } | |
341 | if (sa->sa_len != ETHER_ADDR_LEN) { | |
342 | error = EPROTONOSUPPORT; | |
343 | goto failed; | |
344 | } | |
345 | if (ETHER_IS_MULTICAST(sa->sa_data) || | |
346 | bcmp(sa->sa_data, ifp->if_broadcastaddr, ifp->if_addrlen) == 0) { | |
347 | error = EADDRNOTAVAIL; | |
348 | goto failed; | |
349 | } | |
350 | ||
351 | /* | |
352 | * Accept the config | |
353 | */ | |
4fa8e46e | 354 | pktg->pktg_flags |= PKTG_F_CONFIG; |
4fa8e46e | 355 | |
eec66bd4 SZ |
356 | if (conf->pc_flags & PKTGEN_FLAG_SWITCH_SRCDST) |
357 | pktg->pktg_flags |= PKTG_F_SWITCH_SRCDST; | |
4fa8e46e | 358 | pktg->pktg_duration = conf->pc_duration; |
4fa8e46e | 359 | pktg->pktg_datalen = conf->pc_datalen; |
793f802f | 360 | pktg->pktg_pktenq = pktenq; |
391741ec SZ |
361 | pktg->pktg_ifp = ifp; |
362 | pktg->pktg_src = conf->pc_src; | |
363 | pktg->pktg_ndst = conf->pc_ndst; | |
364 | KKASSERT(pktg->pktg_dst == NULL); | |
365 | pktg->pktg_dst = dst; | |
4fa8e46e | 366 | bcopy(sa->sa_data, pktg->pktg_dst_lladdr, ETHER_ADDR_LEN); |
c95ebcd6 | 367 | |
4fa8e46e | 368 | return 0; |
391741ec SZ |
369 | |
370 | failed: | |
371 | if (dst != NULL) | |
372 | kfree(dst, M_PKTGEN); | |
373 | return error; | |
4fa8e46e SZ |
374 | } |
375 | ||
b0cd2b4b SZ |
376 | static void |
377 | pktgen_start_ifsq(struct pktgen *pktg, struct ifaltq_subque *ifsq) | |
378 | { | |
379 | struct netmsg_pktgen *np; | |
380 | ||
381 | np = kmalloc(sizeof(*np), M_LWKTMSG, M_WAITOK); | |
382 | netmsg_init(&np->np_base, NULL, &netisr_afree_rport, 0, | |
383 | pktgen_start_ifsq_handler); | |
384 | np->np_pktg = pktg; | |
385 | np->np_ifsq = ifsq; | |
386 | ||
ec7f7fc8 | 387 | lwkt_sendmsg(netisr_cpuport(ifsq_get_cpuid(ifsq)), &np->np_base.lmsg); |
b0cd2b4b SZ |
388 | } |
389 | ||
4fa8e46e | 390 | static int |
b0cd2b4b | 391 | pktgen_start(struct pktgen *pktg, int mq) |
4fa8e46e | 392 | { |
b0cd2b4b SZ |
393 | struct ifaltq *ifq; |
394 | ||
395 | if ((pktg->pktg_flags & PKTG_F_CONFIG) == 0) | |
396 | return EINVAL; | |
397 | if (pktg->pktg_flags & PKTG_F_RUNNING) | |
398 | return EBUSY; | |
399 | pktg->pktg_flags |= PKTG_F_RUNNING; | |
400 | ||
401 | ifq = &pktg->pktg_ifp->if_snd; | |
402 | if (!mq) { | |
403 | pktgen_ref(pktg); | |
404 | pktgen_start_ifsq(pktg, ifq_get_subq_default(ifq)); | |
405 | } else { | |
406 | int i; | |
407 | ||
408 | for (i = 0; i < ifq->altq_subq_cnt; ++i) | |
409 | pktgen_ref(pktg); | |
410 | for (i = 0; i < ifq->altq_subq_cnt; ++i) | |
411 | pktgen_start_ifsq(pktg, ifq_get_subq(ifq, i)); | |
412 | } | |
413 | return 0; | |
414 | } | |
415 | ||
416 | static void | |
417 | pktgen_start_ifsq_handler(netmsg_t nmsg) | |
418 | { | |
419 | struct netmsg_pktgen *np = (struct netmsg_pktgen *)nmsg; | |
420 | struct pktgen *pktg = np->np_pktg; | |
421 | struct ifaltq_subque *ifsq = np->np_ifsq; | |
422 | ||
391741ec SZ |
423 | struct mbuf *m, *head = NULL, **next; |
424 | struct ifnet *ifp; | |
1bbaa2ad | 425 | struct pktgen_pcpu *p; |
b0cd2b4b | 426 | int cpuid, i, alloc_cnt, keep_cnt; |
391741ec SZ |
427 | |
428 | u_short ulen, psum; | |
429 | int len, ip_len; | |
430 | ||
b0cd2b4b SZ |
431 | /* Reply ASAP */ |
432 | lwkt_replymsg(&np->np_base.lmsg, 0); | |
4fa8e46e | 433 | |
391741ec | 434 | ifp = pktg->pktg_ifp; |
4fa8e46e | 435 | |
2fb36fab | 436 | cpuid = ifsq_get_cpuid(ifsq); |
b0cd2b4b | 437 | KKASSERT(cpuid == mycpuid); |
4fa8e46e | 438 | |
1bbaa2ad SZ |
439 | p = &pktg->pktg_pcpu[cpuid]; |
440 | ||
793f802f SZ |
441 | keep_cnt = pktg->pktg_pktenq; |
442 | alloc_cnt = keep_cnt * 2; | |
4fa8e46e | 443 | |
391741ec SZ |
444 | /* |
445 | * Prefault enough mbuf into mbuf objcache | |
446 | */ | |
447 | next = &head; | |
448 | for (i = 0; i < alloc_cnt; ++i) { | |
b5523eac | 449 | MGETHDR(m, M_WAITOK, MT_DATA); |
391741ec SZ |
450 | *next = m; |
451 | next = &m->m_nextpkt; | |
452 | } | |
4fa8e46e | 453 | |
391741ec SZ |
454 | for (i = 0; i < alloc_cnt - keep_cnt; ++i) { |
455 | m = head; | |
456 | head = m->m_nextpkt; | |
457 | m->m_nextpkt = NULL; | |
458 | m_freem(m); | |
459 | } | |
460 | KKASSERT(head != NULL); | |
4fa8e46e | 461 | |
391741ec SZ |
462 | /* |
463 | * Setup the packets' data | |
464 | */ | |
465 | ip_len = pktg->pktg_datalen + sizeof(struct udpiphdr); | |
4fa8e46e SZ |
466 | len = ip_len + ETHER_HDR_LEN; |
467 | ||
391741ec SZ |
468 | psum = htons((u_short)pktg->pktg_datalen + sizeof(struct udphdr) + |
469 | IPPROTO_UDP); | |
4fa8e46e SZ |
470 | ulen = htons(pktg->pktg_datalen + sizeof(struct udphdr)); |
471 | ||
391741ec SZ |
472 | m = head; |
473 | i = 0; | |
474 | while (m != NULL) { | |
475 | struct mbuf *nextm; | |
476 | const struct sockaddr_in *dst; | |
477 | struct pktgen_buf *pb; | |
478 | struct ip *ip; | |
479 | struct udpiphdr *ui; | |
480 | struct ether_header *eh; | |
481 | ||
c6830078 | 482 | pktgen_ref(pktg); |
391741ec SZ |
483 | |
484 | pb = kmalloc(sizeof(*pb), M_PKTGEN, M_WAITOK | M_ZERO); | |
485 | pb->pb_ifp = ifp; | |
2fb36fab | 486 | pb->pb_ifsq = ifsq; |
391741ec SZ |
487 | pb->pb_inuse = 1; |
488 | pb->pb_buf = kmalloc(PKTGEN_BUFSZ, M_PKTGEN, M_WAITOK); | |
489 | pb->pb_len = len; | |
490 | pb->pb_cpuid = cpuid; | |
491 | pb->pb_pktg = pktg; | |
492 | netmsg_init(&pb->pb_nmsg, NULL, &netisr_adone_rport, 0, | |
493 | pktgen_buf_send); | |
1bbaa2ad | 494 | LIST_INSERT_HEAD(&p->pktg_buflist, pb, pb_link); |
391741ec SZ |
495 | |
496 | dst = &pktg->pktg_dst[i % pktg->pktg_ndst]; | |
497 | ++i; | |
498 | ||
499 | m->m_ext.ext_arg = pb; | |
500 | m->m_ext.ext_buf = pb->pb_buf; | |
501 | m->m_ext.ext_free = pktgen_buf_free; | |
502 | m->m_ext.ext_ref = pktgen_buf_ref; | |
503 | m->m_ext.ext_size = PKTGEN_BUFSZ; | |
504 | ||
505 | m->m_data = m->m_ext.ext_buf; | |
506 | m->m_flags |= M_EXT; | |
4fa8e46e SZ |
507 | m->m_len = m->m_pkthdr.len = len; |
508 | ||
391741ec SZ |
509 | m->m_data += ETHER_HDR_LEN; |
510 | m->m_len -= ETHER_HDR_LEN; | |
511 | m->m_pkthdr.len -= ETHER_HDR_LEN; | |
4fa8e46e SZ |
512 | |
513 | ui = mtod(m, struct udpiphdr *); | |
514 | ui->ui_pr = IPPROTO_UDP; | |
eec66bd4 SZ |
515 | if (pktg->pktg_flags & PKTG_F_SWITCH_SRCDST) { |
516 | ui->ui_src.s_addr = dst->sin_addr.s_addr; | |
517 | ui->ui_dst.s_addr = pktg->pktg_src.sin_addr.s_addr; | |
518 | ui->ui_sport = dst->sin_port; | |
519 | ui->ui_dport = pktg->pktg_src.sin_port; | |
520 | } else { | |
521 | ui->ui_src.s_addr = pktg->pktg_src.sin_addr.s_addr; | |
522 | ui->ui_dst.s_addr = dst->sin_addr.s_addr; | |
523 | ui->ui_sport = pktg->pktg_src.sin_port; | |
524 | ui->ui_dport = dst->sin_port; | |
525 | } | |
4fa8e46e | 526 | ui->ui_ulen = ulen; |
391741ec SZ |
527 | ui->ui_sum = in_pseudo(ui->ui_src.s_addr, ui->ui_dst.s_addr, |
528 | psum); | |
64c94a5f | 529 | m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); |
4fa8e46e SZ |
530 | |
531 | ip = (struct ip *)ui; | |
532 | ip->ip_len = ip_len; | |
533 | ip->ip_ttl = 64; /* XXX */ | |
534 | ip->ip_tos = 0; /* XXX */ | |
535 | ip->ip_vhl = IP_VHL_BORING; | |
536 | ip->ip_off = 0; | |
2640b9e5 | 537 | ip->ip_sum = 0; |
4fa8e46e SZ |
538 | ip->ip_id = ip_newid(); |
539 | ||
391741ec | 540 | in_delayed_cksum(m); |
4fa8e46e SZ |
541 | |
542 | ip->ip_len = htons(ip->ip_len); | |
391741ec SZ |
543 | ip->ip_sum = in_cksum_hdr(ip); |
544 | ||
545 | m->m_data -= ETHER_HDR_LEN; | |
546 | m->m_len += ETHER_HDR_LEN; | |
547 | m->m_pkthdr.len += ETHER_HDR_LEN; | |
c95ebcd6 | 548 | |
c95ebcd6 SZ |
549 | eh = mtod(m, struct ether_header *); |
550 | bcopy(pktg->pktg_dst_lladdr, eh->ether_dhost, ETHER_ADDR_LEN); | |
551 | bcopy(IF_LLADDR(ifp), eh->ether_shost, ETHER_ADDR_LEN); | |
552 | eh->ether_type = htons(ETHERTYPE_IP); | |
553 | ||
391741ec SZ |
554 | nextm = m->m_nextpkt; |
555 | m->m_nextpkt = NULL; | |
c95ebcd6 | 556 | |
391741ec | 557 | ifq_dispatch(ifp, m, NULL); |
c95ebcd6 | 558 | |
391741ec SZ |
559 | m = nextm; |
560 | } | |
c95ebcd6 | 561 | |
1bbaa2ad SZ |
562 | callout_reset(&p->pktg_stop, pktg->pktg_duration * hz, |
563 | pktgen_pcpu_stop_cb, p); | |
c95ebcd6 | 564 | |
b0cd2b4b | 565 | pktgen_free(pktg); |
391741ec | 566 | } |
c95ebcd6 | 567 | |
391741ec SZ |
568 | static void |
569 | pktgen_mbuf(struct pktgen_buf *pb, struct mbuf *m) | |
570 | { | |
571 | m->m_ext.ext_arg = pb; | |
572 | m->m_ext.ext_buf = pb->pb_buf; | |
573 | m->m_ext.ext_free = pktgen_buf_free; | |
574 | m->m_ext.ext_ref = pktgen_buf_ref; | |
575 | m->m_ext.ext_size = PKTGEN_BUFSZ; | |
576 | ||
577 | m->m_data = m->m_ext.ext_buf; | |
578 | m->m_flags |= M_EXT; | |
579 | m->m_len = m->m_pkthdr.len = pb->pb_len; | |
580 | } | |
c95ebcd6 | 581 | |
391741ec SZ |
582 | static void |
583 | pktgen_buf_send(netmsg_t msg) | |
584 | { | |
585 | struct pktgen_buf *pb = (struct pktgen_buf *)msg; | |
586 | struct mbuf *m; | |
c95ebcd6 | 587 | |
ec7f7fc8 | 588 | KKASSERT(&curthread->td_msgport == netisr_cpuport(pb->pb_cpuid)); |
c95ebcd6 | 589 | |
391741ec SZ |
590 | crit_enter(); |
591 | lwkt_replymsg(&pb->pb_nmsg.lmsg, 0); | |
592 | crit_exit(); | |
c95ebcd6 | 593 | |
b5523eac | 594 | MGETHDR(m, M_WAITOK, MT_DATA); |
391741ec SZ |
595 | pktgen_mbuf(pb, m); |
596 | ifq_dispatch(pb->pb_ifp, m, NULL); | |
597 | } | |
c95ebcd6 | 598 | |
391741ec SZ |
599 | static void |
600 | pktgen_buf_free(void *arg) | |
601 | { | |
602 | struct pktgen_buf *pb = arg; | |
603 | struct mbuf *m; | |
c95ebcd6 | 604 | |
391741ec SZ |
605 | KKASSERT(pb->pb_inuse > 0); |
606 | if (pb->pb_done) { | |
607 | if (atomic_fetchadd_int(&pb->pb_inuse, -1) == 1) { | |
608 | struct pktgen *pktg; | |
c95ebcd6 | 609 | |
391741ec | 610 | pktg = pb->pb_pktg; |
4c9906cb | 611 | crit_enter(); |
391741ec | 612 | LIST_REMOVE(pb, pb_link); |
4c9906cb | 613 | crit_exit(); |
391741ec SZ |
614 | kfree(pb->pb_buf, M_PKTGEN); |
615 | kfree(pb, M_PKTGEN); | |
4fa8e46e | 616 | |
391741ec | 617 | pktgen_free(pktg); |
4fa8e46e | 618 | } |
391741ec | 619 | return; |
4fa8e46e | 620 | } |
db161a19 | 621 | |
ec7f7fc8 | 622 | if (&curthread->td_msgport != netisr_cpuport(pb->pb_cpuid)) { |
391741ec SZ |
623 | KKASSERT(pb->pb_cpuid == mycpuid); |
624 | crit_enter(); | |
625 | KKASSERT(pb->pb_nmsg.lmsg.ms_flags & MSGF_DONE); | |
ec7f7fc8 | 626 | lwkt_sendmsg(netisr_cpuport(pb->pb_cpuid), &pb->pb_nmsg.lmsg); |
391741ec SZ |
627 | crit_exit(); |
628 | return; | |
629 | } | |
630 | ||
b5523eac | 631 | MGETHDR(m, M_WAITOK, MT_DATA); |
391741ec | 632 | pktgen_mbuf(pb, m); |
2fb36fab | 633 | ifsq_enqueue(pb->pb_ifsq, m, NULL); |
db161a19 SZ |
634 | } |
635 | ||
636 | static void | |
391741ec | 637 | pktgen_buf_ref(void *arg) |
db161a19 | 638 | { |
391741ec | 639 | struct pktgen_buf *pb = arg; |
db161a19 | 640 | |
391741ec | 641 | panic("%s should never be called\n", __func__); |
4fa8e46e | 642 | |
391741ec SZ |
643 | KKASSERT(pb->pb_inuse > 0); |
644 | atomic_add_int(&pb->pb_inuse, 1); | |
645 | } | |
4fa8e46e | 646 | |
391741ec SZ |
647 | static void |
648 | pktgen_free(struct pktgen *pktg) | |
649 | { | |
4fa8e46e | 650 | KKASSERT(pktg->pktg_refcnt > 0); |
4c9906cb | 651 | if (atomic_fetchadd_int(&pktg->pktg_refcnt, -1) == 1) { |
1bbaa2ad SZ |
652 | int i; |
653 | ||
391741ec SZ |
654 | if (pktg->pktg_dst != NULL) |
655 | kfree(pktg->pktg_dst, M_PKTGEN); | |
1bbaa2ad SZ |
656 | |
657 | for (i = 0; i < ncpus; ++i) | |
658 | KKASSERT(LIST_EMPTY(&pktg->pktg_pcpu[i].pktg_buflist)); | |
391741ec SZ |
659 | kfree(pktg, M_PKTGEN); |
660 | } | |
4fa8e46e SZ |
661 | |
662 | KKASSERT(pktgen_refcnt > 0); | |
4c9906cb | 663 | atomic_subtract_int(&pktgen_refcnt, 1); |
391741ec SZ |
664 | } |
665 | ||
666 | static void | |
1bbaa2ad | 667 | pktgen_pcpu_stop_cb(void *arg) |
391741ec | 668 | { |
1bbaa2ad | 669 | struct pktgen_pcpu *p = arg; |
391741ec | 670 | struct pktgen_buf *pb; |
4fa8e46e | 671 | |
4c9906cb | 672 | crit_enter(); |
1bbaa2ad | 673 | LIST_FOREACH(pb, &p->pktg_buflist, pb_link) |
391741ec | 674 | pb->pb_done = 1; |
4c9906cb | 675 | crit_exit(); |
4fa8e46e | 676 | } |
c6830078 SZ |
677 | |
678 | static void | |
679 | pktgen_ref(struct pktgen *pktg) | |
680 | { | |
681 | atomic_add_int(&pktg->pktg_refcnt, 1); | |
682 | atomic_add_int(&pktgen_refcnt, 1); | |
683 | } |