Get an mbuf packet header to begin with instead of getting an mbuf and then
[dragonfly.git] / sys / kern / uipc_mbuf.c
CommitLineData
984263bc 1/*
0c33f36d 2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved.
66d6c637
JH
3 * Copyright (c) 2004 The DragonFly Project. All rights reserved.
4 *
5 * This code is derived from software contributed to The DragonFly Project
6 * by Jeffrey M. Hsu.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the name of The DragonFly Project nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific, prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
32 */
33
34/*
35 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved.
36 *
37 * License terms: all terms for the DragonFly license above plus the following:
38 *
39 * 4. All advertising materials mentioning features or use of this software
40 * must display the following acknowledgement:
41 *
42 * This product includes software developed by Jeffrey M. Hsu
43 * for the DragonFly Project.
44 *
45 * This requirement may be waived with permission from Jeffrey Hsu.
46 * This requirement will sunset and may be removed on July 8 2005,
47 * after which the standard DragonFly license (as shown above) will
48 * apply.
49 */
50
51/*
984263bc
MD
52 * Copyright (c) 1982, 1986, 1988, 1991, 1993
53 * The Regents of the University of California. All rights reserved.
54 *
55 * Redistribution and use in source and binary forms, with or without
56 * modification, are permitted provided that the following conditions
57 * are met:
58 * 1. Redistributions of source code must retain the above copyright
59 * notice, this list of conditions and the following disclaimer.
60 * 2. Redistributions in binary form must reproduce the above copyright
61 * notice, this list of conditions and the following disclaimer in the
62 * documentation and/or other materials provided with the distribution.
63 * 3. All advertising materials mentioning features or use of this software
64 * must display the following acknowledgement:
65 * This product includes software developed by the University of
66 * California, Berkeley and its contributors.
67 * 4. Neither the name of the University nor the names of its contributors
68 * may be used to endorse or promote products derived from this software
69 * without specific prior written permission.
70 *
71 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
72 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
73 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
74 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
75 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
76 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
77 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
78 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
79 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
80 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
81 * SUCH DAMAGE.
82 *
8a3125c6 83 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94
984263bc 84 * $FreeBSD: src/sys/kern/uipc_mbuf.c,v 1.51.2.24 2003/04/15 06:59:29 silby Exp $
e0d05288 85 * $DragonFly: src/sys/kern/uipc_mbuf.c,v 1.46 2005/06/08 23:39:08 hsu Exp $
984263bc
MD
86 */
87
88#include "opt_param.h"
7b6f875f 89#include "opt_ddb.h"
984263bc
MD
90#include "opt_mbuf_stress_test.h"
91#include <sys/param.h>
92#include <sys/systm.h>
93#include <sys/malloc.h>
94#include <sys/mbuf.h>
95#include <sys/kernel.h>
96#include <sys/sysctl.h>
97#include <sys/domain.h>
7b6f875f 98#include <sys/objcache.h>
984263bc 99#include <sys/protosw.h>
0c33f36d 100#include <sys/uio.h>
ef0fdad1 101#include <sys/thread.h>
a2a5ad0d 102#include <sys/globaldata.h>
90775e29 103#include <sys/thread2.h>
984263bc
MD
104
105#include <vm/vm.h>
106#include <vm/vm_kern.h>
107#include <vm/vm_extern.h>
108
109#ifdef INVARIANTS
110#include <machine/cpu.h>
111#endif
112
90775e29
MD
113/*
114 * mbuf cluster meta-data
115 */
7b6f875f 116struct mbcluster {
90775e29
MD
117 int32_t mcl_refs;
118 void *mcl_data;
7b6f875f 119};
90775e29 120
7b6f875f 121static void mbinit(void *);
984263bc
MD
122SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbinit, NULL)
123
90775e29
MD
124static u_long mbtypes[MT_NTYPES];
125
984263bc 126struct mbstat mbstat;
984263bc
MD
127int max_linkhdr;
128int max_protohdr;
129int max_hdr;
130int max_datalen;
131int m_defragpackets;
132int m_defragbytes;
133int m_defraguseless;
134int m_defragfailure;
135#ifdef MBUF_STRESS_TEST
136int m_defragrandomfailures;
137#endif
138
7b6f875f
JH
139struct objcache *mbuf_cache, *mbufphdr_cache;
140struct objcache *mclmeta_cache;
141struct objcache *mbufcluster_cache, *mbufphdrcluster_cache;
142
984263bc
MD
143int nmbclusters;
144int nmbufs;
984263bc 145
984263bc
MD
146SYSCTL_INT(_kern_ipc, KIPC_MAX_LINKHDR, max_linkhdr, CTLFLAG_RW,
147 &max_linkhdr, 0, "");
148SYSCTL_INT(_kern_ipc, KIPC_MAX_PROTOHDR, max_protohdr, CTLFLAG_RW,
149 &max_protohdr, 0, "");
150SYSCTL_INT(_kern_ipc, KIPC_MAX_HDR, max_hdr, CTLFLAG_RW, &max_hdr, 0, "");
151SYSCTL_INT(_kern_ipc, KIPC_MAX_DATALEN, max_datalen, CTLFLAG_RW,
152 &max_datalen, 0, "");
153SYSCTL_INT(_kern_ipc, OID_AUTO, mbuf_wait, CTLFLAG_RW,
154 &mbuf_wait, 0, "");
155SYSCTL_STRUCT(_kern_ipc, KIPC_MBSTAT, mbstat, CTLFLAG_RW, &mbstat, mbstat, "");
156SYSCTL_OPAQUE(_kern_ipc, OID_AUTO, mbtypes, CTLFLAG_RD, mbtypes,
157 sizeof(mbtypes), "LU", "");
b35fbf6c 158SYSCTL_INT(_kern_ipc, KIPC_NMBCLUSTERS, nmbclusters, CTLFLAG_RW,
984263bc 159 &nmbclusters, 0, "Maximum number of mbuf clusters available");
b35fbf6c 160SYSCTL_INT(_kern_ipc, OID_AUTO, nmbufs, CTLFLAG_RW, &nmbufs, 0,
984263bc 161 "Maximum number of mbufs available");
7b6f875f 162
984263bc
MD
163SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragpackets, CTLFLAG_RD,
164 &m_defragpackets, 0, "");
165SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragbytes, CTLFLAG_RD,
166 &m_defragbytes, 0, "");
167SYSCTL_INT(_kern_ipc, OID_AUTO, m_defraguseless, CTLFLAG_RD,
168 &m_defraguseless, 0, "");
169SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragfailure, CTLFLAG_RD,
170 &m_defragfailure, 0, "");
171#ifdef MBUF_STRESS_TEST
172SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW,
173 &m_defragrandomfailures, 0, "");
174#endif
175
90775e29
MD
176static MALLOC_DEFINE(M_MBUF, "mbuf", "mbuf");
177static MALLOC_DEFINE(M_MBUFCL, "mbufcl", "mbufcl");
7b6f875f 178static MALLOC_DEFINE(M_MCLMETA, "mclmeta", "mclmeta");
90775e29
MD
179
180static void m_reclaim (void);
90775e29
MD
181static void m_mclref(void *arg);
182static void m_mclfree(void *arg);
984263bc
MD
183
184#ifndef NMBCLUSTERS
185#define NMBCLUSTERS (512 + maxusers * 16)
186#endif
187#ifndef NMBUFS
7b6f875f 188#define NMBUFS (nmbclusters * 2)
984263bc
MD
189#endif
190
191/*
192 * Perform sanity checks of tunables declared above.
193 */
194static void
195tunable_mbinit(void *dummy)
196{
197
198 /*
199 * This has to be done before VM init.
200 */
201 nmbclusters = NMBCLUSTERS;
202 TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters);
203 nmbufs = NMBUFS;
204 TUNABLE_INT_FETCH("kern.ipc.nmbufs", &nmbufs);
205 /* Sanity checks */
206 if (nmbufs < nmbclusters * 2)
207 nmbufs = nmbclusters * 2;
208
209 return;
210}
211SYSINIT(tunable_mbinit, SI_SUB_TUNABLES, SI_ORDER_ANY, tunable_mbinit, NULL);
212
213/* "number of clusters of pages" */
214#define NCL_INIT 1
215
216#define NMB_INIT 16
217
7b6f875f
JH
218/*
219 * The mbuf object cache only guarantees that m_next and m_nextpkt are
220 * NULL and that m_data points to the beginning of the data area. In
221 * particular, m_len and m_pkthdr.len are uninitialized. It is the
222 * responsibility of the caller to initialize those fields before use.
223 */
224
225static boolean_t __inline
226mbuf_ctor(void *obj, void *private, int ocflags)
984263bc 227{
7b6f875f 228 struct mbuf *m = obj;
984263bc 229
7b6f875f
JH
230 m->m_next = NULL;
231 m->m_nextpkt = NULL;
232 m->m_data = m->m_dat;
233 m->m_flags = 0;
234
235 return (TRUE);
984263bc
MD
236}
237
238/*
7b6f875f 239 * Initialize the mbuf and the packet header fields.
984263bc 240 */
7b6f875f
JH
241static boolean_t
242mbufphdr_ctor(void *obj, void *private, int ocflags)
984263bc 243{
7b6f875f 244 struct mbuf *m = obj;
984263bc 245
7b6f875f
JH
246 m->m_next = NULL;
247 m->m_nextpkt = NULL;
248 m->m_data = m->m_pktdat;
77e294a1 249 m->m_flags = M_PKTHDR | M_PHCACHE;
984263bc 250
7b6f875f
JH
251 m->m_pkthdr.rcvif = NULL; /* eliminate XXX JH */
252 SLIST_INIT(&m->m_pkthdr.tags);
253 m->m_pkthdr.csum_flags = 0; /* eliminate XXX JH */
254 m->m_pkthdr.fw_flags = 0; /* eliminate XXX JH */
255
256 return (TRUE);
984263bc
MD
257}
258
259/*
7b6f875f 260 * A mbcluster object consists of 2K (MCLBYTES) cluster and a refcount.
984263bc 261 */
7b6f875f
JH
262static boolean_t
263mclmeta_ctor(void *obj, void *private, int ocflags)
984263bc 264{
7b6f875f
JH
265 struct mbcluster *cl = obj;
266 void *buf;
267
268 if (ocflags & M_NOWAIT)
269 buf = malloc(MCLBYTES, M_MBUFCL, M_NOWAIT | M_ZERO);
270 else
271 buf = malloc(MCLBYTES, M_MBUFCL, M_INTWAIT | M_ZERO);
272 if (buf == NULL)
273 return (FALSE);
77e294a1 274 cl->mcl_refs = 0;
7b6f875f
JH
275 cl->mcl_data = buf;
276 return (TRUE);
277}
984263bc 278
7b6f875f
JH
279static void
280linkcluster(struct mbuf *m, struct mbcluster *cl)
281{
984263bc 282 /*
7b6f875f
JH
283 * Add the cluster to the mbuf. The caller will detect that the
284 * mbuf now has an attached cluster.
984263bc 285 */
7b6f875f
JH
286 m->m_ext.ext_arg = cl;
287 m->m_ext.ext_buf = cl->mcl_data;
288 m->m_ext.ext_ref = m_mclref;
289 m->m_ext.ext_free = m_mclfree;
290 m->m_ext.ext_size = MCLBYTES;
77e294a1 291 ++cl->mcl_refs;
984263bc 292
7b6f875f
JH
293 m->m_data = m->m_ext.ext_buf;
294 m->m_flags |= M_EXT | M_EXT_CLUSTER;
984263bc
MD
295}
296
7b6f875f
JH
297static boolean_t
298mbufphdrcluster_ctor(void *obj, void *private, int ocflags)
299{
300 struct mbuf *m = obj;
301 struct mbcluster *cl;
302
303 mbufphdr_ctor(obj, private, ocflags);
304 cl = objcache_get(mclmeta_cache, ocflags);
305 if (cl == NULL)
306 return (FALSE);
77e294a1 307 m->m_flags |= M_CLCACHE;
7b6f875f
JH
308 linkcluster(m, cl);
309 return (TRUE);
310}
984263bc 311
7b6f875f
JH
312static boolean_t
313mbufcluster_ctor(void *obj, void *private, int ocflags)
984263bc 314{
7b6f875f
JH
315 struct mbuf *m = obj;
316 struct mbcluster *cl;
317
318 mbuf_ctor(obj, private, ocflags);
319 cl = objcache_get(mclmeta_cache, ocflags);
320 if (cl == NULL)
321 return (FALSE);
77e294a1 322 m->m_flags |= M_CLCACHE;
7b6f875f
JH
323 linkcluster(m, cl);
324 return (TRUE);
325}
984263bc 326
7b6f875f
JH
327static void
328mclmeta_dtor(void *obj, void *private)
329{
330 struct mbcluster *mcl = obj;
984263bc 331
77e294a1 332 KKASSERT(mcl->mcl_refs == 0);
7b6f875f 333 free(mcl->mcl_data, M_MBUFCL);
984263bc
MD
334}
335
77e294a1
MD
336/*
337 * Used for both the cluster and cluster PHDR caches.
338 *
339 * The mbuf may have lost its cluster due to sharing, deal
340 * with the situation by checking M_EXT.
341 */
7b6f875f
JH
342static void
343mbufcluster_dtor(void *obj, void *private)
984263bc 344{
7b6f875f 345 struct mbuf *m = obj;
77e294a1 346 struct mbcluster *mcl;
984263bc 347
77e294a1
MD
348 if (m->m_flags & M_EXT) {
349 KKASSERT((m->m_flags & M_EXT_CLUSTER) != 0);
350 mcl = m->m_ext.ext_arg;
351 KKASSERT(mcl->mcl_refs == 1);
352 mcl->mcl_refs = 0;
353 objcache_put(mclmeta_cache, mcl);
354 }
984263bc
MD
355}
356
7b6f875f
JH
357struct objcache_malloc_args mbuf_malloc_args = { MSIZE, M_MBUF };
358struct objcache_malloc_args mclmeta_malloc_args =
359 { sizeof(struct mbcluster), M_MCLMETA };
360
361/* ARGSUSED*/
90775e29 362static void
7b6f875f 363mbinit(void *dummy)
984263bc 364{
7b6f875f
JH
365 mbstat.m_msize = MSIZE;
366 mbstat.m_mclbytes = MCLBYTES;
367 mbstat.m_minclsize = MINCLSIZE;
368 mbstat.m_mlen = MLEN;
369 mbstat.m_mhlen = MHLEN;
984263bc 370
7b6f875f
JH
371 mbuf_cache = objcache_create("mbuf", nmbufs, 0,
372 mbuf_ctor, null_dtor, NULL,
373 objcache_malloc_alloc, objcache_malloc_free, &mbuf_malloc_args);
374 mbufphdr_cache = objcache_create("mbuf pkt hdr", nmbufs, 64,
375 mbufphdr_ctor, null_dtor, NULL,
376 objcache_malloc_alloc, objcache_malloc_free, &mbuf_malloc_args);
377 mclmeta_cache = objcache_create("cluster mbuf", nmbclusters , 0,
378 mclmeta_ctor, mclmeta_dtor, NULL,
379 objcache_malloc_alloc, objcache_malloc_free, &mclmeta_malloc_args);
380 mbufcluster_cache = objcache_create("mbuf + cluster", nmbclusters, 0,
381 mbufcluster_ctor, mbufcluster_dtor, NULL,
382 objcache_malloc_alloc, objcache_malloc_free, &mbuf_malloc_args);
383 mbufphdrcluster_cache = objcache_create("mbuf pkt hdr + cluster",
77e294a1 384 nmbclusters, 64, mbufphdrcluster_ctor, mbufcluster_dtor, NULL,
7b6f875f
JH
385 objcache_malloc_alloc, objcache_malloc_free, &mbuf_malloc_args);
386 return;
90775e29 387}
984263bc 388
90775e29
MD
389/*
390 * Return the number of references to this mbuf's data. 0 is returned
391 * if the mbuf is not M_EXT, a reference count is returned if it is
7b6f875f 392 * M_EXT | M_EXT_CLUSTER, and 99 is returned if it is a special M_EXT.
90775e29
MD
393 */
394int
395m_sharecount(struct mbuf *m)
396{
7b6f875f
JH
397 switch (m->m_flags & (M_EXT | M_EXT_CLUSTER)) {
398 case 0:
399 return (0);
400 case M_EXT:
401 return (99);
402 case M_EXT | M_EXT_CLUSTER:
403 return (((struct mbcluster *)m->m_ext.ext_arg)->mcl_refs);
404 }
405 /* NOTREACHED */
406 return (0); /* to shut up compiler */
90775e29
MD
407}
408
409/*
410 * change mbuf to new type
411 */
412void
413m_chtype(struct mbuf *m, int type)
414{
c6339e39 415 crit_enter();
90775e29 416 ++mbtypes[type];
87b85c14
MD
417 --mbtypes[m->m_type];
418 m->m_type = type;
c6339e39 419 crit_exit();
984263bc
MD
420}
421
984263bc 422static void
8a3125c6 423m_reclaim(void)
984263bc 424{
1fd87d54
RG
425 struct domain *dp;
426 struct protosw *pr;
984263bc 427
c6339e39 428 crit_enter();
9c70fe43 429 SLIST_FOREACH(dp, &domains, dom_next) {
8a3125c6 430 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) {
984263bc
MD
431 if (pr->pr_drain)
432 (*pr->pr_drain)();
8a3125c6
MD
433 }
434 }
c6339e39 435 crit_exit();
984263bc
MD
436 mbstat.m_drain++;
437}
438
7b6f875f
JH
439static void __inline
440updatestats(struct mbuf *m, int type)
441{
442 m->m_type = type;
443
444 crit_enter();
445 ++mbtypes[type];
446 ++mbstat.m_mbufs;
447 crit_exit();
448}
449
984263bc 450/*
7b6f875f 451 * Allocate an mbuf.
984263bc
MD
452 */
453struct mbuf *
8a3125c6 454m_get(int how, int type)
984263bc 455{
12496bdf 456 struct mbuf *m;
7b6f875f
JH
457 int ntries = 0;
458 int ocf = MBTOM(how);
12496bdf 459
7b6f875f
JH
460retryonce:
461
462 m = objcache_get(mbuf_cache, ocf);
463
464 if (m == NULL) {
465 if ((how & MB_TRYWAIT) && ntries++ == 0) {
466 struct objcache *reclaimlist[] = {
467 mbufphdr_cache,
468 mbufcluster_cache, mbufphdrcluster_cache
469 };
470 const int nreclaims = __arysize(reclaimlist);
471
472 if (!objcache_reclaimlist(reclaimlist, nreclaims, ocf))
473 m_reclaim();
474 goto retryonce;
c6339e39 475 }
7b6f875f 476 return (NULL);
12496bdf 477 }
c6339e39 478
7b6f875f 479 updatestats(m, type);
984263bc
MD
480 return (m);
481}
482
483struct mbuf *
8a3125c6 484m_gethdr(int how, int type)
984263bc 485{
12496bdf 486 struct mbuf *m;
7b6f875f
JH
487 int ocf = MBTOM(how);
488 int ntries = 0;
12496bdf 489
7b6f875f
JH
490retryonce:
491
492 m = objcache_get(mbufphdr_cache, ocf);
493
494 if (m == NULL) {
495 if ((how & MB_TRYWAIT) && ntries++ == 0) {
496 struct objcache *reclaimlist[] = {
497 mbuf_cache,
498 mbufcluster_cache, mbufphdrcluster_cache
499 };
500 const int nreclaims = __arysize(reclaimlist);
501
502 if (!objcache_reclaimlist(reclaimlist, nreclaims, ocf))
503 m_reclaim();
504 goto retryonce;
c6339e39 505 }
7b6f875f 506 return (NULL);
12496bdf 507 }
c6339e39 508
7b6f875f 509 updatestats(m, type);
984263bc
MD
510 return (m);
511}
512
7b6f875f
JH
513/*
514 * Get a mbuf (not a mbuf cluster!) and zero it.
515 * Deprecated.
516 */
984263bc 517struct mbuf *
8a3125c6 518m_getclr(int how, int type)
984263bc 519{
1fd87d54 520 struct mbuf *m;
984263bc 521
7b6f875f
JH
522 m = m_get(how, type);
523 if (m != NULL)
524 bzero(m->m_data, MLEN);
984263bc
MD
525 return (m);
526}
527
528/*
7b6f875f 529 * Returns an mbuf with an attached cluster.
984263bc
MD
530 * Because many network drivers use this kind of buffers a lot, it is
531 * convenient to keep a small pool of free buffers of this kind.
532 * Even a small size such as 10 gives about 10% improvement in the
533 * forwarding rate in a bridge or router.
984263bc 534 */
984263bc
MD
535struct mbuf *
536m_getcl(int how, short type, int flags)
537{
7b6f875f
JH
538 struct mbuf *m;
539 int ocflags = MBTOM(how);
540 int ntries = 0;
984263bc 541
7b6f875f
JH
542retryonce:
543
544 if (flags & M_PKTHDR)
545 m = objcache_get(mbufphdrcluster_cache, ocflags);
546 else
547 m = objcache_get(mbufcluster_cache, ocflags);
548
549 if (m == NULL) {
550 if ((how & MB_TRYWAIT) && ntries++ == 0) {
551 struct objcache *reclaimlist[1];
552
553 if (flags & M_PKTHDR)
554 reclaimlist[0] = mbufcluster_cache;
555 else
556 reclaimlist[0] = mbufphdrcluster_cache;
557 if (!objcache_reclaimlist(reclaimlist, 1, ocflags))
558 m_reclaim();
559 goto retryonce;
984263bc 560 }
7b6f875f 561 return (NULL);
984263bc 562 }
7b6f875f
JH
563
564 m->m_type = type;
565
566 crit_enter();
567 ++mbtypes[type];
568 ++mbstat.m_clusters;
c6339e39 569 crit_exit();
7b6f875f 570 return (m);
984263bc
MD
571}
572
573/*
50503f0f
JH
574 * Allocate chain of requested length.
575 */
576struct mbuf *
577m_getc(int len, int how, int type)
578{
579 struct mbuf *n, *nfirst = NULL, **ntail = &nfirst;
580 int nsize;
581
582 while (len > 0) {
583 n = m_getl(len, how, type, 0, &nsize);
584 if (n == NULL)
585 goto failed;
586 n->m_len = 0;
587 *ntail = n;
588 ntail = &n->m_next;
589 len -= nsize;
590 }
591 return (nfirst);
592
593failed:
594 m_freem(nfirst);
595 return (NULL);
596}
597
598/*
599 * Allocate len-worth of mbufs and/or mbuf clusters (whatever fits best)
600 * and return a pointer to the head of the allocated chain. If m0 is
984263bc
MD
601 * non-null, then we assume that it is a single mbuf or an mbuf chain to
602 * which we want len bytes worth of mbufs and/or clusters attached, and so
50503f0f 603 * if we succeed in allocating it, we will just return a pointer to m0.
984263bc
MD
604 *
605 * If we happen to fail at any point during the allocation, we will free
606 * up everything we have already allocated and return NULL.
607 *
50503f0f 608 * Deprecated. Use m_getc() and m_cat() instead.
984263bc
MD
609 */
610struct mbuf *
50503f0f 611m_getm(struct mbuf *m0, int len, int how, int type)
984263bc 612{
50503f0f 613 struct mbuf *nfirst;
984263bc 614
50503f0f 615 nfirst = m_getc(len, how, type);
984263bc 616
50503f0f
JH
617 if (m0 != NULL) {
618 m_last(m0)->m_next = nfirst;
619 return (m0);
984263bc
MD
620 }
621
50503f0f 622 return (nfirst);
984263bc
MD
623}
624
625/*
7b6f875f
JH
626 * Adds a cluster to a normal mbuf, M_EXT is set on success.
627 * Deprecated. Use m_getcl() instead.
b6650ec0 628 */
90775e29
MD
629void
630m_mclget(struct mbuf *m, int how)
b6650ec0 631{
7b6f875f 632 struct mbcluster *mcl;
b6650ec0 633
77e294a1 634 KKASSERT((m->m_flags & M_EXT) == 0);
7b6f875f
JH
635 mcl = objcache_get(mclmeta_cache, MBTOM(how));
636 if (mcl == NULL)
637 return;
638 linkcluster(m, mcl);
90775e29 639
7b6f875f
JH
640 crit_enter();
641 --mbstat.m_mbufs;
642 ++mbstat.m_clusters;
c6339e39 643 crit_exit();
b6650ec0
MD
644}
645
90775e29 646static void
7b6f875f 647m_mclref(void *arg)
b6650ec0 648{
7b6f875f 649 struct mbcluster *mcl = arg;
90775e29 650
7b6f875f 651 atomic_add_int(&mcl->mcl_refs, 1);
b6650ec0
MD
652}
653
90775e29 654static void
7b6f875f 655m_mclfree(void *arg)
b6650ec0 656{
7b6f875f 657 struct mbcluster *mcl = arg;
90775e29 658
77e294a1
MD
659 /* XXX interrupt race. Currently called from a critical section */
660 if (mcl->mcl_refs > 1) {
661 atomic_subtract_int(&mcl->mcl_refs, 1);
662 } else {
663 KKASSERT(mcl->mcl_refs == 1);
664 mcl->mcl_refs = 0;
665 objcache_put(mclmeta_cache, mcl);
666 }
b6650ec0
MD
667}
668
7b6f875f 669extern void db_print_backtrace(void);
7eccf245 670
b6650ec0 671/*
b6650ec0
MD
672 * Free a single mbuf and any associated external storage. The successor,
673 * if any, is returned.
984263bc 674 *
b6650ec0 675 * We do need to check non-first mbuf for m_aux, since some of existing
984263bc
MD
676 * code does not call M_PREPEND properly.
677 * (example: call to bpf_mtap from drivers)
678 */
984263bc 679struct mbuf *
b6650ec0 680m_free(struct mbuf *m)
984263bc 681{
b6650ec0
MD
682 struct mbuf *n;
683
361af367 684 KASSERT(m->m_type != MT_FREE, ("freeing free mbuf %p", m));
7b6f875f 685 --mbtypes[m->m_type];
90775e29 686
7b6f875f 687 n = m->m_next;
90775e29
MD
688
689 /*
7b6f875f
JH
690 * Make sure the mbuf is in constructed state before returning it
691 * to the objcache.
90775e29 692 */
90775e29 693 m->m_next = NULL;
7b6f875f
JH
694#ifdef notyet
695 KKASSERT(m->m_nextpkt == NULL);
696#else
697 if (m->m_nextpkt != NULL) {
698#ifdef DDB
699 static int afewtimes = 10;
700
701 if (afewtimes-- > 0) {
702 printf("mfree: m->m_nextpkt != NULL\n");
703 db_print_backtrace();
90775e29 704 }
7b6f875f
JH
705#endif
706 m->m_nextpkt = NULL;
707 }
708#endif
709 if (m->m_flags & M_PKTHDR) {
7b6f875f 710 m_tag_delete_chain(m); /* eliminate XXX JH */
77e294a1
MD
711 }
712
713 m->m_flags &= (M_EXT | M_EXT_CLUSTER | M_CLCACHE | M_PHCACHE);
714
715 /*
716 * Clean the M_PKTHDR state so we can return the mbuf to its original
717 * cache. This is based on the PHCACHE flag which tells us whether
718 * the mbuf was originally allocated out of a packet-header cache
719 * or a non-packet-header cache.
720 */
721 if (m->m_flags & M_PHCACHE) {
722 m->m_flags |= M_PKTHDR;
723 m->m_pkthdr.rcvif = NULL; /* eliminate XXX JH */
7b6f875f
JH
724 m->m_pkthdr.csum_flags = 0; /* eliminate XXX JH */
725 m->m_pkthdr.fw_flags = 0; /* eliminate XXX JH */
90775e29 726 }
7b6f875f 727
77e294a1
MD
728 /*
729 * Handle remaining flags combinations. M_CLCACHE tells us whether
730 * the mbuf was originally allocated from a cluster cache or not,
731 * and is totally separate from whether the mbuf is currently
732 * associated with a cluster.
733 */
734 crit_enter();
735 switch(m->m_flags & (M_CLCACHE | M_EXT | M_EXT_CLUSTER)) {
736 case M_CLCACHE | M_EXT | M_EXT_CLUSTER:
737 /*
738 * mbuf+cluster cache case. The mbuf was allocated from the
739 * combined mbuf_cluster cache and can be returned to the
740 * cache if the cluster hasn't been shared.
741 */
742 if (m_sharecount(m) == 1) {
743 /*
744 * The cluster has not been shared, we can just
745 * reset the data pointer and return the mbuf
746 * to the cluster cache. Note that the reference
747 * count is left intact (it is still associated with
748 * an mbuf).
749 */
750 m->m_data = m->m_ext.ext_buf;
751 if (m->m_flags & M_PHCACHE)
752 objcache_put(mbufphdrcluster_cache, m);
753 else
754 objcache_put(mbufcluster_cache, m);
755 } else {
756 /*
757 * Hell. Someone else has a ref on this cluster,
758 * we have to disconnect it which means we can't
759 * put it back into the mbufcluster_cache, we
760 * have to destroy the mbuf.
761 *
762 * XXX we could try to connect another cluster to
763 * it.
764 */
7b6f875f
JH
765 m->m_ext.ext_free(m->m_ext.ext_arg);
766 m->m_flags &= ~(M_EXT | M_EXT_CLUSTER);
77e294a1
MD
767 if (m->m_flags & M_PHCACHE)
768 objcache_dtor(mbufphdrcluster_cache, m);
769 else
770 objcache_dtor(mbufcluster_cache, m);
7b6f875f 771 }
7b6f875f 772 --mbstat.m_clusters;
77e294a1
MD
773 break;
774 case M_EXT | M_EXT_CLUSTER:
775 /*
776 * Normal cluster associated with an mbuf that was allocated
777 * from the normal mbuf pool rather then the cluster pool.
778 * The cluster has to be independantly disassociated from the
779 * mbuf.
780 */
781 --mbstat.m_clusters;
782 /* fall through */
783 case M_EXT:
784 /*
785 * Normal cluster association case, disconnect the cluster from
786 * the mbuf. The cluster may or may not be custom.
787 */
788 m->m_ext.ext_free(m->m_ext.ext_arg);
789 m->m_flags &= ~(M_EXT | M_EXT_CLUSTER);
790 /* fall through */
791 case 0:
792 /*
793 * return the mbuf to the mbuf cache.
794 */
795 if (m->m_flags & M_PHCACHE) {
7b6f875f
JH
796 m->m_data = m->m_pktdat;
797 objcache_put(mbufphdr_cache, m);
90775e29 798 } else {
7b6f875f
JH
799 m->m_data = m->m_dat;
800 objcache_put(mbuf_cache, m);
90775e29 801 }
7b6f875f 802 --mbstat.m_mbufs;
77e294a1
MD
803 break;
804 default:
805 if (!panicstr)
806 panic("bad mbuf flags %p %08x\n", m, m->m_flags);
807 break;
b6650ec0 808 }
77e294a1 809 crit_exit();
984263bc
MD
810 return (n);
811}
812
813void
b6650ec0 814m_freem(struct mbuf *m)
984263bc 815{
c6339e39 816 crit_enter();
90775e29
MD
817 while (m)
818 m = m_free(m);
c6339e39 819 crit_exit();
984263bc
MD
820}
821
822/*
df80f2ea 823 * mbuf utility routines
984263bc
MD
824 */
825
826/*
7b6f875f 827 * Lesser-used path for M_PREPEND: allocate new mbuf to prepend to chain and
984263bc
MD
828 * copy junk along.
829 */
830struct mbuf *
8a3125c6 831m_prepend(struct mbuf *m, int len, int how)
984263bc
MD
832{
833 struct mbuf *mn;
834
410ed080 835 mn = m_getl(MLEN, how, m->m_type, m->m_flags & M_PKTHDR, NULL);
7b6f875f 836 if (mn == NULL) {
984263bc 837 m_freem(m);
7b6f875f 838 return (NULL);
984263bc
MD
839 }
840 if (m->m_flags & M_PKTHDR)
841 M_MOVE_PKTHDR(mn, m);
842 mn->m_next = m;
843 m = mn;
844 if (len < MHLEN)
845 MH_ALIGN(m, len);
846 m->m_len = len;
847 return (m);
848}
849
850/*
851 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
852 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
74f1caca 853 * The wait parameter is a choice of MB_WAIT/MB_DONTWAIT from caller.
984263bc
MD
854 * Note that the copy is read-only, because clusters are not copied,
855 * only their reference counts are incremented.
856 */
984263bc 857struct mbuf *
8a3125c6 858m_copym(const struct mbuf *m, int off0, int len, int wait)
984263bc 859{
1fd87d54
RG
860 struct mbuf *n, **np;
861 int off = off0;
984263bc
MD
862 struct mbuf *top;
863 int copyhdr = 0;
864
865 KASSERT(off >= 0, ("m_copym, negative off %d", off));
866 KASSERT(len >= 0, ("m_copym, negative len %d", len));
867 if (off == 0 && m->m_flags & M_PKTHDR)
868 copyhdr = 1;
869 while (off > 0) {
870 KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain"));
871 if (off < m->m_len)
872 break;
873 off -= m->m_len;
874 m = m->m_next;
875 }
876 np = &top;
877 top = 0;
878 while (len > 0) {
7b6f875f 879 if (m == NULL) {
984263bc
MD
880 KASSERT(len == M_COPYALL,
881 ("m_copym, length > size of mbuf chain"));
882 break;
883 }
7b6f875f 884 n = m_get(wait, m->m_type);
984263bc 885 *np = n;
7b6f875f 886 if (n == NULL)
984263bc
MD
887 goto nospace;
888 if (copyhdr) {
889 if (!m_dup_pkthdr(n, m, wait))
890 goto nospace;
891 if (len == M_COPYALL)
892 n->m_pkthdr.len -= off0;
893 else
894 n->m_pkthdr.len = len;
895 copyhdr = 0;
896 }
897 n->m_len = min(len, m->m_len - off);
898 if (m->m_flags & M_EXT) {
899 n->m_data = m->m_data + off;
7b6f875f 900 m->m_ext.ext_ref(m->m_ext.ext_arg);
984263bc 901 n->m_ext = m->m_ext;
b542cd49 902 n->m_flags |= m->m_flags & (M_EXT | M_EXT_CLUSTER);
7eccf245 903 } else {
984263bc
MD
904 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
905 (unsigned)n->m_len);
7eccf245 906 }
984263bc
MD
907 if (len != M_COPYALL)
908 len -= n->m_len;
909 off = 0;
910 m = m->m_next;
911 np = &n->m_next;
912 }
7b6f875f
JH
913 if (top == NULL)
914 mbstat.m_mcfail++;
984263bc
MD
915 return (top);
916nospace:
917 m_freem(top);
7b6f875f
JH
918 mbstat.m_mcfail++;
919 return (NULL);
984263bc
MD
920}
921
922/*
923 * Copy an entire packet, including header (which must be present).
924 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'.
925 * Note that the copy is read-only, because clusters are not copied,
926 * only their reference counts are incremented.
927 * Preserve alignment of the first mbuf so if the creator has left
928 * some room at the beginning (e.g. for inserting protocol headers)
929 * the copies also have the room available.
930 */
931struct mbuf *
8a3125c6 932m_copypacket(struct mbuf *m, int how)
984263bc
MD
933{
934 struct mbuf *top, *n, *o;
935
7b6f875f 936 n = m_get(how, m->m_type);
984263bc
MD
937 top = n;
938 if (!n)
939 goto nospace;
940
941 if (!m_dup_pkthdr(n, m, how))
942 goto nospace;
943 n->m_len = m->m_len;
944 if (m->m_flags & M_EXT) {
945 n->m_data = m->m_data;
7b6f875f 946 m->m_ext.ext_ref(m->m_ext.ext_arg);
984263bc 947 n->m_ext = m->m_ext;
b542cd49 948 n->m_flags |= m->m_flags & (M_EXT | M_EXT_CLUSTER);
984263bc
MD
949 } else {
950 n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat );
951 bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
952 }
953
954 m = m->m_next;
955 while (m) {
7b6f875f 956 o = m_get(how, m->m_type);
984263bc
MD
957 if (!o)
958 goto nospace;
959
960 n->m_next = o;
961 n = n->m_next;
962
963 n->m_len = m->m_len;
964 if (m->m_flags & M_EXT) {
965 n->m_data = m->m_data;
7b6f875f 966 m->m_ext.ext_ref(m->m_ext.ext_arg);
984263bc 967 n->m_ext = m->m_ext;
b542cd49 968 n->m_flags |= m->m_flags & (M_EXT | M_EXT_CLUSTER);
984263bc
MD
969 } else {
970 bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
971 }
972
973 m = m->m_next;
974 }
975 return top;
976nospace:
977 m_freem(top);
7b6f875f
JH
978 mbstat.m_mcfail++;
979 return (NULL);
984263bc
MD
980}
981
982/*
983 * Copy data from an mbuf chain starting "off" bytes from the beginning,
984 * continuing for "len" bytes, into the indicated buffer.
985 */
986void
8a3125c6 987m_copydata(const struct mbuf *m, int off, int len, caddr_t cp)
984263bc 988{
1fd87d54 989 unsigned count;
984263bc
MD
990
991 KASSERT(off >= 0, ("m_copydata, negative off %d", off));
992 KASSERT(len >= 0, ("m_copydata, negative len %d", len));
993 while (off > 0) {
994 KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain"));
995 if (off < m->m_len)
996 break;
997 off -= m->m_len;
998 m = m->m_next;
999 }
1000 while (len > 0) {
1001 KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain"));
1002 count = min(m->m_len - off, len);
1003 bcopy(mtod(m, caddr_t) + off, cp, count);
1004 len -= count;
1005 cp += count;
1006 off = 0;
1007 m = m->m_next;
1008 }
1009}
1010
1011/*
1012 * Copy a packet header mbuf chain into a completely new chain, including
1013 * copying any mbuf clusters. Use this instead of m_copypacket() when
1014 * you need a writable copy of an mbuf chain.
1015 */
1016struct mbuf *
8a3125c6 1017m_dup(struct mbuf *m, int how)
984263bc
MD
1018{
1019 struct mbuf **p, *top = NULL;
1020 int remain, moff, nsize;
1021
1022 /* Sanity check */
1023 if (m == NULL)
50503f0f 1024 return (NULL);
5e2195bf 1025 KASSERT((m->m_flags & M_PKTHDR) != 0, ("%s: !PKTHDR", __func__));
984263bc
MD
1026
1027 /* While there's more data, get a new mbuf, tack it on, and fill it */
1028 remain = m->m_pkthdr.len;
1029 moff = 0;
1030 p = &top;
1031 while (remain > 0 || top == NULL) { /* allow m->m_pkthdr.len == 0 */
1032 struct mbuf *n;
1033
1034 /* Get the next new mbuf */
50503f0f
JH
1035 n = m_getl(remain, how, m->m_type, top == NULL ? M_PKTHDR : 0,
1036 &nsize);
984263bc
MD
1037 if (n == NULL)
1038 goto nospace;
50503f0f 1039 if (top == NULL)
984263bc 1040 if (!m_dup_pkthdr(n, m, how))
50503f0f 1041 goto nospace0;
984263bc
MD
1042
1043 /* Link it into the new chain */
1044 *p = n;
1045 p = &n->m_next;
1046
1047 /* Copy data from original mbuf(s) into new mbuf */
50503f0f 1048 n->m_len = 0;
984263bc
MD
1049 while (n->m_len < nsize && m != NULL) {
1050 int chunk = min(nsize - n->m_len, m->m_len - moff);
1051
1052 bcopy(m->m_data + moff, n->m_data + n->m_len, chunk);
1053 moff += chunk;
1054 n->m_len += chunk;
1055 remain -= chunk;
1056 if (moff == m->m_len) {
1057 m = m->m_next;
1058 moff = 0;
1059 }
1060 }
1061
1062 /* Check correct total mbuf length */
1063 KASSERT((remain > 0 && m != NULL) || (remain == 0 && m == NULL),
50503f0f 1064 ("%s: bogus m_pkthdr.len", __func__));
984263bc
MD
1065 }
1066 return (top);
1067
1068nospace:
1069 m_freem(top);
50503f0f
JH
1070nospace0:
1071 mbstat.m_mcfail++;
1072 return (NULL);
984263bc
MD
1073}
1074
1075/*
1076 * Concatenate mbuf chain n to m.
1077 * Both chains must be of the same type (e.g. MT_DATA).
1078 * Any m_pkthdr is not updated.
1079 */
1080void
8a3125c6 1081m_cat(struct mbuf *m, struct mbuf *n)
984263bc 1082{
50503f0f 1083 m = m_last(m);
984263bc
MD
1084 while (n) {
1085 if (m->m_flags & M_EXT ||
1086 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
1087 /* just join the two chains */
1088 m->m_next = n;
1089 return;
1090 }
1091 /* splat the data from one into the other */
1092 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1093 (u_int)n->m_len);
1094 m->m_len += n->m_len;
1095 n = m_free(n);
1096 }
1097}
1098
1099void
8a3125c6 1100m_adj(struct mbuf *mp, int req_len)
984263bc 1101{
1fd87d54
RG
1102 int len = req_len;
1103 struct mbuf *m;
1104 int count;
984263bc
MD
1105
1106 if ((m = mp) == NULL)
1107 return;
1108 if (len >= 0) {
1109 /*
1110 * Trim from head.
1111 */
1112 while (m != NULL && len > 0) {
1113 if (m->m_len <= len) {
1114 len -= m->m_len;
1115 m->m_len = 0;
1116 m = m->m_next;
1117 } else {
1118 m->m_len -= len;
1119 m->m_data += len;
1120 len = 0;
1121 }
1122 }
1123 m = mp;
1124 if (mp->m_flags & M_PKTHDR)
1125 m->m_pkthdr.len -= (req_len - len);
1126 } else {
1127 /*
1128 * Trim from tail. Scan the mbuf chain,
1129 * calculating its length and finding the last mbuf.
1130 * If the adjustment only affects this mbuf, then just
1131 * adjust and return. Otherwise, rescan and truncate
1132 * after the remaining size.
1133 */
1134 len = -len;
1135 count = 0;
1136 for (;;) {
1137 count += m->m_len;
1138 if (m->m_next == (struct mbuf *)0)
1139 break;
1140 m = m->m_next;
1141 }
1142 if (m->m_len >= len) {
1143 m->m_len -= len;
1144 if (mp->m_flags & M_PKTHDR)
1145 mp->m_pkthdr.len -= len;
1146 return;
1147 }
1148 count -= len;
1149 if (count < 0)
1150 count = 0;
1151 /*
1152 * Correct length for chain is "count".
1153 * Find the mbuf with last data, adjust its length,
1154 * and toss data from remaining mbufs on chain.
1155 */
1156 m = mp;
1157 if (m->m_flags & M_PKTHDR)
1158 m->m_pkthdr.len = count;
1159 for (; m; m = m->m_next) {
1160 if (m->m_len >= count) {
1161 m->m_len = count;
1162 break;
1163 }
1164 count -= m->m_len;
1165 }
1166 while (m->m_next)
1167 (m = m->m_next) ->m_len = 0;
1168 }
1169}
1170
1171/*
7b6f875f 1172 * Rearrange an mbuf chain so that len bytes are contiguous
9e4465af
MD
1173 * and in the data area of an mbuf (so that mtod will work for a structure
1174 * of size len). Returns the resulting mbuf chain on success, frees it and
1175 * returns null on failure. If there is room, it will add up to
1176 * max_protohdr-len extra bytes to the contiguous region in an attempt to
1177 * avoid being called next time.
984263bc 1178 */
984263bc 1179struct mbuf *
8a3125c6 1180m_pullup(struct mbuf *n, int len)
984263bc 1181{
1fd87d54
RG
1182 struct mbuf *m;
1183 int count;
984263bc
MD
1184 int space;
1185
1186 /*
1187 * If first mbuf has no cluster, and has room for len bytes
1188 * without shifting current data, pullup into it,
1189 * otherwise allocate a new mbuf to prepend to the chain.
1190 */
7b6f875f
JH
1191 if (!(n->m_flags & M_EXT) &&
1192 n->m_data + len < &n->m_dat[MLEN] &&
1193 n->m_next) {
984263bc
MD
1194 if (n->m_len >= len)
1195 return (n);
1196 m = n;
1197 n = n->m_next;
1198 len -= m->m_len;
1199 } else {
1200 if (len > MHLEN)
1201 goto bad;
410ed080
JH
1202 m = m_getl(MLEN, MB_DONTWAIT, n->m_type, n->m_flags & M_PKTHDR,
1203 NULL);
7b6f875f 1204 if (m == NULL)
984263bc
MD
1205 goto bad;
1206 m->m_len = 0;
1207 if (n->m_flags & M_PKTHDR)
1208 M_MOVE_PKTHDR(m, n);
1209 }
1210 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
1211 do {
1212 count = min(min(max(len, max_protohdr), space), n->m_len);
1213 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1214 (unsigned)count);
1215 len -= count;
1216 m->m_len += count;
1217 n->m_len -= count;
1218 space -= count;
1219 if (n->m_len)
1220 n->m_data += count;
1221 else
1222 n = m_free(n);
1223 } while (len > 0 && n);
1224 if (len > 0) {
7b6f875f 1225 m_free(m);
984263bc
MD
1226 goto bad;
1227 }
1228 m->m_next = n;
1229 return (m);
1230bad:
1231 m_freem(n);
7b6f875f
JH
1232 mbstat.m_mpfail++;
1233 return (NULL);
984263bc
MD
1234}
1235
1236/*
1237 * Partition an mbuf chain in two pieces, returning the tail --
1238 * all but the first len0 bytes. In case of failure, it returns NULL and
1239 * attempts to restore the chain to its original state.
1240 *
1241 * Note that the resulting mbufs might be read-only, because the new
1242 * mbuf can end up sharing an mbuf cluster with the original mbuf if
1243 * the "breaking point" happens to lie within a cluster mbuf. Use the
1244 * M_WRITABLE() macro to check for this case.
1245 */
1246struct mbuf *
8a3125c6 1247m_split(struct mbuf *m0, int len0, int wait)
984263bc 1248{
1fd87d54 1249 struct mbuf *m, *n;
984263bc
MD
1250 unsigned len = len0, remain;
1251
1252 for (m = m0; m && len > m->m_len; m = m->m_next)
1253 len -= m->m_len;
7b6f875f
JH
1254 if (m == NULL)
1255 return (NULL);
984263bc
MD
1256 remain = m->m_len - len;
1257 if (m0->m_flags & M_PKTHDR) {
7b6f875f
JH
1258 n = m_gethdr(wait, m0->m_type);
1259 if (n == NULL)
1260 return (NULL);
984263bc
MD
1261 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1262 n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1263 m0->m_pkthdr.len = len0;
1264 if (m->m_flags & M_EXT)
1265 goto extpacket;
1266 if (remain > MHLEN) {
1267 /* m can't be the lead packet */
1268 MH_ALIGN(n, 0);
1269 n->m_next = m_split(m, len, wait);
7b6f875f
JH
1270 if (n->m_next == NULL) {
1271 m_free(n);
1272 return (NULL);
984263bc
MD
1273 } else {
1274 n->m_len = 0;
1275 return (n);
1276 }
1277 } else
1278 MH_ALIGN(n, remain);
1279 } else if (remain == 0) {
1280 n = m->m_next;
1281 m->m_next = 0;
1282 return (n);
1283 } else {
7b6f875f
JH
1284 n = m_get(wait, m->m_type);
1285 if (n == NULL)
1286 return (NULL);
984263bc
MD
1287 M_ALIGN(n, remain);
1288 }
1289extpacket:
1290 if (m->m_flags & M_EXT) {
984263bc 1291 n->m_data = m->m_data + len;
7b6f875f 1292 m->m_ext.ext_ref(m->m_ext.ext_arg);
7eccf245 1293 n->m_ext = m->m_ext;
b542cd49 1294 n->m_flags |= m->m_flags & (M_EXT | M_EXT_CLUSTER);
984263bc
MD
1295 } else {
1296 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
1297 }
1298 n->m_len = remain;
1299 m->m_len = len;
1300 n->m_next = m->m_next;
1301 m->m_next = 0;
1302 return (n);
1303}
50503f0f 1304
984263bc
MD
1305/*
1306 * Routine to copy from device local memory into mbufs.
50503f0f 1307 * Note: "offset" is ill-defined and always called as 0, so ignore it.
984263bc
MD
1308 */
1309struct mbuf *
50503f0f
JH
1310m_devget(char *buf, int len, int offset, struct ifnet *ifp,
1311 void (*copy)(volatile const void *from, volatile void *to, size_t length))
984263bc 1312{
50503f0f
JH
1313 struct mbuf *m, *mfirst = NULL, **mtail;
1314 int nsize, flags;
1315
1316 if (copy == NULL)
1317 copy = bcopy;
1318 mtail = &mfirst;
1319 flags = M_PKTHDR;
1320
1321 while (len > 0) {
1322 m = m_getl(len, MB_DONTWAIT, MT_DATA, flags, &nsize);
1323 if (m == NULL) {
1324 m_freem(mfirst);
1325 return (NULL);
984263bc 1326 }
50503f0f
JH
1327 m->m_len = min(len, nsize);
1328
1329 if (flags & M_PKTHDR) {
1330 if (len + max_linkhdr <= nsize)
1331 m->m_data += max_linkhdr;
1332 m->m_pkthdr.rcvif = ifp;
1333 m->m_pkthdr.len = len;
1334 flags = 0;
984263bc 1335 }
50503f0f
JH
1336
1337 copy(buf, m->m_data, (unsigned)m->m_len);
1338 buf += m->m_len;
1339 len -= m->m_len;
1340 *mtail = m;
1341 mtail = &m->m_next;
984263bc 1342 }
50503f0f
JH
1343
1344 return (mfirst);
984263bc
MD
1345}
1346
1347/*
1348 * Copy data from a buffer back into the indicated mbuf chain,
1349 * starting "off" bytes from the beginning, extending the mbuf
1350 * chain if necessary.
1351 */
1352void
8a3125c6 1353m_copyback(struct mbuf *m0, int off, int len, caddr_t cp)
984263bc 1354{
1fd87d54
RG
1355 int mlen;
1356 struct mbuf *m = m0, *n;
984263bc
MD
1357 int totlen = 0;
1358
7b6f875f 1359 if (m0 == NULL)
984263bc
MD
1360 return;
1361 while (off > (mlen = m->m_len)) {
1362 off -= mlen;
1363 totlen += mlen;
7b6f875f 1364 if (m->m_next == NULL) {
74f1caca 1365 n = m_getclr(MB_DONTWAIT, m->m_type);
7b6f875f 1366 if (n == NULL)
984263bc
MD
1367 goto out;
1368 n->m_len = min(MLEN, len + off);
1369 m->m_next = n;
1370 }
1371 m = m->m_next;
1372 }
1373 while (len > 0) {
1374 mlen = min (m->m_len - off, len);
1375 bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen);
1376 cp += mlen;
1377 len -= mlen;
1378 mlen += off;
1379 off = 0;
1380 totlen += mlen;
1381 if (len == 0)
1382 break;
7b6f875f 1383 if (m->m_next == NULL) {
74f1caca 1384 n = m_get(MB_DONTWAIT, m->m_type);
7b6f875f 1385 if (n == NULL)
984263bc
MD
1386 break;
1387 n->m_len = min(MLEN, len);
1388 m->m_next = n;
1389 }
1390 m = m->m_next;
1391 }
1392out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
1393 m->m_pkthdr.len = totlen;
1394}
1395
1396void
1397m_print(const struct mbuf *m)
1398{
1399 int len;
1400 const struct mbuf *m2;
1401
1402 len = m->m_pkthdr.len;
1403 m2 = m;
1404 while (len) {
1405 printf("%p %*D\n", m2, m2->m_len, (u_char *)m2->m_data, "-");
1406 len -= m2->m_len;
1407 m2 = m2->m_next;
1408 }
1409 return;
1410}
1411
1412/*
1413 * "Move" mbuf pkthdr from "from" to "to".
1414 * "from" must have M_PKTHDR set, and "to" must be empty.
1415 */
1416void
1417m_move_pkthdr(struct mbuf *to, struct mbuf *from)
1418{
e0d05288 1419 KASSERT((to->m_flags & M_PKTHDR), ("m_move_pkthdr: not packet header"));
984263bc 1420
77e294a1 1421 to->m_flags |= from->m_flags & M_COPYFLAGS;
984263bc
MD
1422 to->m_pkthdr = from->m_pkthdr; /* especially tags */
1423 SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */
1424 from->m_flags &= ~M_PKTHDR;
1425}
1426
1427/*
1428 * Duplicate "from"'s mbuf pkthdr in "to".
1429 * "from" must have M_PKTHDR set, and "to" must be empty.
1430 * In particular, this does a deep copy of the packet tags.
1431 */
1432int
f15db79e 1433m_dup_pkthdr(struct mbuf *to, const struct mbuf *from, int how)
984263bc
MD
1434{
1435 to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
7b6f875f 1436 if (!(to->m_flags & M_EXT))
984263bc
MD
1437 to->m_data = to->m_pktdat;
1438 to->m_pkthdr = from->m_pkthdr;
1439 SLIST_INIT(&to->m_pkthdr.tags);
1440 return (m_tag_copy_chain(to, from, how));
1441}
1442
1443/*
1444 * Defragment a mbuf chain, returning the shortest possible
1445 * chain of mbufs and clusters. If allocation fails and
1446 * this cannot be completed, NULL will be returned, but
1447 * the passed in chain will be unchanged. Upon success,
1448 * the original chain will be freed, and the new chain
1449 * will be returned.
1450 *
1451 * If a non-packet header is passed in, the original
1452 * mbuf (chain?) will be returned unharmed.
c8f5127a
JS
1453 *
1454 * m_defrag_nofree doesn't free the passed in mbuf.
984263bc
MD
1455 */
1456struct mbuf *
1457m_defrag(struct mbuf *m0, int how)
c8f5127a
JS
1458{
1459 struct mbuf *m_new;
1460
1461 if ((m_new = m_defrag_nofree(m0, how)) == NULL)
1462 return (NULL);
1463 if (m_new != m0)
1464 m_freem(m0);
1465 return (m_new);
1466}
1467
1468struct mbuf *
1469m_defrag_nofree(struct mbuf *m0, int how)
984263bc
MD
1470{
1471 struct mbuf *m_new = NULL, *m_final = NULL;
61721e90 1472 int progress = 0, length, nsize;
984263bc
MD
1473
1474 if (!(m0->m_flags & M_PKTHDR))
1475 return (m0);
1476
1477#ifdef MBUF_STRESS_TEST
1478 if (m_defragrandomfailures) {
1479 int temp = arc4random() & 0xff;
1480 if (temp == 0xba)
1481 goto nospace;
1482 }
1483#endif
1484
61721e90 1485 m_final = m_getl(m0->m_pkthdr.len, how, MT_DATA, M_PKTHDR, &nsize);
984263bc
MD
1486 if (m_final == NULL)
1487 goto nospace;
61721e90 1488 m_final->m_len = 0; /* in case m0->m_pkthdr.len is zero */
984263bc
MD
1489
1490 if (m_dup_pkthdr(m_final, m0, how) == NULL)
1491 goto nospace;
1492
1493 m_new = m_final;
1494
1495 while (progress < m0->m_pkthdr.len) {
1496 length = m0->m_pkthdr.len - progress;
1497 if (length > MCLBYTES)
1498 length = MCLBYTES;
1499
1500 if (m_new == NULL) {
61721e90 1501 m_new = m_getl(length, how, MT_DATA, 0, &nsize);
984263bc
MD
1502 if (m_new == NULL)
1503 goto nospace;
1504 }
1505
1506 m_copydata(m0, progress, length, mtod(m_new, caddr_t));
1507 progress += length;
1508 m_new->m_len = length;
1509 if (m_new != m_final)
1510 m_cat(m_final, m_new);
1511 m_new = NULL;
1512 }
1513 if (m0->m_next == NULL)
1514 m_defraguseless++;
984263bc 1515 m_defragpackets++;
c8f5127a
JS
1516 m_defragbytes += m_final->m_pkthdr.len;
1517 return (m_final);
984263bc
MD
1518nospace:
1519 m_defragfailure++;
1520 if (m_new)
1521 m_free(m_new);
61721e90 1522 m_freem(m_final);
984263bc
MD
1523 return (NULL);
1524}
0c33f36d
JH
1525
1526/*
1527 * Move data from uio into mbufs.
0c33f36d
JH
1528 */
1529struct mbuf *
e12241e1 1530m_uiomove(struct uio *uio)
0c33f36d 1531{
0c33f36d 1532 struct mbuf *m; /* current working mbuf */
e12241e1
JH
1533 struct mbuf *head = NULL; /* result mbuf chain */
1534 struct mbuf **mp = &head;
1535 int resid = uio->uio_resid, nsize, flags = M_PKTHDR, error;
0c33f36d 1536
0c33f36d 1537 do {
e12241e1 1538 m = m_getl(resid, MB_WAIT, MT_DATA, flags, &nsize);
61721e90
JH
1539 if (flags) {
1540 m->m_pkthdr.len = 0;
1541 /* Leave room for protocol headers. */
1542 if (resid < MHLEN)
1543 MH_ALIGN(m, resid);
1544 flags = 0;
0c33f36d 1545 }
61721e90
JH
1546 m->m_len = min(nsize, resid);
1547 error = uiomove(mtod(m, caddr_t), m->m_len, uio);
0c33f36d
JH
1548 if (error) {
1549 m_free(m);
1550 goto failed;
1551 }
0c33f36d
JH
1552 *mp = m;
1553 mp = &m->m_next;
61721e90
JH
1554 head->m_pkthdr.len += m->m_len;
1555 resid -= m->m_len;
0c33f36d
JH
1556 } while (resid > 0);
1557
1558 return (head);
1559
1560failed:
61721e90 1561 m_freem(head);
0c33f36d
JH
1562 return (NULL);
1563}
df80f2ea 1564
50503f0f
JH
1565struct mbuf *
1566m_last(struct mbuf *m)
1567{
1568 while (m->m_next)
1569 m = m->m_next;
1570 return (m);
1571}
1572
df80f2ea
JH
1573/*
1574 * Return the number of bytes in an mbuf chain.
1575 * If lastm is not NULL, also return the last mbuf.
1576 */
1577u_int
1578m_lengthm(struct mbuf *m, struct mbuf **lastm)
1579{
1580 u_int len = 0;
1581 struct mbuf *prev = m;
1582
1583 while (m) {
1584 len += m->m_len;
1585 prev = m;
1586 m = m->m_next;
1587 }
1588 if (lastm != NULL)
1589 *lastm = prev;
1590 return (len);
1591}
1592
1593/*
1594 * Like m_lengthm(), except also keep track of mbuf usage.
1595 */
1596u_int
1597m_countm(struct mbuf *m, struct mbuf **lastm, u_int *pmbcnt)
1598{
1599 u_int len = 0, mbcnt = 0;
1600 struct mbuf *prev = m;
1601
1602 while (m) {
1603 len += m->m_len;
1604 mbcnt += MSIZE;
1605 if (m->m_flags & M_EXT)
1606 mbcnt += m->m_ext.ext_size;
1607 prev = m;
1608 m = m->m_next;
1609 }
1610 if (lastm != NULL)
1611 *lastm = prev;
1612 *pmbcnt = mbcnt;
1613 return (len);
1614}