2 * Copyright (c) 2005 Jeffrey M. Hsu. All rights reserved.
3 * Copyright (c) 1982, 1986, 1988, 1990, 1993
4 * The Regents of the University of California. All rights reserved.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 3. Neither the name of the University nor the names of its contributors
15 * may be used to endorse or promote products derived from this software
16 * without specific prior written permission.
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * @(#)uipc_socket2.c 8.1 (Berkeley) 6/10/93
31 * $FreeBSD: src/sys/kern/uipc_socket2.c,v 1.55.2.17 2002/08/31 19:04:55 dwmalone Exp $
32 * $DragonFly: src/sys/kern/uipc_sockbuf.c,v 1.3 2007/08/09 01:10:04 dillon Exp $
35 #include "opt_param.h"
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/domain.h>
39 #include <sys/file.h> /* for maxfiles */
40 #include <sys/kernel.h>
42 #include <sys/malloc.h>
44 #include <sys/protosw.h>
45 #include <sys/resourcevar.h>
47 #include <sys/socket.h>
48 #include <sys/socketvar.h>
50 #include <sys/thread2.h>
51 #include <sys/msgport2.h>
54 * Routines to add and remove data from an mbuf queue.
56 * The routines sbappend() or sbappendrecord() are normally called to
57 * append new mbufs to a socket buffer. sbappendrecord() differs from
58 * sbappend() in that data supplied is treated as the beginning of a new
59 * record. sbappend() only begins a new record if the last mbuf in the
60 * sockbuf is marked M_EOR.
62 * To place a sender's address, optional access rights, and data in a
63 * socket receive buffer, sbappendaddr() or sbappendcontrol() should be
64 * used. These functions also begin a new record.
66 * Reliable protocols may use the socket send buffer to hold data
67 * awaiting acknowledgement. Data is normally copied from a socket
68 * send buffer in a protocol with m_copy for output to a peer,
69 * and then removing the data from the socket buffer with sbdrop()
70 * or sbdroprecord() when the data is acknowledged by the peer.
74 * Append mbuf chain m to the last record in the socket buffer sb.
75 * The additional space associated the mbuf chain is recorded in sb.
76 * Empty mbufs are discarded and mbufs are compacted where possible.
78 * If M_EOR is set in the first or last mbuf of the last record, the
79 * mbuf chain is appended as a new record. M_EOR is usually just set
80 * in the last mbuf of the last record's mbuf chain (see sbcompress()),
81 * but this may be changed in the future since there is no real need
82 * to propogate the flag any more.
85 sbappend(struct sockbuf *sb, struct mbuf *m)
92 n = sb->sb_lastrecord;
94 if (n->m_flags & M_EOR) {
95 sbappendrecord(sb, m);
101 if (n->m_flags & M_EOR) {
102 sbappendrecord(sb, m);
106 sbcompress(sb, m, n);
111 * sbappendstream() is an optimized form of sbappend() for protocols
112 * such as TCP that only have one record in the socket buffer, are
113 * not PR_ATOMIC, nor allow MT_CONTROL data. A protocol that uses
114 * sbappendstream() must use sbappendstream() exclusively.
117 sbappendstream(struct sockbuf *sb, struct mbuf *m)
120 KKASSERT(m->m_nextpkt == NULL);
121 sbcompress(sb, m, sb->sb_lastmbuf);
127 _sbcheck(struct sockbuf *sb)
130 struct mbuf *n = NULL;
131 u_long len = 0, mbcnt = 0;
133 for (m = sb->sb_mb; m; m = n) {
135 if (n == NULL && sb->sb_lastrecord != m) {
136 kprintf("sockbuf %p mismatched lastrecord %p vs %p\n", sb, sb->sb_lastrecord, m);
140 for (; m; m = m->m_next) {
143 if (m->m_flags & M_EXT) /*XXX*/ /* pretty sure this is bogus */
144 mbcnt += m->m_ext.ext_size;
145 if (n == NULL && m->m_next == NULL) {
146 if (sb->sb_lastmbuf != m) {
147 kprintf("sockbuf %p mismatched lastmbuf %p vs %p\n", sb, sb->sb_lastmbuf, m);
153 if (sb->sb_mb == NULL) {
154 if (sb->sb_lastrecord != NULL) {
155 kprintf("sockbuf %p is empty, lastrecord not NULL: %p\n",
156 sb, sb->sb_lastrecord);
159 if (sb->sb_lastmbuf != NULL) {
160 kprintf("sockbuf %p is empty, lastmbuf not NULL: %p\n",
161 sb, sb->sb_lastmbuf);
165 if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) {
166 kprintf("sockbuf %p cc %ld != %ld || mbcnt %ld != %ld\n",
167 sb, len, sb->sb_cc, mbcnt, sb->sb_mbcnt);
175 * Same as sbappend(), except the mbuf chain begins a new record.
178 sbappendrecord(struct sockbuf *sb, struct mbuf *m0)
180 struct mbuf *firstmbuf;
181 struct mbuf *secondmbuf;
190 * Break the first mbuf off from the rest of the mbuf chain.
193 secondmbuf = m0->m_next;
197 * Insert the first mbuf of the m0 mbuf chain as the last record of
198 * the sockbuf. Note this permits zero length records! Keep the
199 * sockbuf state consistent.
201 if (sb->sb_mb == NULL)
202 sb->sb_mb = firstmbuf;
204 sb->sb_lastrecord->m_nextpkt = firstmbuf;
205 sb->sb_lastrecord = firstmbuf; /* update hint for new last record */
206 sb->sb_lastmbuf = firstmbuf; /* update hint for new last mbuf */
209 * propagate the EOR flag so sbcompress() can pick it up
211 if ((firstmbuf->m_flags & M_EOR) && (secondmbuf != NULL)) {
212 firstmbuf->m_flags &= ~M_EOR;
213 secondmbuf->m_flags |= M_EOR;
217 * The succeeding call to sbcompress() omits accounting for
218 * the first mbuf, so do it here.
220 sballoc(sb, firstmbuf);
222 /* Compact the rest of the mbuf chain in after the first mbuf. */
223 sbcompress(sb, secondmbuf, firstmbuf);
227 * Append address and data, and optionally, control (ancillary) data
228 * to the receive queue of a socket. If present,
229 * m0 must include a packet header with total length.
230 * Returns 0 if insufficient mbufs.
233 sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa, struct mbuf *m0,
234 struct mbuf *control)
240 mbuftrackid(control, 20);
241 if (m0 && (m0->m_flags & M_PKTHDR) == 0)
242 panic("sbappendaddr");
245 for (n = control; n; n = n->m_next) {
246 if (n->m_next == NULL) /* keep pointer to last control buf */
249 if (asa->sa_len > MLEN)
251 MGET(m, MB_DONTWAIT, MT_SONAME);
254 KKASSERT(m->m_nextpkt == NULL);
255 m->m_len = asa->sa_len;
256 bcopy(asa, mtod(m, caddr_t), asa->sa_len);
258 n->m_next = m0; /* concatenate data to control */
262 for (n = m; n; n = n->m_next)
265 if (sb->sb_mb == NULL)
268 sb->sb_lastrecord->m_nextpkt = m;
269 sb->sb_lastrecord = m;
272 * Propogate M_EOR to the last mbuf and calculate sb_lastmbuf
273 * so sbappend() can find it.
277 m->m_flags &= ~M_EOR;
281 m->m_flags |= eor & M_EOR;
288 * Append control information followed by data. Both the control and data
292 sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control)
295 u_int length, cmbcnt, m0mbcnt;
298 KASSERT(control != NULL, ("sbappendcontrol"));
299 KKASSERT(control->m_nextpkt == NULL);
303 mbuftrackid(control, 22);
305 length = m_countm(control, &n, &cmbcnt) + m_countm(m0, NULL, &m0mbcnt);
307 KKASSERT(m0 != NULL);
309 n->m_next = m0; /* concatenate data to control */
311 if (sb->sb_mb == NULL)
314 sb->sb_lastrecord->m_nextpkt = control;
315 sb->sb_lastrecord = control;
318 * Propogate M_EOR to the last mbuf and calculate sb_lastmbuf
319 * so sbappend() can find it.
323 m0->m_flags &= ~M_EOR;
327 m0->m_flags |= eor & M_EOR;
328 sb->sb_lastmbuf = m0;
331 sb->sb_mbcnt += cmbcnt + m0mbcnt;
337 * Compress mbuf chain m into the socket buffer sb following mbuf tailm.
338 * If tailm is null, the buffer is presumed empty. Also, as a side-effect,
339 * increment the sockbuf counts for each mbuf in the chain.
342 sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *tailm)
345 struct mbuf *free_chain = NULL;
353 eor |= m->m_flags & M_EOR;
355 * Disregard empty mbufs as long as we don't encounter
356 * an end-of-record or there is a trailing mbuf of
357 * the same type to propagate the EOR flag to.
359 * Defer the m_free() call because it can block and break
360 * the atomicy of the sockbuf.
364 (((o = m->m_next) || (o = tailm)) &&
365 o->m_type == m->m_type))) {
367 m->m_next = free_chain;
373 /* See if we can coalesce with preceding mbuf. */
374 if (tailm && !(tailm->m_flags & M_EOR) && M_WRITABLE(tailm) &&
375 m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */
376 m->m_len <= M_TRAILINGSPACE(tailm) &&
377 tailm->m_type == m->m_type) {
380 bcopy(mtod(m, caddr_t),
381 mtod(tailm, caddr_t) + tailm->m_len,
383 tailm->m_len += m->m_len;
385 sb->sb_cc += m->m_len; /* update sb counter */
388 * Fix the wrongly updated mbcnt_prealloc
391 if (m->m_flags & M_EXT)
392 mbcnt_sz += m->m_ext.ext_size;
393 atomic_subtract_long(&sb->sb_mbcnt_prealloc, mbcnt_sz);
396 m->m_next = free_chain;
402 /* Insert whole mbuf. */
404 KASSERT(sb->sb_mb == NULL,
405 ("sbcompress: sb_mb not NULL"));
406 sb->sb_mb = m; /* only mbuf in sockbuf */
407 sb->sb_lastrecord = m; /* new last record */
409 tailm->m_next = m; /* tack m on following tailm */
411 sb->sb_lastmbuf = m; /* update last mbuf hint */
413 tailm = m; /* just inserted mbuf becomes the new tail */
414 m = m->m_next; /* advance to next mbuf */
415 tailm->m_next = NULL; /* split inserted mbuf off from chain */
417 /* update sb counters for just added mbuf */
420 /* clear EOR on intermediate mbufs */
421 tailm->m_flags &= ~M_EOR;
425 * Propogate EOR to the last mbuf
429 tailm->m_flags |= eor;
431 kprintf("semi-panic: sbcompress");
435 * Clean up any defered frees.
438 free_chain = m_free(free_chain);
444 * Free all mbufs in a sockbuf.
445 * Check that all resources are reclaimed.
448 sbflush(struct sockbuf *sb)
450 while (sb->sb_mbcnt) {
452 * Don't call sbdrop(sb, 0) if the leading mbuf is non-empty:
453 * we would loop forever. Panic instead.
455 if (!sb->sb_cc && (sb->sb_mb == NULL || sb->sb_mb->m_len))
457 sbdrop(sb, (int)sb->sb_cc);
459 KASSERT(!(sb->sb_cc || sb->sb_mb || sb->sb_mbcnt || sb->sb_lastmbuf),
460 ("sbflush: cc %ld || mb %p || mbcnt %ld || lastmbuf %p",
461 sb->sb_cc, sb->sb_mb, sb->sb_mbcnt, sb->sb_lastmbuf));
465 * Drop data from (the front of) a sockbuf. If the current record is
466 * exhausted this routine will move onto the next one and continue dropping
470 sbdrop(struct sockbuf *sb, int len)
473 struct mbuf *free_chain = NULL;
479 while (m && len > 0) {
480 if (m->m_len > len) {
484 atomic_subtract_long(&sb->sb_cc_prealloc, len);
488 m = sbunlinkmbuf(sb, m, &free_chain);
489 if (m == NULL && len)
494 * Remove any trailing 0-length mbufs in the current record. If
495 * the last record for which data was removed is now empty, m will be
498 while (m && m->m_len == 0) {
499 m = sbunlinkmbuf(sb, m, &free_chain);
508 * Drop a record off the front of a sockbuf and move the next record
511 * Must be called while holding a critical section.
514 sbdroprecord(struct sockbuf *sb)
522 if ((sb->sb_mb = m->m_nextpkt) == NULL) {
523 sb->sb_lastrecord = NULL;
524 sb->sb_lastmbuf = NULL;
527 for (n = m; n; n = n->m_next)
535 * Drop the first mbuf off the sockbuf and move the next mbuf to the front.
536 * Currently only the head mbuf of the sockbuf may be dropped this way.
538 * The next mbuf in the same record as the mbuf being removed is returned
539 * or NULL if the record is exhausted. Note that other records may remain
540 * in the sockbuf when NULL is returned.
542 * Must be called while holding a critical section.
545 sbunlinkmbuf(struct sockbuf *sb, struct mbuf *m, struct mbuf **free_chain)
549 KKASSERT(sb->sb_mb == m);
554 if (sb->sb_lastrecord == m)
555 sb->sb_lastrecord = n;
556 KKASSERT(sb->sb_lastmbuf != m);
557 n->m_nextpkt = m->m_nextpkt;
559 sb->sb_mb = m->m_nextpkt;
560 if (sb->sb_lastrecord == m) {
561 KKASSERT(sb->sb_mb == NULL);
562 sb->sb_lastrecord = NULL;
564 if (sb->sb_mb == NULL)
565 sb->sb_lastmbuf = NULL;
569 m->m_next = *free_chain;
578 * Create a "control" mbuf containing the specified data
579 * with the specified type for presentation on a socket buffer.
582 sbcreatecontrol(caddr_t p, int size, int type, int level)
587 if (CMSG_SPACE((u_int)size) > MCLBYTES)
589 m = m_getl(CMSG_SPACE((u_int)size), MB_DONTWAIT, MT_CONTROL, 0, NULL);
592 m->m_len = CMSG_SPACE(size);
593 cp = mtod(m, struct cmsghdr *);
595 memcpy(CMSG_DATA(cp), p, size);
596 cp->cmsg_len = CMSG_LEN(size);
597 cp->cmsg_level = level;
598 cp->cmsg_type = type;