sosendudp: Try to optimize out the additional mbuf alloc on output path

This optimization leaves enough space at the beginning of the mbuf, so
later on M_PREPEND() probably will not allocate addition mbuf.

This probably will not benefit any data that will be fragmented, e.g. by
IPv4, so this optimization is only performed when the size of data and
max size of protocol+link headers fit into one mbuf cluster.

This optimization could be turned off by net.inet.udp.sosend_prepend,
which is on by default.

diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
index 847db40..629afa0 100644 (file)
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -102,6 +102,7 @@
 extern int tcp_sosend_agglim;
 extern int tcp_sosend_async;
 extern int udp_sosend_async;
+extern int udp_sosend_prepend;
 
 #ifdef INET
 static int      do_setopt_accept_filter(struct socket *so, struct sockopt *sopt);
@@ -941,9 +942,44 @@ restart:
        }
 
        if (uio) {
-               top = m_uiomove(uio);
-               if (top == NULL)
-                       goto release;
+               int hdrlen = max_hdr;
+
+               /*
+                * We try to optimize out the additional mbuf
+                * allocations in M_PREPEND() on output path, e.g.
+                * - udp_output(), when it tries to prepend protocol
+                *   headers.
+                * - Link layer output function, when it tries to
+                *   prepend link layer header.
+                *
+                * This probably will not benefit any data that will
+                * be fragmented, so this optimization is only performed
+                * when the size of data and max size of protocol+link
+                * headers fit into one mbuf cluster.
+                */
+               if (uio->uio_resid > MCLBYTES - hdrlen ||
+                   !udp_sosend_prepend) {
+                       top = m_uiomove(uio);
+                       if (top == NULL)
+                               goto release;
+               } else {
+                       int nsize;
+
+                       top = m_getl(uio->uio_resid + hdrlen, MB_WAIT,
+                           MT_DATA, M_PKTHDR, &nsize);
+                       KASSERT(nsize >= uio->uio_resid + hdrlen,
+                           ("sosendudp invalid nsize %d, "
+                            "resid %zu, hdrlen %d",
+                            nsize, uio->uio_resid, hdrlen));
+
+                       top->m_len = uio->uio_resid;
+                       top->m_pkthdr.len = uio->uio_resid;
+                       top->m_data += hdrlen;
+
+                       error = uiomove(mtod(top, caddr_t), top->m_len, uio);
+                       if (error)
+                               goto out;
+               }
        }
 
        if (flags & MSG_DONTROUTE)

diff --git a/sys/netinet/udp_usrreq.c b/sys/netinet/udp_usrreq.c
index 4ad44c2..96551c9 100644 (file)
--- a/sys/netinet/udp_usrreq.c
+++ b/sys/netinet/udp_usrreq.c
@@ -148,6 +148,11 @@ int        udp_sosend_async = 1;
 SYSCTL_INT(_net_inet_udp, OID_AUTO, sosend_async, CTLFLAG_RW,
        &udp_sosend_async, 0, "UDP asynchronized pru_send");
 
+int    udp_sosend_prepend = 1;
+SYSCTL_INT(_net_inet_udp, OID_AUTO, sosend_prepend, CTLFLAG_RW,
+       &udp_sosend_prepend, 0,
+       "Prepend enough space for proto and link header in pru_send");
+
 struct inpcbinfo udbinfo;
 
 static struct netisr_barrier *udbinfo_br;