sockbuf: Pre-alloc sockbuf space, so asynchronized pru_send could be used.

When doing asynchronized pru_send, it is quite possible that we could
break the send sockbuf's upper limit, if we only use ssb_space() to
calculate the remaining sockbuf space.

Now, we add another two counters into sockbuf, when we send a mbuf
chain to the netisr, we update the these two counters to reflect how
much mbufs will be pending on the send sockbuf.  When calculating the
remaining space of the send sockbuf we also take these two counters
into consideration, so we will not break the upper limit of the send
sockbuf.  These two counter will also be updated when the mbufs in
the sockbuf is to be dropped/freed.

Reenable the asynchronized pru_send in sendfile and sosendtcp

diff --git a/sys/kern/uipc_sockbuf.c b/sys/kern/uipc_sockbuf.c
index 4908728..7a6a85b 100644 (file)
--- a/sys/kern/uipc_sockbuf.c
+++ b/sys/kern/uipc_sockbuf.c
@@ -379,11 +379,23 @@ sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *tailm)
                    m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */
                    m->m_len <= M_TRAILINGSPACE(tailm) &&
                    tailm->m_type == m->m_type) {
+                       u_long mbcnt_sz;
+
                        bcopy(mtod(m, caddr_t),
                              mtod(tailm, caddr_t) + tailm->m_len,
                              (unsigned)m->m_len);
                        tailm->m_len += m->m_len;
+
                        sb->sb_cc += m->m_len;          /* update sb counter */
+
+                       /*
+                        * Fix the wrongly updated mbcnt_prealloc
+                        */
+                       mbcnt_sz = MSIZE;
+                       if (m->m_flags & M_EXT)
+                               mbcnt_sz += m->m_ext.ext_size;
+                       atomic_subtract_long(&sb->sb_mbcnt_prealloc, mbcnt_sz);
+
                        o = m->m_next;
                        m->m_next = free_chain;
                        free_chain = m;
@@ -473,6 +485,7 @@ sbdrop(struct sockbuf *sb, int len)
                        m->m_len -= len;
                        m->m_data += len;
                        sb->sb_cc -= len;
+                       atomic_subtract_long(&sb->sb_cc_prealloc, len);
                        break;
                }
                len -= m->m_len;

diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
index 6e96525..79e5c6d 100644 (file)
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -142,7 +142,7 @@ int use_soaccept_pred_fast = 1;
 SYSCTL_INT(_kern_ipc, OID_AUTO, soaccept_pred_fast, CTLFLAG_RW,
     &use_soaccept_pred_fast, 0, "Fast socket accept predication");
 
-int use_sendfile_async = 0;
+int use_sendfile_async = 1;
 SYSCTL_INT(_kern_ipc, OID_AUTO, sendfile_async, CTLFLAG_RW,
     &use_sendfile_async, 0, "sendfile uses asynchronized pru_send");
 
@@ -1033,7 +1033,7 @@ restart:
                if (allatonce && resid > so->so_snd.ssb_hiwat)
                        gotoerr(EMSGSIZE);
 
-               space = ssb_space(&so->so_snd);
+               space = ssb_space_prealloc(&so->so_snd);
                if (flags & MSG_OOB)
                        space += 1024;
                if ((space < 0 || (size_t)space < resid) && !allatonce &&
@@ -1107,6 +1107,8 @@ restart:
                     * here, but there are probably other places that this
                     * also happens.  We must rethink this.
                     */
+                   for (m = top; m; m = m->m_next)
+                           ssb_preallocstream(&so->so_snd, m);
                    if (!async) {
                            error = so_pru_send(so, pru_flags, top,
                                NULL, NULL, td);

diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c
index f02d404..76dcf5c 100644 (file)
--- a/sys/kern/uipc_syscalls.c
+++ b/sys/kern/uipc_syscalls.c
@@ -1552,7 +1552,7 @@ kern_sendfile(struct vnode *vp, int sfd, off_t offset, size_t nbytes,
        struct vm_object *obj;
        struct socket *so;
        struct file *fp;
-       struct mbuf *m;
+       struct mbuf *m, *mp;
        struct sf_buf *sf;
        struct vm_page *pg;
        off_t off, xfsize;
@@ -1620,7 +1620,8 @@ retry_lookup:
                 * Optimize the non-blocking case by looking at the socket space
                 * before going to the extra work of constituting the sf_buf.
                 */
-               if ((fp->f_flag & FNONBLOCK) && ssb_space(&so->so_snd) <= 0) {
+               if ((fp->f_flag & FNONBLOCK) &&
+                   ssb_space_prealloc(&so->so_snd) <= 0) {
                        if (so->so_state & SS_CANTSENDMORE)
                                error = EPIPE;
                        else
@@ -1780,7 +1781,7 @@ retry_space:
                 * after checking the connection state above in order to avoid
                 * a race condition with ssb_wait().
                 */
-               if (ssb_space(&so->so_snd) < so->so_snd.ssb_lowat) {
+               if (ssb_space_prealloc(&so->so_snd) < so->so_snd.ssb_lowat) {
                        if (fp->f_flag & FNONBLOCK) {
                                m_freem(m);
                                ssb_unlock(&so->so_snd);
@@ -1802,10 +1803,14 @@ retry_space:
                        }
                        goto retry_space;
                }
+
+               for (mp = m; mp != NULL; mp = mp->m_next)
+                       ssb_preallocstream(&so->so_snd, mp);
                if (use_sendfile_async)
                        error = so_pru_senda(so, 0, m, NULL, NULL, td);
                else
                        error = so_pru_send(so, 0, m, NULL, NULL, td);
+
                crit_exit();
                if (error) {
                        ssb_unlock(&so->so_snd);
@@ -1814,10 +1819,14 @@ retry_space:
        }
        if (mheader != NULL) {
                *sbytes += mheader->m_pkthdr.len;
+
+               for (mp = mheader; mp != NULL; mp = mp->m_next)
+                       ssb_preallocstream(&so->so_snd, mp);
                if (use_sendfile_async)
                        error = so_pru_senda(so, 0, mheader, NULL, NULL, td);
                else
                        error = so_pru_send(so, 0, mheader, NULL, NULL, td);
+
                mheader = NULL;
        }
        ssb_unlock(&so->so_snd);

diff --git a/sys/netinet/tcp_input.c b/sys/netinet/tcp_input.c
index 5238fad..2edd2ae 100644 (file)
--- a/sys/netinet/tcp_input.c
+++ b/sys/netinet/tcp_input.c
@@ -230,7 +230,7 @@ int tcp_sosnd_agglim = 2;
 SYSCTL_INT(_net_inet_tcp, OID_AUTO, sosnd_agglim, CTLFLAG_RW,
     &tcp_sosnd_agglim, 0, "TCP sosend mbuf aggregation limit");
 
-int tcp_sosnd_async = 0;
+int tcp_sosnd_async = 1;
 SYSCTL_INT(_net_inet_tcp, OID_AUTO, sosnd_async, CTLFLAG_RW,
     &tcp_sosnd_async, 0, "TCP asynchronized pru_send");
 

diff --git a/sys/sys/sockbuf.h b/sys/sys/sockbuf.h
index 8b7d377..98a94c0 100644 (file)
--- a/sys/sys/sockbuf.h
+++ b/sys/sys/sockbuf.h
@@ -52,6 +52,8 @@
 struct sockbuf {
        u_long  sb_cc;          /* actual chars in buffer */
        u_long  sb_mbcnt;       /* chars of mbufs used */
+       u_long  sb_cc_prealloc;
+       u_long  sb_mbcnt_prealloc;
        u_long  sb_climit;      /* data limit when used for I/O */
        struct  mbuf *sb_mb;    /* the mbuf chain */
        struct  mbuf *sb_lastmbuf;      /* last mbuf in sb_mb */
@@ -65,6 +67,11 @@ struct sockbuf {
 
 #ifdef _KERNEL
 
+#include <machine/atomic.h>
+#ifndef _SYS_MBUF_H_
+#include <sys/mbuf.h>
+#endif
+
 /*
  * Macros for sockets and socket buffering.
  */
@@ -83,12 +90,30 @@ struct sockbuf {
                (sb)->sb_mbcnt += (m)->m_ext.ext_size; \
 }
 
+/* adjust counters in sb reflecting allocation of m */
+#define        sbprealloc(sb, m) { \
+       u_long __mbcnt_sz; \
+ \
+       atomic_add_long(&((sb)->sb_cc_prealloc), (m)->m_len); \
+ \
+       __mbcnt_sz = MSIZE; \
+       if ((m)->m_flags & M_EXT) \
+               __mbcnt_sz += (m)->m_ext.ext_size; \
+       atomic_add_long(&((sb)->sb_mbcnt_prealloc), __mbcnt_sz); \
+}
+
 /* adjust counters in sb reflecting freeing of m */
 #define        sbfree(sb, m) { \
+       u_long __mbcnt_sz; \
+ \
        (sb)->sb_cc -= (m)->m_len; \
-       (sb)->sb_mbcnt -= MSIZE; \
+       atomic_subtract_long(&((sb)->sb_cc_prealloc), (m)->m_len); \
+ \
+       __mbcnt_sz = MSIZE; \
        if ((m)->m_flags & M_EXT) \
-               (sb)->sb_mbcnt -= (m)->m_ext.ext_size; \
+               __mbcnt_sz += (m)->m_ext.ext_size; \
+       (sb)->sb_mbcnt -= __mbcnt_sz; \
+       atomic_subtract_long(&((sb)->sb_mbcnt_prealloc), __mbcnt_sz); \
 }
 
 static __inline void
@@ -96,6 +121,8 @@ sbinit(struct sockbuf *sb, u_long climit)
 {
        sb->sb_cc = 0;
        sb->sb_mbcnt = 0;
+       sb->sb_cc_prealloc = 0;
+       sb->sb_mbcnt_prealloc = 0;
        sb->sb_climit = climit;
        sb->sb_mb = NULL;
        sb->sb_lastmbuf = NULL;

diff --git a/sys/sys/socketvar.h b/sys/sys/socketvar.h
index b3d83f2..fe14c4c 100644 (file)
--- a/sys/sys/socketvar.h
+++ b/sys/sys/socketvar.h
@@ -83,6 +83,8 @@ struct signalsockbuf {
 #define ssb_cc         sb.sb_cc        /* commonly used fields */
 #define ssb_mb         sb.sb_mb        /* commonly used fields */
 #define ssb_mbcnt      sb.sb_mbcnt     /* commonly used fields */
+#define ssb_cc_prealloc        sb.sb_cc_prealloc
+#define ssb_mbcnt_prealloc sb.sb_mbcnt_prealloc
 
 #define        SSB_LOCK        0x0001          /* lock on data queue */
 #define        SSB_WANT        0x0002          /* someone is waiting to lock */
@@ -276,6 +278,39 @@ ssb_space(struct signalsockbuf *ssb)
        return((bleft < mleft) ? bleft : mleft);
 }
 
+static __inline long
+ssb_space_prealloc(struct signalsockbuf *ssb)
+{
+       long bleft, bleft_prealloc;
+       long mleft, mleft_prealloc;
+
+       if (ssb->ssb_flags & SSB_STOP)
+               return(0);
+
+       bleft = ssb->ssb_hiwat - ssb->ssb_cc;
+       bleft_prealloc = ssb->ssb_hiwat - ssb->ssb_cc_prealloc;
+       if (bleft_prealloc < bleft)
+               bleft = bleft_prealloc;
+
+       mleft = ssb->ssb_mbmax - ssb->ssb_mbcnt;
+       mleft_prealloc = ssb->ssb_mbmax - ssb->ssb_mbcnt_prealloc;
+       if (mleft_prealloc < mleft)
+               mleft = mleft_prealloc;
+
+       return((bleft < mleft) ? bleft : mleft);
+}
+
+/*
+ * NOTE: Only works w/ later ssb_appendstream() on m
+ */
+static __inline void
+ssb_preallocstream(struct signalsockbuf *ssb, struct mbuf *m)
+{
+       if (m->m_len == 0)
+               return;
+       sbprealloc(&ssb->sb, m);
+}
+
 #endif
 
 #define ssb_append(ssb, m)                                             \