tcp: Let sosendtcp() call tcp_usrreq.pru_send asynchronous

- Embed netmsg_pru_send into mbuf.m_hdr, which shares the space with
  netmsg_pru_packet.
- Use the newly added netmsg_pru_send in mbuf to perform asynchronous
  pru_send.  For asynchronous pru_send, PRUS_NOREPLY is added, which
  prevents pru_send to reply the message.
- In sosendtcp(), if we have more data to call pru_send, we call it
  asynchronously.  The last piece of data or OOB data will still be
  passed to pru_send synchronously.

On Phenom 9550 (4 core, 2.2GHz):
8 parallel netperf -H 127.0.0.1 (4 runs, unit: Mbps)

old  5863.85  5773.13  5534.14  5506.72
new  6735.98  6903.13  6971.89  7056.66

This give ~20% performance improvement.

It has no obvious impact on 1000BaseT or 100baseTX network performace.

diff --git a/sys/kern/uipc_msg.c b/sys/kern/uipc_msg.c
index b2670ce..a81285f 100644 (file)
--- a/sys/kern/uipc_msg.c
+++ b/sys/kern/uipc_msg.c
@@ -44,6 +44,7 @@
 #include <sys/thread.h>
 #include <sys/thread2.h>
 #include <sys/msgport2.h>
+#include <sys/mbuf.h>
 #include <vm/pmap.h>
 #include <net/netmsg2.h>
 
@@ -316,6 +317,23 @@ so_pru_send(struct socket *so, int flags, struct mbuf *m,
        return (error);
 }
 
+void
+so_pru_send_async(struct socket *so, int flags, struct mbuf *m,
+           struct sockaddr *addr, struct mbuf *control, struct thread *td)
+{
+       struct netmsg_pru_send *msg;
+
+       msg = &m->m_hdr.mh_sndmsg;
+       netmsg_init(&msg->base, so, &netisr_apanic_rport,
+                   0, so->so_proto->pr_usrreqs->pru_send);
+       msg->nm_flags = flags | PRUS_NOREPLY;
+       msg->nm_m = m;
+       msg->nm_addr = addr;
+       msg->nm_control = control;
+       msg->nm_td = td;
+       lwkt_sendmsg(so->so_port, &msg->base.lmsg);
+}
+
 int
 so_pru_sense(struct socket *so, struct stat *sb)
 {

diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
index 4f33215..924839d 100644 (file)
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -956,7 +956,15 @@ restart:
                     * here, but there are probably other places that this
                     * also happens.  We must rethink this.
                     */
-                   error = so_pru_send(so, pru_flags, top, NULL, NULL, td);
+                   if ((pru_flags & PRUS_OOB) ||
+                       (pru_flags & PRUS_MORETOCOME) == 0) {
+                           error = so_pru_send(so, pru_flags, top,
+                               NULL, NULL, td);
+                   } else {
+                           so_pru_send_async(so, pru_flags, top,
+                               NULL, NULL, td);
+                           error = 0;
+                   }
 
                    top = NULL;
                    mp = ⊤

diff --git a/sys/net/netmsg.h b/sys/net/netmsg.h
index 99df2b1..6ea7573 100644 (file)
--- a/sys/net/netmsg.h
+++ b/sys/net/netmsg.h
@@ -191,6 +191,7 @@ struct netmsg_pru_send {
 #define PRUS_EOF               0x2
 #define PRUS_MORETOCOME                0x4
 #define PRUS_NAMALLOC          0x8
+#define PRUS_NOREPLY           0x10
 
 struct netmsg_pru_sense {
        struct netmsg_base      base;

diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c
index 5848a8f..36d7ab2 100644 (file)
--- a/sys/netinet/tcp_usrreq.c
+++ b/sys/netinet/tcp_usrreq.c
@@ -257,13 +257,16 @@ tcp_usr_detach(netmsg_t msg)
                 TCPDEBUG1();                                   \
        } while(0)
 
-#define COMMON_END(req)                                                \
+#define COMMON_END1(req, noreply)                              \
        out: do {                                               \
                TCPDEBUG2(req);                                 \
-               lwkt_replymsg(&msg->lmsg, error);               \
+               if (!(noreply))                                 \
+                       lwkt_replymsg(&msg->lmsg, error);       \
                return;                                         \
        } while(0)
 
+#define COMMON_END(req)                COMMON_END1((req), 0)
+
 /*
  * Give the socket an address.
  */
@@ -800,8 +803,9 @@ tcp_usr_send(netmsg_t msg)
                                tp->t_flags &= ~TF_MORETOCOME;
                }
        }
-       COMMON_END((flags & PRUS_OOB) ? PRU_SENDOOB :
-                  ((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND));
+       COMMON_END1((flags & PRUS_OOB) ? PRU_SENDOOB :
+                  ((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND),
+                  (flags & PRUS_NOREPLY));
 }
 
 /*

diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h
index 6e6f491..82c2820 100644 (file)
--- a/sys/sys/mbuf.h
+++ b/sys/sys/mbuf.h
@@ -93,8 +93,13 @@ struct m_hdr {
 #ifdef MBUF_DEBUG
        const char *mh_lastfunc;
 #endif
-       struct netmsg_packet mh_netmsg; /* hardware->proto stack msg */
+       union {
+               struct netmsg_packet mhm_pkt;   /* hardware->proto stack msg */
+               struct netmsg_pru_send mhm_snd; /* usrspace->proto stack msg */
+       } mh_msgu;
 };
+#define mh_netmsg      mh_msgu.mhm_pkt
+#define mh_sndmsg      mh_msgu.mhm_snd
 
 /* pf stuff */
 struct pkthdr_pf {

diff --git a/sys/sys/socketops.h b/sys/sys/socketops.h
index e2f2d47..2274435 100644 (file)
--- a/sys/sys/socketops.h
+++ b/sys/sys/socketops.h
@@ -95,6 +95,9 @@ int so_pru_rcvoob (struct socket *so, struct mbuf *m, int flags);
 int so_pru_send (struct socket *so, int flags, struct mbuf *m,
                struct sockaddr *addr, struct mbuf *control,
                struct thread *td);
+void so_pru_send_async (struct socket *so, int flags, struct mbuf *m,
+               struct sockaddr *addr, struct mbuf *control,
+               struct thread *td);
 int so_pru_sense (struct socket *so, struct stat *sb);
 int so_pru_shutdown (struct socket *so);
 int so_pru_sockaddr (struct socket *so, struct sockaddr **nam);