From 0ad8e15ea0059b225c9aa255b844cb9ec48d5227 Mon Sep 17 00:00:00 2001 From: Sepherosa Ziehau Date: Sun, 23 Oct 2011 20:55:16 +0800 Subject: [PATCH] tcp: Let sosendtcp() call tcp_usrreq.pru_send asynchronous - Embed netmsg_pru_send into mbuf.m_hdr, which shares the space with netmsg_pru_packet. - Use the newly added netmsg_pru_send in mbuf to perform asynchronous pru_send. For asynchronous pru_send, PRUS_NOREPLY is added, which prevents pru_send to reply the message. - In sosendtcp(), if we have more data to call pru_send, we call it asynchronously. The last piece of data or OOB data will still be passed to pru_send synchronously. On Phenom 9550 (4 core, 2.2GHz): 8 parallel netperf -H 127.0.0.1 (4 runs, unit: Mbps) old 5863.85 5773.13 5534.14 5506.72 new 6735.98 6903.13 6971.89 7056.66 This give ~20% performance improvement. It has no obvious impact on 1000BaseT or 100baseTX network performace. --- sys/kern/uipc_msg.c | 18 ++++++++++++++++++ sys/kern/uipc_socket.c | 10 +++++++++- sys/net/netmsg.h | 1 + sys/netinet/tcp_usrreq.c | 12 ++++++++---- sys/sys/mbuf.h | 7 ++++++- sys/sys/socketops.h | 3 +++ 6 files changed, 45 insertions(+), 6 deletions(-) diff --git a/sys/kern/uipc_msg.c b/sys/kern/uipc_msg.c index b2670ce319..a81285fe4d 100644 --- a/sys/kern/uipc_msg.c +++ b/sys/kern/uipc_msg.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include @@ -316,6 +317,23 @@ so_pru_send(struct socket *so, int flags, struct mbuf *m, return (error); } +void +so_pru_send_async(struct socket *so, int flags, struct mbuf *m, + struct sockaddr *addr, struct mbuf *control, struct thread *td) +{ + struct netmsg_pru_send *msg; + + msg = &m->m_hdr.mh_sndmsg; + netmsg_init(&msg->base, so, &netisr_apanic_rport, + 0, so->so_proto->pr_usrreqs->pru_send); + msg->nm_flags = flags | PRUS_NOREPLY; + msg->nm_m = m; + msg->nm_addr = addr; + msg->nm_control = control; + msg->nm_td = td; + lwkt_sendmsg(so->so_port, &msg->base.lmsg); +} + int so_pru_sense(struct socket *so, struct stat *sb) { diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c index 4f3321550c..924839d494 100644 --- a/sys/kern/uipc_socket.c +++ b/sys/kern/uipc_socket.c @@ -956,7 +956,15 @@ restart: * here, but there are probably other places that this * also happens. We must rethink this. */ - error = so_pru_send(so, pru_flags, top, NULL, NULL, td); + if ((pru_flags & PRUS_OOB) || + (pru_flags & PRUS_MORETOCOME) == 0) { + error = so_pru_send(so, pru_flags, top, + NULL, NULL, td); + } else { + so_pru_send_async(so, pru_flags, top, + NULL, NULL, td); + error = 0; + } top = NULL; mp = ⊤ diff --git a/sys/net/netmsg.h b/sys/net/netmsg.h index 99df2b17c4..6ea757327b 100644 --- a/sys/net/netmsg.h +++ b/sys/net/netmsg.h @@ -191,6 +191,7 @@ struct netmsg_pru_send { #define PRUS_EOF 0x2 #define PRUS_MORETOCOME 0x4 #define PRUS_NAMALLOC 0x8 +#define PRUS_NOREPLY 0x10 struct netmsg_pru_sense { struct netmsg_base base; diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c index 5848a8f6e6..36d7ab2b99 100644 --- a/sys/netinet/tcp_usrreq.c +++ b/sys/netinet/tcp_usrreq.c @@ -257,13 +257,16 @@ tcp_usr_detach(netmsg_t msg) TCPDEBUG1(); \ } while(0) -#define COMMON_END(req) \ +#define COMMON_END1(req, noreply) \ out: do { \ TCPDEBUG2(req); \ - lwkt_replymsg(&msg->lmsg, error); \ + if (!(noreply)) \ + lwkt_replymsg(&msg->lmsg, error); \ return; \ } while(0) +#define COMMON_END(req) COMMON_END1((req), 0) + /* * Give the socket an address. */ @@ -800,8 +803,9 @@ tcp_usr_send(netmsg_t msg) tp->t_flags &= ~TF_MORETOCOME; } } - COMMON_END((flags & PRUS_OOB) ? PRU_SENDOOB : - ((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND)); + COMMON_END1((flags & PRUS_OOB) ? PRU_SENDOOB : + ((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND), + (flags & PRUS_NOREPLY)); } /* diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h index 6e6f491cdb..82c282072c 100644 --- a/sys/sys/mbuf.h +++ b/sys/sys/mbuf.h @@ -93,8 +93,13 @@ struct m_hdr { #ifdef MBUF_DEBUG const char *mh_lastfunc; #endif - struct netmsg_packet mh_netmsg; /* hardware->proto stack msg */ + union { + struct netmsg_packet mhm_pkt; /* hardware->proto stack msg */ + struct netmsg_pru_send mhm_snd; /* usrspace->proto stack msg */ + } mh_msgu; }; +#define mh_netmsg mh_msgu.mhm_pkt +#define mh_sndmsg mh_msgu.mhm_snd /* pf stuff */ struct pkthdr_pf { diff --git a/sys/sys/socketops.h b/sys/sys/socketops.h index e2f2d474fc..2274435127 100644 --- a/sys/sys/socketops.h +++ b/sys/sys/socketops.h @@ -95,6 +95,9 @@ int so_pru_rcvoob (struct socket *so, struct mbuf *m, int flags); int so_pru_send (struct socket *so, int flags, struct mbuf *m, struct sockaddr *addr, struct mbuf *control, struct thread *td); +void so_pru_send_async (struct socket *so, int flags, struct mbuf *m, + struct sockaddr *addr, struct mbuf *control, + struct thread *td); int so_pru_sense (struct socket *so, struct stat *sb); int so_pru_shutdown (struct socket *so); int so_pru_sockaddr (struct socket *so, struct sockaddr **nam); -- 2.41.0