socket: Speed up soclose by avoiding putting the user thread into sleep

- Embed a netmsg_base into socket, it will be used if fast soclose
  is possible
- Factor out sodiscard(), which abort the connections on the listen
  socket and set the SS_NOFDREF bit.  This function is shared across
  fast soclose and synchronized soclose
- Rename the original soclose() to soclose_sync(), which uses domsg
  to perform proto-specific operation
- If kern.ipc.soclose_fast is 1 (it is 1 by default) and SO_LINGER
  socket option is not set and the socket does not use synchronized
  msgport (e.g. UNIX domain socket), fast soclose will be used
- The fast soclose is implemented to avoid putting the caller thread
  (usually a user thread) into sleep.  It uses the socket's embeded
  "close message" with different dispatch functions based on the
  current socket state and send the "close message" (asynchronized)
  to proto thread to carry out various tasks.

The result:
On Phenom 9550 (4 core, 2.2GHz):
route change -host 127.0.0.1 -msl 50 (set MSL to 50ms)
8 parallel netperf -H 127.0.0.1 -t TCP_CC -P0 (4 runs, unit: tps)

old  33181.18  33005.66  33130.48  33010.50
new  39109.07  39032.48  39022.75  38993.72

This gives 18% performance improvement

diff --git a/sys/kern/uipc_msg.c b/sys/kern/uipc_msg.c
index ea7c3bb..9cd1e3a 100644 (file)
--- a/sys/kern/uipc_msg.c
+++ b/sys/kern/uipc_msg.c
@@ -231,6 +231,19 @@ so_pru_detach(struct socket *so)
        return (error);
 }
 
+void
+so_pru_detach_direct(struct socket *so)
+{
+       struct netmsg_pru_detach msg;
+       netisr_fn_t func = so->so_proto->pr_usrreqs->pru_detach;
+
+       netmsg_init(&msg.base, so, &netisr_adone_rport, 0, func);
+       msg.base.lmsg.ms_flags &= ~(MSGF_REPLY | MSGF_DONE);
+       msg.base.lmsg.ms_flags |= MSGF_SYNC;
+       func((netmsg_t)&msg);
+       KKASSERT(msg.base.lmsg.ms_flags & MSGF_DONE);
+}
+
 int
 so_pru_disconnect(struct socket *so)
 {
@@ -243,6 +256,19 @@ so_pru_disconnect(struct socket *so)
        return (error);
 }
 
+void
+so_pru_disconnect_direct(struct socket *so)
+{
+       struct netmsg_pru_disconnect msg;
+       netisr_fn_t func = so->so_proto->pr_usrreqs->pru_disconnect;
+
+       netmsg_init(&msg.base, so, &netisr_adone_rport, 0, func);
+       msg.base.lmsg.ms_flags &= ~(MSGF_REPLY | MSGF_DONE);
+       msg.base.lmsg.ms_flags |= MSGF_SYNC;
+       func((netmsg_t)&msg);
+       KKASSERT(msg.base.lmsg.ms_flags & MSGF_DONE);
+}
+
 int
 so_pru_listen(struct socket *so, struct thread *td)
 {

diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
index d02c4af..5a21de3 100644 (file)
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -91,6 +91,7 @@
 #include <sys/jail.h>
 #include <vm/vm_zone.h>
 #include <vm/pmap.h>
+#include <net/netmsg2.h>
 
 #include <sys/thread2.h>
 #include <sys/socketvar2.h>
@@ -110,6 +111,10 @@ static void        filt_sowdetach(struct knote *kn);
 static int     filt_sowrite(struct knote *kn, long hint);
 static int     filt_solisten(struct knote *kn, long hint);
 
+static void    sodiscard(struct socket *so);
+static int     soclose_sync(struct socket *so, int fflag);
+static void    soclose_fast(struct socket *so);
+
 static struct filterops solisten_filtops = 
        { FILTEROP_ISFD|FILTEROP_MPSAFE, NULL, filt_sordetach, filt_solisten };
 static struct filterops soread_filtops =
@@ -128,6 +133,10 @@ static int somaxconn = SOMAXCONN;
 SYSCTL_INT(_kern_ipc, KIPC_SOMAXCONN, somaxconn, CTLFLAG_RW,
     &somaxconn, 0, "Maximum pending socket connection queue size");
 
+static int use_soclose_fast = 1;
+SYSCTL_INT(_kern_ipc, OID_AUTO, soclose_fast, CTLFLAG_RW,
+    &use_soclose_fast, 0, "Fast socket close");
+
 /*
  * Socket operation routines.
  * These routines are called by the routines in
@@ -385,9 +394,54 @@ sofree(struct socket *so)
 int
 soclose(struct socket *so, int fflag)
 {
-       int error = 0;
+       int error;
 
        funsetown(&so->so_sigio);
+       if (!use_soclose_fast ||
+           (so->so_proto->pr_flags & PR_SYNC_PORT) ||
+           (so->so_options & SO_LINGER)) {
+               error = soclose_sync(so, fflag);
+       } else {
+               soclose_fast(so);
+               error = 0;
+       }
+       return error;
+}
+
+static void
+sodiscard(struct socket *so)
+{
+       lwkt_getpooltoken(so);
+       if (so->so_options & SO_ACCEPTCONN) {
+               struct socket *sp;
+
+               while ((sp = TAILQ_FIRST(&so->so_incomp)) != NULL) {
+                       TAILQ_REMOVE(&so->so_incomp, sp, so_list);
+                       soclrstate(sp, SS_INCOMP);
+                       sp->so_head = NULL;
+                       so->so_incqlen--;
+                       soaborta(sp);
+               }
+               while ((sp = TAILQ_FIRST(&so->so_comp)) != NULL) {
+                       TAILQ_REMOVE(&so->so_comp, sp, so_list);
+                       soclrstate(sp, SS_COMP);
+                       sp->so_head = NULL;
+                       so->so_qlen--;
+                       soaborta(sp);
+               }
+       }
+       lwkt_relpooltoken(so);
+
+       if (so->so_state & SS_NOFDREF)
+               panic("soclose: NOFDREF");
+       sosetstate(so, SS_NOFDREF);     /* take ref */
+}
+
+static int
+soclose_sync(struct socket *so, int fflag)
+{
+       int error = 0;
+
        if (so->so_pcb == NULL)
                goto discard;
        if (so->so_state & SS_ISCONNECTED) {
@@ -417,34 +471,97 @@ drop:
                        error = error2;
        }
 discard:
-       lwkt_getpooltoken(so);
-       if (so->so_options & SO_ACCEPTCONN) {
-               struct socket *sp;
+       sodiscard(so);
+       so_pru_sync(so);        /* unpend async sending */
+       sofree(so);             /* dispose of ref */
 
-               while ((sp = TAILQ_FIRST(&so->so_incomp)) != NULL) {
-                       TAILQ_REMOVE(&so->so_incomp, sp, so_list);
-                       soclrstate(sp, SS_INCOMP);
-                       sp->so_head = NULL;
-                       so->so_incqlen--;
-                       soaborta(sp);
-               }
-               while ((sp = TAILQ_FIRST(&so->so_comp)) != NULL) {
-                       TAILQ_REMOVE(&so->so_comp, sp, so_list);
-                       soclrstate(sp, SS_COMP);
-                       sp->so_head = NULL;
-                       so->so_qlen--;
-                       soaborta(sp);
-               }
+       return (error);
+}
+
+static void
+soclose_sofree_async_handler(netmsg_t msg)
+{
+       sofree(msg->base.nm_so);
+}
+
+static void
+soclose_sofree_async(struct socket *so)
+{
+       struct netmsg_base *base = &so->so_clomsg;
+
+       netmsg_init(base, so, &netisr_apanic_rport, 0,
+           soclose_sofree_async_handler);
+       lwkt_sendmsg(so->so_port, &base->lmsg);
+}
+
+static void
+soclose_disconn_async_handler(netmsg_t msg)
+{
+       struct socket *so = msg->base.nm_so;
+
+       if ((so->so_state & SS_ISCONNECTED) &&
+           (so->so_state & SS_ISDISCONNECTING) == 0)
+               so_pru_disconnect_direct(so);
+
+       if (so->so_pcb)
+               so_pru_detach_direct(so);
+
+       sodiscard(so);
+       sofree(so);
+}
+
+static void
+soclose_disconn_async(struct socket *so)
+{
+       struct netmsg_base *base = &so->so_clomsg;
+
+       netmsg_init(base, so, &netisr_apanic_rport, 0,
+           soclose_disconn_async_handler);
+       lwkt_sendmsg(so->so_port, &base->lmsg);
+}
+
+static void
+soclose_detach_async_handler(netmsg_t msg)
+{
+       struct socket *so = msg->base.nm_so;
+
+       if (so->so_pcb)
+               so_pru_detach_direct(so);
+
+       sodiscard(so);
+       sofree(so);
+}
+
+static void
+soclose_detach_async(struct socket *so)
+{
+       struct netmsg_base *base = &so->so_clomsg;
+
+       netmsg_init(base, so, &netisr_apanic_rport, 0,
+           soclose_detach_async_handler);
+       lwkt_sendmsg(so->so_port, &base->lmsg);
+}
+
+static void
+soclose_fast(struct socket *so)
+{
+       if (so->so_pcb == NULL)
+               goto discard;
+
+       if ((so->so_state & SS_ISCONNECTED) &&
+           (so->so_state & SS_ISDISCONNECTING) == 0) {
+               soclose_disconn_async(so);
+               return;
        }
-       lwkt_relpooltoken(so);
-       if (so->so_state & SS_NOFDREF)
-               panic("soclose: NOFDREF");
-       sosetstate(so, SS_NOFDREF);     /* take ref */
 
-       /* Make sure all asychronous sending are done */
-       so_pru_sync(so);
-       sofree(so);                     /* dispose of ref */
-       return (error);
+       if (so->so_pcb) {
+               soclose_detach_async(so);
+               return;
+       }
+
+discard:
+       sodiscard(so);
+       soclose_sofree_async(so);
 }
 
 /*

diff --git a/sys/sys/socketops.h b/sys/sys/socketops.h
index ea43075..b0a9b79 100644 (file)
--- a/sys/sys/socketops.h
+++ b/sys/sys/socketops.h
@@ -87,7 +87,9 @@ int so_pru_connect2 (struct socket *so1, struct socket *so2);
 int so_pru_control_direct(struct socket *so, u_long cmd, caddr_t data,
                struct ifnet *ifp);
 int so_pru_detach (struct socket *so);
+void so_pru_detach_direct (struct socket *so);
 int so_pru_disconnect (struct socket *so);
+void so_pru_disconnect_direct (struct socket *so);
 int so_pru_listen (struct socket *so, struct thread *td);
 int so_pru_peeraddr (struct socket *so, struct sockaddr **nam);
 int so_pru_rcvd (struct socket *so, int flags);

diff --git a/sys/sys/socketvar.h b/sys/sys/socketvar.h
index 5f9c440..ab494be 100644 (file)
--- a/sys/sys/socketvar.h
+++ b/sys/sys/socketvar.h
@@ -56,6 +56,10 @@
 
 #if defined(_KERNEL) || defined(_KERNEL_STRUCTURES)
 
+#ifndef _NET_NETMSG_H_
+#include <net/netmsg.h>
+#endif
+
 struct accept_filter;
 
 /*
@@ -148,6 +152,8 @@ struct socket {
                void    *so_accept_filter_arg;  /* saved filter args */
                char    *so_accept_filter_str;  /* saved user args */
        } *so_accf;
+
+       struct netmsg_base so_clomsg;
 };
 
 #endif