From acd31a69ea41698798c8dcc03369dd7c70e4741e Mon Sep 17 00:00:00 2001 From: Sepherosa Ziehau Date: Sun, 20 Nov 2011 19:25:05 +0800 Subject: [PATCH 1/1] socket: Speed up soclose by avoiding putting the user thread into sleep - Embed a netmsg_base into socket, it will be used if fast soclose is possible - Factor out sodiscard(), which abort the connections on the listen socket and set the SS_NOFDREF bit. This function is shared across fast soclose and synchronized soclose - Rename the original soclose() to soclose_sync(), which uses domsg to perform proto-specific operation - If kern.ipc.soclose_fast is 1 (it is 1 by default) and SO_LINGER socket option is not set and the socket does not use synchronized msgport (e.g. UNIX domain socket), fast soclose will be used - The fast soclose is implemented to avoid putting the caller thread (usually a user thread) into sleep. It uses the socket's embeded "close message" with different dispatch functions based on the current socket state and send the "close message" (asynchronized) to proto thread to carry out various tasks. The result: On Phenom 9550 (4 core, 2.2GHz): route change -host 127.0.0.1 -msl 50 (set MSL to 50ms) 8 parallel netperf -H 127.0.0.1 -t TCP_CC -P0 (4 runs, unit: tps) old 33181.18 33005.66 33130.48 33010.50 new 39109.07 39032.48 39022.75 38993.72 This gives 18% performance improvement --- sys/kern/uipc_msg.c | 26 +++++++ sys/kern/uipc_socket.c | 169 ++++++++++++++++++++++++++++++++++------- sys/sys/socketops.h | 2 + sys/sys/socketvar.h | 6 ++ 4 files changed, 177 insertions(+), 26 deletions(-) diff --git a/sys/kern/uipc_msg.c b/sys/kern/uipc_msg.c index ea7c3bb843..9cd1e3a0e6 100644 --- a/sys/kern/uipc_msg.c +++ b/sys/kern/uipc_msg.c @@ -231,6 +231,19 @@ so_pru_detach(struct socket *so) return (error); } +void +so_pru_detach_direct(struct socket *so) +{ + struct netmsg_pru_detach msg; + netisr_fn_t func = so->so_proto->pr_usrreqs->pru_detach; + + netmsg_init(&msg.base, so, &netisr_adone_rport, 0, func); + msg.base.lmsg.ms_flags &= ~(MSGF_REPLY | MSGF_DONE); + msg.base.lmsg.ms_flags |= MSGF_SYNC; + func((netmsg_t)&msg); + KKASSERT(msg.base.lmsg.ms_flags & MSGF_DONE); +} + int so_pru_disconnect(struct socket *so) { @@ -243,6 +256,19 @@ so_pru_disconnect(struct socket *so) return (error); } +void +so_pru_disconnect_direct(struct socket *so) +{ + struct netmsg_pru_disconnect msg; + netisr_fn_t func = so->so_proto->pr_usrreqs->pru_disconnect; + + netmsg_init(&msg.base, so, &netisr_adone_rport, 0, func); + msg.base.lmsg.ms_flags &= ~(MSGF_REPLY | MSGF_DONE); + msg.base.lmsg.ms_flags |= MSGF_SYNC; + func((netmsg_t)&msg); + KKASSERT(msg.base.lmsg.ms_flags & MSGF_DONE); +} + int so_pru_listen(struct socket *so, struct thread *td) { diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c index d02c4af804..5a21de35c8 100644 --- a/sys/kern/uipc_socket.c +++ b/sys/kern/uipc_socket.c @@ -91,6 +91,7 @@ #include #include #include +#include #include #include @@ -110,6 +111,10 @@ static void filt_sowdetach(struct knote *kn); static int filt_sowrite(struct knote *kn, long hint); static int filt_solisten(struct knote *kn, long hint); +static void sodiscard(struct socket *so); +static int soclose_sync(struct socket *so, int fflag); +static void soclose_fast(struct socket *so); + static struct filterops solisten_filtops = { FILTEROP_ISFD|FILTEROP_MPSAFE, NULL, filt_sordetach, filt_solisten }; static struct filterops soread_filtops = @@ -128,6 +133,10 @@ static int somaxconn = SOMAXCONN; SYSCTL_INT(_kern_ipc, KIPC_SOMAXCONN, somaxconn, CTLFLAG_RW, &somaxconn, 0, "Maximum pending socket connection queue size"); +static int use_soclose_fast = 1; +SYSCTL_INT(_kern_ipc, OID_AUTO, soclose_fast, CTLFLAG_RW, + &use_soclose_fast, 0, "Fast socket close"); + /* * Socket operation routines. * These routines are called by the routines in @@ -385,9 +394,54 @@ sofree(struct socket *so) int soclose(struct socket *so, int fflag) { - int error = 0; + int error; funsetown(&so->so_sigio); + if (!use_soclose_fast || + (so->so_proto->pr_flags & PR_SYNC_PORT) || + (so->so_options & SO_LINGER)) { + error = soclose_sync(so, fflag); + } else { + soclose_fast(so); + error = 0; + } + return error; +} + +static void +sodiscard(struct socket *so) +{ + lwkt_getpooltoken(so); + if (so->so_options & SO_ACCEPTCONN) { + struct socket *sp; + + while ((sp = TAILQ_FIRST(&so->so_incomp)) != NULL) { + TAILQ_REMOVE(&so->so_incomp, sp, so_list); + soclrstate(sp, SS_INCOMP); + sp->so_head = NULL; + so->so_incqlen--; + soaborta(sp); + } + while ((sp = TAILQ_FIRST(&so->so_comp)) != NULL) { + TAILQ_REMOVE(&so->so_comp, sp, so_list); + soclrstate(sp, SS_COMP); + sp->so_head = NULL; + so->so_qlen--; + soaborta(sp); + } + } + lwkt_relpooltoken(so); + + if (so->so_state & SS_NOFDREF) + panic("soclose: NOFDREF"); + sosetstate(so, SS_NOFDREF); /* take ref */ +} + +static int +soclose_sync(struct socket *so, int fflag) +{ + int error = 0; + if (so->so_pcb == NULL) goto discard; if (so->so_state & SS_ISCONNECTED) { @@ -417,34 +471,97 @@ drop: error = error2; } discard: - lwkt_getpooltoken(so); - if (so->so_options & SO_ACCEPTCONN) { - struct socket *sp; + sodiscard(so); + so_pru_sync(so); /* unpend async sending */ + sofree(so); /* dispose of ref */ - while ((sp = TAILQ_FIRST(&so->so_incomp)) != NULL) { - TAILQ_REMOVE(&so->so_incomp, sp, so_list); - soclrstate(sp, SS_INCOMP); - sp->so_head = NULL; - so->so_incqlen--; - soaborta(sp); - } - while ((sp = TAILQ_FIRST(&so->so_comp)) != NULL) { - TAILQ_REMOVE(&so->so_comp, sp, so_list); - soclrstate(sp, SS_COMP); - sp->so_head = NULL; - so->so_qlen--; - soaborta(sp); - } + return (error); +} + +static void +soclose_sofree_async_handler(netmsg_t msg) +{ + sofree(msg->base.nm_so); +} + +static void +soclose_sofree_async(struct socket *so) +{ + struct netmsg_base *base = &so->so_clomsg; + + netmsg_init(base, so, &netisr_apanic_rport, 0, + soclose_sofree_async_handler); + lwkt_sendmsg(so->so_port, &base->lmsg); +} + +static void +soclose_disconn_async_handler(netmsg_t msg) +{ + struct socket *so = msg->base.nm_so; + + if ((so->so_state & SS_ISCONNECTED) && + (so->so_state & SS_ISDISCONNECTING) == 0) + so_pru_disconnect_direct(so); + + if (so->so_pcb) + so_pru_detach_direct(so); + + sodiscard(so); + sofree(so); +} + +static void +soclose_disconn_async(struct socket *so) +{ + struct netmsg_base *base = &so->so_clomsg; + + netmsg_init(base, so, &netisr_apanic_rport, 0, + soclose_disconn_async_handler); + lwkt_sendmsg(so->so_port, &base->lmsg); +} + +static void +soclose_detach_async_handler(netmsg_t msg) +{ + struct socket *so = msg->base.nm_so; + + if (so->so_pcb) + so_pru_detach_direct(so); + + sodiscard(so); + sofree(so); +} + +static void +soclose_detach_async(struct socket *so) +{ + struct netmsg_base *base = &so->so_clomsg; + + netmsg_init(base, so, &netisr_apanic_rport, 0, + soclose_detach_async_handler); + lwkt_sendmsg(so->so_port, &base->lmsg); +} + +static void +soclose_fast(struct socket *so) +{ + if (so->so_pcb == NULL) + goto discard; + + if ((so->so_state & SS_ISCONNECTED) && + (so->so_state & SS_ISDISCONNECTING) == 0) { + soclose_disconn_async(so); + return; } - lwkt_relpooltoken(so); - if (so->so_state & SS_NOFDREF) - panic("soclose: NOFDREF"); - sosetstate(so, SS_NOFDREF); /* take ref */ - /* Make sure all asychronous sending are done */ - so_pru_sync(so); - sofree(so); /* dispose of ref */ - return (error); + if (so->so_pcb) { + soclose_detach_async(so); + return; + } + +discard: + sodiscard(so); + soclose_sofree_async(so); } /* diff --git a/sys/sys/socketops.h b/sys/sys/socketops.h index ea43075276..b0a9b79203 100644 --- a/sys/sys/socketops.h +++ b/sys/sys/socketops.h @@ -87,7 +87,9 @@ int so_pru_connect2 (struct socket *so1, struct socket *so2); int so_pru_control_direct(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp); int so_pru_detach (struct socket *so); +void so_pru_detach_direct (struct socket *so); int so_pru_disconnect (struct socket *so); +void so_pru_disconnect_direct (struct socket *so); int so_pru_listen (struct socket *so, struct thread *td); int so_pru_peeraddr (struct socket *so, struct sockaddr **nam); int so_pru_rcvd (struct socket *so, int flags); diff --git a/sys/sys/socketvar.h b/sys/sys/socketvar.h index 5f9c440b97..ab494be669 100644 --- a/sys/sys/socketvar.h +++ b/sys/sys/socketvar.h @@ -56,6 +56,10 @@ #if defined(_KERNEL) || defined(_KERNEL_STRUCTURES) +#ifndef _NET_NETMSG_H_ +#include +#endif + struct accept_filter; /* @@ -148,6 +152,8 @@ struct socket { void *so_accept_filter_arg; /* saved filter args */ char *so_accept_filter_str; /* saved user args */ } *so_accf; + + struct netmsg_base so_clomsg; }; #endif -- 2.41.0