From 6cef7136f04e2b24a6db289e78720d6d8c60274e Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Thu, 9 Sep 2010 21:01:20 -0700 Subject: [PATCH] network - MP socket free & abort interactions, so_state * Add so_refs and ref-count the socket structure to deal with MP races on sofree(). * Ref the socket structure for all soabort() operations (they are usually asynchronous). The netmsg_pru_abort() handler will sofree() the ref after calling the protocol stack's abort function. * Use atomic ops to set and clear bits in so_state, because it is modified by both the fronttend and the backend. * Remove numerous critical sections that are no longer effective. * Protect the accept queues with so_rcv.ssb_token. * Protect after-the-fact calls to soisdisconnected() with a soreference() to avoid use-after-free cases. * Wrap unix domain, mroute, div, raw, and key sockets/protocols with their own private tokens. --- sys/ddb/db_output.c | 2 + sys/kern/sys_generic.c | 5 + sys/kern/sys_socket.c | 5 +- sys/kern/uipc_mbuf.c | 2 +- sys/kern/uipc_msg.c | 16 +- sys/kern/uipc_socket.c | 133 +++--- sys/kern/uipc_socket2.c | 73 +++- sys/kern/uipc_syscalls.c | 11 +- sys/kern/uipc_usrreq.c | 406 +++++++++++++----- sys/net/ip_mroute/ip_mroute.c | 153 +++---- sys/net/raw_cb.c | 5 +- sys/net/raw_usrreq.c | 10 +- sys/net/rtsock.c | 3 + sys/netbt/hci_socket.c | 15 +- sys/netbt/l2cap_socket.c | 8 +- sys/netbt/rfcomm_socket.c | 8 +- sys/netbt/sco_socket.c | 8 +- sys/netgraph/ksocket/ng_ksocket.c | 22 +- sys/netgraph/socket/ng_socket.c | 7 +- .../bluetooth/socket/ng_btsocket_rfcomm.c | 10 +- sys/netgraph7/ng_ksocket.c | 8 +- sys/netgraph7/ng_socket.c | 8 +- sys/netinet/in_pcb.c | 5 +- sys/netinet/ip_divert.c | 38 +- sys/netinet/raw_ip.c | 60 ++- sys/netinet/sctp_indata.c | 12 + sys/netinet/sctp_input.c | 2 +- sys/netinet/sctp_pcb.c | 31 +- sys/netinet/sctp_peeloff.c | 4 +- sys/netinet/sctp_usrreq.c | 21 +- sys/netinet/sctputil.c | 27 ++ sys/netinet/tcp_input.c | 25 +- sys/netinet/tcp_subr.c | 1 + sys/netinet/tcp_usrreq.c | 60 ++- sys/netinet/udp_usrreq.c | 40 +- sys/netinet6/in6_pcb.c | 2 +- sys/netinet6/ipsec.c | 12 + sys/netinet6/raw_ip6.c | 19 +- sys/netinet6/sctp6_usrreq.c | 22 +- sys/netinet6/udp6_usrreq.c | 25 +- sys/netproto/atalk/ddp_usrreq.c | 45 +- sys/netproto/atm/atm_aal5.c | 3 +- sys/netproto/atm/atm_socket.c | 2 +- sys/netproto/ipsec/keysock.c | 4 +- sys/netproto/ipx/ipx_pcb.c | 3 +- sys/netproto/ipx/ipx_usrreq.c | 27 +- sys/netproto/ipx/spx_usrreq.c | 11 +- sys/netproto/key/key.c | 144 ++++--- sys/netproto/key/key.h | 1 + sys/netproto/key/keydb.c | 9 +- sys/netproto/key/keysock.c | 61 +-- sys/netproto/natm/natm.c | 18 +- sys/netproto/ncp/ncp_sock.c | 2 +- sys/netproto/ns/idp_usrreq.c | 13 +- sys/netproto/ns/ns_pcb.c | 2 +- sys/netproto/ns/spp_usrreq.c | 11 +- sys/netproto/smb/smb_trantcp.c | 2 +- sys/sys/socketvar.h | 13 +- sys/sys/socketvar2.h | 26 +- sys/vfs/fifofs/fifo_vnops.c | 3 +- sys/vfs/nfs/nfs_socket.c | 3 +- 61 files changed, 1107 insertions(+), 620 deletions(-) diff --git a/sys/ddb/db_output.c b/sys/ddb/db_output.c index cf674a0680..7347b3b285 100644 --- a/sys/ddb/db_output.c +++ b/sys/ddb/db_output.c @@ -181,12 +181,14 @@ db_printf(const char *fmt, ...) __va_start(listp, fmt); kvcprintf (fmt, db_putchar, NULL, db_radix, listp); __va_end(listp); +/* DELAY(100000);*/ } void db_vprintf(const char *fmt, __va_list va) { kvcprintf (fmt, db_putchar, NULL, db_radix, va); +/* DELAY(100000);*/ } int db_indent; diff --git a/sys/kern/sys_generic.c b/sys/kern/sys_generic.c index 18b6f6abec..8f70272152 100644 --- a/sys/kern/sys_generic.c +++ b/sys/kern/sys_generic.c @@ -1499,6 +1499,11 @@ socket_wait_copyout(void *arg, struct kevent *kevp, int count, int *res) } extern struct fileops socketops; + +/* + * NOTE: Callers of socket_wait() must already have a reference on the + * socket. + */ int socket_wait(struct socket *so, struct timespec *ts, int *res) { diff --git a/sys/kern/sys_socket.c b/sys/kern/sys_socket.c index 0b0eeb03a5..92527aa161 100644 --- a/sys/kern/sys_socket.c +++ b/sys/kern/sys_socket.c @@ -52,6 +52,7 @@ #include #include +#include #include #include @@ -136,11 +137,11 @@ soo_ioctl(struct file *fp, u_long cmd, caddr_t data, switch (cmd) { case FIOASYNC: if (*(int *)data) { - so->so_state |= SS_ASYNC; + sosetstate(so, SS_ASYNC); atomic_set_int(&so->so_rcv.ssb_flags, SSB_ASYNC); atomic_set_int(&so->so_snd.ssb_flags, SSB_ASYNC); } else { - so->so_state &= ~SS_ASYNC; + soclrstate(so, SS_ASYNC); atomic_clear_int(&so->so_rcv.ssb_flags, SSB_ASYNC); atomic_clear_int(&so->so_snd.ssb_flags, SSB_ASYNC); } diff --git a/sys/kern/uipc_mbuf.c b/sys/kern/uipc_mbuf.c index 18f9b56e90..e0c803086e 100644 --- a/sys/kern/uipc_mbuf.c +++ b/sys/kern/uipc_mbuf.c @@ -167,9 +167,9 @@ mbufuntrack(struct mbuf *m) panic("mbufuntrack: mbuf %p was not tracked\n", m); } else { mbuf_rb_tree_RB_REMOVE(&mbuf_track_root, mbt); + spin_unlock(&mbuf_track_spin); kfree(mbt, M_MTRACK); } - spin_unlock(&mbuf_track_spin); } void diff --git a/sys/kern/uipc_msg.c b/sys/kern/uipc_msg.c index 02a895fe13..1c9e786e02 100644 --- a/sys/kern/uipc_msg.c +++ b/sys/kern/uipc_msg.c @@ -52,15 +52,12 @@ /* * Abort a socket and free it. Called from soabort() only. - * - * The SS_ABORTING flag must already be set. */ void so_pru_abort(struct socket *so) { struct netmsg_pru_abort msg; - KKASSERT(so->so_state & SS_ABORTING); netmsg_init(&msg.nm_netmsg, so, &curthread->td_msgport, 0, netmsg_pru_abort); msg.nm_prufn = so->so_proto->pr_usrreqs->pru_abort; @@ -70,15 +67,12 @@ so_pru_abort(struct socket *so) /* * Abort a socket and free it, asynchronously. Called from * soaborta() only. - * - * The SS_ABORTING flag must already be set. */ void so_pru_aborta(struct socket *so) { struct netmsg_pru_abort *msg; - KKASSERT(so->so_state & SS_ABORTING); msg = kmalloc(sizeof(*msg), M_LWKTMSG, M_WAITOK | M_ZERO); netmsg_init(&msg->nm_netmsg, so, &netisr_afree_rport, 0, netmsg_pru_abort); @@ -89,8 +83,6 @@ so_pru_aborta(struct socket *so) /* * Abort a socket and free it. Called from soabort_oncpu() only. * Caller must make sure that the current CPU is inpcb's owner CPU. - * - * The SS_ABORTING flag must already be set. */ void so_pru_abort_oncpu(struct socket *so) @@ -418,6 +410,9 @@ so_pru_ctlinput(struct protosw *pr, int cmd, struct sockaddr *arg, void *extra) /* * Abort and destroy a socket. + * + * The originator referenced the socket so we must dereference it when + * done. */ void netmsg_pru_abort(netmsg_t msg) @@ -426,11 +421,8 @@ netmsg_pru_abort(netmsg_t msg) struct socket *so = msg->nm_so; int error; - KKASSERT(so->so_state & SS_ABORTING); - so->so_state &= ~SS_ABORTING; error = nm->nm_prufn(so); - if (error) - sofree(so); + sofree(so); /* from soabort*() */ lwkt_replymsg(&msg->nm_lmsg, error); } diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c index 96e4261336..66160a63bf 100644 --- a/sys/kern/uipc_socket.c +++ b/sys/kern/uipc_socket.c @@ -154,6 +154,10 @@ soalloc(int waitok) TAILQ_INIT(&so->so_aiojobq); TAILQ_INIT(&so->so_rcv.ssb_kq.ki_mlist); TAILQ_INIT(&so->so_snd.ssb_kq.ki_mlist); + lwkt_token_init(&so->so_rcv.ssb_token, 1, "rcvtok"); + lwkt_token_init(&so->so_snd.ssb_token, 1, "rcvtok"); + so->so_state = SS_NOFDREF; + so->so_refs = 1; } return so; } @@ -187,9 +191,16 @@ socreate(int dom, struct socket **aso, int type, if (prp->pr_type != type) return (EPROTOTYPE); so = soalloc(p != 0); - if (so == 0) + if (so == NULL) return (ENOBUFS); + /* + * Callers of socreate() presumably will connect up a descriptor + * and call soclose() if they cannot. This represents our so_refs + * (which should be 1) from soalloc(). + */ + soclrstate(so, SS_NOFDREF); + /* * Set a default port for protocol processing. No action will occur * on the socket on this port until an inpcb is attached to it and @@ -213,11 +224,14 @@ socreate(int dom, struct socket **aso, int type, */ error = so_pru_attach(so, proto, &ai); if (error) { - so->so_state |= SS_NOFDREF; - sofree(so); - return (error); + sosetstate(so, SS_NOFDREF); + sofree(so); /* from soalloc */ + return error; } + /* + * NOTE: Returns referenced socket. + */ *aso = so; return (0); } @@ -227,13 +241,11 @@ sobind(struct socket *so, struct sockaddr *nam, struct thread *td) { int error; - crit_enter(); error = so_pru_bind(so, nam, td); - crit_exit(); return (error); } -void +static void sodealloc(struct socket *so) { if (so->so_rcv.ssb_hiwat) @@ -259,19 +271,18 @@ solisten(struct socket *so, int backlog, struct thread *td) short oldopt, oldqlimit; #endif /* SCTP */ - crit_enter(); - if (so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING)) { - crit_exit(); + if (so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING)) return (EINVAL); - } #ifdef SCTP oldopt = so->so_options; oldqlimit = so->so_qlimit; #endif /* SCTP */ + lwkt_gettoken(&so->so_rcv.ssb_token); if (TAILQ_EMPTY(&so->so_comp)) so->so_options |= SO_ACCEPTCONN; + lwkt_reltoken(&so->so_rcv.ssb_token); if (backlog < 0 || backlog > somaxconn) backlog = somaxconn; so->so_qlimit = backlog; @@ -286,10 +297,8 @@ solisten(struct socket *so, int backlog, struct thread *td) so->so_options = oldopt; so->so_qlimit = oldqlimit; #endif /* SCTP */ - crit_exit(); return (error); } - crit_exit(); return (0); } @@ -299,18 +308,26 @@ solisten(struct socket *so, int backlog, struct thread *td) * * so_pcb - The protocol stack still has a reference * SS_NOFDREF - There is no longer a file pointer reference - * SS_ABORTING - An abort netmsg is in-flight */ void sofree(struct socket *so) { struct socket *head = so->so_head; - if (so->so_pcb || (so->so_state & SS_NOFDREF) == 0) - return; - if (so->so_state & SS_ABORTING) + /* + * Arbitrage the last free. + */ + KKASSERT(so->so_refs > 0); + if (atomic_fetchadd_int(&so->so_refs, -1) != 1) return; + + KKASSERT(so->so_pcb == NULL && (so->so_state & SS_NOFDREF)); + + /* + * We're done, clean up + */ if (head != NULL) { + lwkt_gettoken(&head->so_rcv.ssb_token); if (so->so_state & SS_INCOMP) { TAILQ_REMOVE(&head->so_incomp, so, so_list); head->so_incqlen--; @@ -321,12 +338,14 @@ sofree(struct socket *so) * accept(2) may hang after select(2) indicated * that the listening socket was ready. */ + lwkt_reltoken(&head->so_rcv.ssb_token); return; } else { panic("sofree: not queued"); } - so->so_state &= ~SS_INCOMP; + soclrstate(so, SS_INCOMP); so->so_head = NULL; + lwkt_reltoken(&head->so_rcv.ssb_token); } ssb_release(&so->so_snd, so); sorflush(so); @@ -343,7 +362,6 @@ soclose(struct socket *so, int fflag) { int error = 0; - crit_enter(); funsetown(so->so_sigio); if (so->so_pcb == NULL) goto discard; @@ -358,8 +376,8 @@ soclose(struct socket *so, int fflag) (fflag & FNONBLOCK)) goto drop; while (so->so_state & SS_ISCONNECTED) { - error = tsleep((caddr_t)&so->so_timeo, - PCATCH, "soclos", so->so_linger * hz); + error = tsleep(&so->so_timeo, PCATCH, + "soclos", so->so_linger * hz); if (error) break; } @@ -374,29 +392,30 @@ drop: error = error2; } discard: + lwkt_gettoken(&so->so_rcv.ssb_token); if (so->so_options & SO_ACCEPTCONN) { struct socket *sp; while ((sp = TAILQ_FIRST(&so->so_incomp)) != NULL) { TAILQ_REMOVE(&so->so_incomp, sp, so_list); - sp->so_state &= ~SS_INCOMP; + soclrstate(sp, SS_INCOMP); sp->so_head = NULL; so->so_incqlen--; soaborta(sp); } while ((sp = TAILQ_FIRST(&so->so_comp)) != NULL) { TAILQ_REMOVE(&so->so_comp, sp, so_list); - sp->so_state &= ~SS_COMP; + soclrstate(sp, SS_COMP); sp->so_head = NULL; so->so_qlen--; soaborta(sp); } } + lwkt_reltoken(&so->so_rcv.ssb_token); if (so->so_state & SS_NOFDREF) panic("soclose: NOFDREF"); - so->so_state |= SS_NOFDREF; - sofree(so); - crit_exit(); + sosetstate(so, SS_NOFDREF); /* take ref */ + sofree(so); /* dispose of ref */ return (error); } @@ -407,28 +426,22 @@ discard: void soabort(struct socket *so) { - if ((so->so_state & SS_ABORTING) == 0) { - so->so_state |= SS_ABORTING; - so_pru_abort(so); - } + soreference(so); + so_pru_abort(so); } void soaborta(struct socket *so) { - if ((so->so_state & SS_ABORTING) == 0) { - so->so_state |= SS_ABORTING; - so_pru_aborta(so); - } + soreference(so); + so_pru_aborta(so); } void soabort_oncpu(struct socket *so) { - if ((so->so_state & SS_ABORTING) == 0) { - so->so_state |= SS_ABORTING; - so_pru_abort_oncpu(so); - } + soreference(so); + so_pru_abort_oncpu(so); } int @@ -436,12 +449,11 @@ soaccept(struct socket *so, struct sockaddr **nam) { int error; - crit_enter(); if ((so->so_state & SS_NOFDREF) == 0) panic("soaccept: !NOFDREF"); - so->so_state &= ~SS_NOFDREF; + soreference(so); /* create ref */ + soclrstate(so, SS_NOFDREF); /* owned by lack of SS_NOFDREF */ error = so_pru_accept(so, nam); - crit_exit(); return (error); } @@ -452,7 +464,6 @@ soconnect(struct socket *so, struct sockaddr *nam, struct thread *td) if (so->so_options & SO_ACCEPTCONN) return (EOPNOTSUPP); - crit_enter(); /* * If protocol is connection-based, can only connect once. * Otherwise, if connected, try to disconnect first. @@ -471,7 +482,6 @@ soconnect(struct socket *so, struct sockaddr *nam, struct thread *td) so->so_error = 0; error = so_pru_connect(so, nam, td); } - crit_exit(); return (error); } @@ -480,9 +490,7 @@ soconnect2(struct socket *so1, struct socket *so2) { int error; - crit_enter(); error = so_pru_connect2(so1, so2); - crit_exit(); return (error); } @@ -491,7 +499,6 @@ sodisconnect(struct socket *so) { int error; - crit_enter(); if ((so->so_state & SS_ISCONNECTED) == 0) { error = ENOTCONN; goto bad; @@ -502,7 +509,6 @@ sodisconnect(struct socket *so) } error = so_pru_disconnect(so); bad: - crit_exit(); return (error); } @@ -568,7 +574,7 @@ sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, td->td_lwp->lwp_ru.ru_msgsnd++; if (control) clen = control->m_len; -#define gotoerr(errcode) { error = errcode; crit_exit(); goto release; } +#define gotoerr(errcode) { error = errcode; goto release; } restart: error = ssb_lock(&so->so_snd, SBLOCKWAIT(flags)); @@ -576,13 +582,11 @@ restart: goto out; do { - crit_enter(); if (so->so_state & SS_CANTSENDMORE) gotoerr(EPIPE); if (so->so_error) { error = so->so_error; so->so_error = 0; - crit_exit(); goto release; } if ((so->so_state & SS_ISCONNECTED) == 0) { @@ -614,12 +618,10 @@ restart: gotoerr(EWOULDBLOCK); ssb_unlock(&so->so_snd); error = ssb_wait(&so->so_snd); - crit_exit(); if (error) goto out; goto restart; } - crit_exit(); mp = ⊤ space -= clen; do { @@ -682,7 +684,6 @@ restart: } else { pru_flags = 0; } - crit_enter(); /* * XXX all the SS_CANTSENDMORE checks previously * done could be out of date. We could have recieved @@ -693,7 +694,6 @@ restart: * also happens. We must rethink this. */ error = so_pru_send(so, pru_flags, top, addr, control, td); - crit_exit(); if (dontroute) so->so_options &= ~SO_DONTROUTE; clen = 0; @@ -749,13 +749,11 @@ restart: if (error) goto out; - crit_enter(); if (so->so_state & SS_CANTSENDMORE) gotoerr(EPIPE); if (so->so_error) { error = so->so_error; so->so_error = 0; - crit_exit(); goto release; } if (!(so->so_state & SS_ISCONNECTED) && addr == NULL) @@ -768,12 +766,10 @@ restart: gotoerr(EWOULDBLOCK); ssb_unlock(&so->so_snd); error = ssb_wait(&so->so_snd); - crit_exit(); if (error) goto out; goto restart; } - crit_exit(); if (uio) { top = m_uiomove(uio); @@ -801,15 +797,16 @@ out: /* * Implement receive operations on a socket. + * * We depend on the way that records are added to the signalsockbuf * by sbappend*. In particular, each record (mbufs linked through m_next) * must begin with an address if the protocol so specifies, * followed by an optional mbuf or mbufs containing ancillary data, * and then zero or more mbufs of data. - * In order to avoid blocking network interrupts for the entire time here, - * we exit the critical section while doing the actual copy to user space. - * Although the signalsockbuf is locked, new data may still be appended, - * and thus we must maintain consistency of the signalsockbuf during that time. + * + * Although the signalsockbuf is locked, new data may still be appended. + * A token inside the ssb_lock deals with MP issues and still allows + * the network to access the socket if we block in a uio. * * The caller may receive the data as a single mbuf chain by supplying * an mbuf **mp0 for use in returning the chain. The uio is then used @@ -872,7 +869,6 @@ bad: so_pru_rcvd(so, 0); restart: - crit_enter(); error = ssb_lock(&so->so_rcv, SBLOCKWAIT(flags)); if (error) goto done; @@ -930,7 +926,6 @@ restart: error = ssb_wait(&so->so_rcv); if (error) goto done; - crit_exit(); goto restart; } dontblock: @@ -1025,7 +1020,7 @@ dontblock: else KASSERT(m->m_type == MT_DATA || m->m_type == MT_HEADER, ("receive 3")); - so->so_state &= ~SS_RCVATMARK; + soclrstate(so, SS_RCVATMARK); len = (resid > INT_MAX) ? INT_MAX : resid; if (so->so_oobmark && len > so->so_oobmark - offset) len = so->so_oobmark - offset; @@ -1038,11 +1033,9 @@ dontblock: * with the resid here either way. */ if (uio) { - crit_exit(); uio->uio_resid = resid; error = uiomove(mtod(m, caddr_t) + moff, len, uio); resid = uio->uio_resid; - crit_enter(); if (error) goto release; } else { @@ -1089,7 +1082,7 @@ dontblock: if ((flags & MSG_PEEK) == 0) { so->so_oobmark -= len; if (so->so_oobmark == 0) { - so->so_state |= SS_RCVATMARK; + sosetstate(so, SS_RCVATMARK); break; } } else { @@ -1150,7 +1143,6 @@ dontblock: if (orig_resid == resid && orig_resid && (flags & MSG_EOR) == 0 && (so->so_state & SS_CANTRCVMORE) == 0) { ssb_unlock(&so->so_rcv); - crit_exit(); goto restart; } @@ -1159,7 +1151,6 @@ dontblock: release: ssb_unlock(&so->so_rcv); done: - crit_exit(); if (free_chain) m_freem(free_chain); return (error); @@ -1786,7 +1777,7 @@ filt_soread(struct knote *kn, long hint) if (kn->kn_sfflags & NOTE_LOWAT) return (kn->kn_data >= kn->kn_sdata); return ((kn->kn_data >= so->so_rcv.ssb_lowat) || - !TAILQ_EMPTY(&so->so_comp)); + !TAILQ_EMPTY(&so->so_comp)); } static void diff --git a/sys/kern/uipc_socket2.c b/sys/kern/uipc_socket2.c index 4ad52b7ce1..4369da5548 100644 --- a/sys/kern/uipc_socket2.c +++ b/sys/kern/uipc_socket2.c @@ -57,6 +57,7 @@ #include #include +#include int maxsockets; @@ -152,6 +153,7 @@ _ssb_lock(struct signalsockbuf *ssb) } else { if (atomic_cmpset_int(&ssb->ssb_flags, flags, flags | SSB_LOCK)) { + lwkt_gettoken(&ssb->ssb_token); error = 0; break; } @@ -212,8 +214,8 @@ ssbtoxsockbuf(struct signalsockbuf *ssb, struct xsockbuf *xsb) void soisconnecting(struct socket *so) { - so->so_state &= ~(SS_ISCONNECTED|SS_ISDISCONNECTING); - so->so_state |= SS_ISCONNECTING; + soclrstate(so, SS_ISCONNECTED | SS_ISDISCONNECTING); + sosetstate(so, SS_ISCONNECTING); } void @@ -221,8 +223,8 @@ soisconnected(struct socket *so) { struct socket *head = so->so_head; - so->so_state &= ~(SS_ISCONNECTING|SS_ISDISCONNECTING|SS_ISCONFIRMING); - so->so_state |= SS_ISCONNECTED; + soclrstate(so, SS_ISCONNECTING | SS_ISDISCONNECTING | SS_ISCONFIRMING); + sosetstate(so, SS_ISCONNECTED); if (head && (so->so_state & SS_INCOMP)) { if ((so->so_options & SO_ACCEPTFILTER) != 0) { so->so_upcall = head->so_accf->so_accept_filter->accf_callback; @@ -232,12 +234,19 @@ soisconnected(struct socket *so) so->so_upcall(so, so->so_upcallarg, 0); return; } + + /* + * Listen socket are not per-cpu. + */ + lwkt_gettoken(&head->so_rcv.ssb_token); TAILQ_REMOVE(&head->so_incomp, so, so_list); head->so_incqlen--; - so->so_state &= ~SS_INCOMP; + soclrstate(so, SS_INCOMP); TAILQ_INSERT_TAIL(&head->so_comp, so, so_list); head->so_qlen++; - so->so_state |= SS_COMP; + sosetstate(so, SS_COMP); + lwkt_reltoken(&head->so_rcv.ssb_token); + sorwakeup(head); wakeup_one(&head->so_timeo); } else { @@ -250,8 +259,8 @@ soisconnected(struct socket *so) void soisdisconnecting(struct socket *so) { - so->so_state &= ~SS_ISCONNECTING; - so->so_state |= (SS_ISDISCONNECTING|SS_CANTRCVMORE|SS_CANTSENDMORE); + soclrstate(so, SS_ISCONNECTING); + sosetstate(so, SS_ISDISCONNECTING | SS_CANTRCVMORE | SS_CANTSENDMORE); wakeup((caddr_t)&so->so_timeo); sowwakeup(so); sorwakeup(so); @@ -260,8 +269,8 @@ soisdisconnecting(struct socket *so) void soisdisconnected(struct socket *so) { - so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING); - so->so_state |= (SS_CANTRCVMORE|SS_CANTSENDMORE|SS_ISDISCONNECTED); + soclrstate(so, SS_ISCONNECTING | SS_ISCONNECTED | SS_ISDISCONNECTING); + sosetstate(so, SS_CANTRCVMORE | SS_CANTSENDMORE | SS_ISDISCONNECTED); wakeup((caddr_t)&so->so_timeo); sbdrop(&so->so_snd.sb, so->so_snd.ssb_cc); sowwakeup(so); @@ -271,15 +280,15 @@ soisdisconnected(struct socket *so) void soisreconnecting(struct socket *so) { - so->so_state &= ~(SS_ISDISCONNECTING|SS_ISDISCONNECTED|SS_CANTRCVMORE| - SS_CANTSENDMORE); - so->so_state |= SS_ISCONNECTING; + soclrstate(so, SS_ISDISCONNECTING | SS_ISDISCONNECTED | + SS_CANTRCVMORE | SS_CANTSENDMORE); + sosetstate(so, SS_ISCONNECTING); } void soisreconnected(struct socket *so) { - so->so_state &= ~(SS_ISDISCONNECTED|SS_CANTRCVMORE|SS_CANTSENDMORE); + soclrstate(so, SS_ISDISCONNECTED | SS_CANTRCVMORE | SS_CANTSENDMORE); soisconnected(so); } @@ -301,6 +310,9 @@ sosetport(struct socket *so, lwkt_port_t port) * then we allocate a new structure, propoerly linked into the * data structure of the original socket, and return this. * Connstatus may be 0, or SO_ISCONFIRMING, or SO_ISCONNECTED. + * + * The new socket is returned with one ref and so_pcb assigned. + * The reference is implied by so_pcb. */ struct socket * sonewconn(struct socket *head, int connstatus) @@ -320,18 +332,31 @@ sonewconn(struct socket *head, int connstatus) so->so_type = head->so_type; so->so_options = head->so_options &~ SO_ACCEPTCONN; so->so_linger = head->so_linger; + + /* + * NOTE: Clearing NOFDREF implies referencing the so with + * soreference(). + */ so->so_state = head->so_state | SS_NOFDREF; so->so_proto = head->so_proto; so->so_cred = crhold(head->so_cred); ai.sb_rlimit = NULL; ai.p_ucred = NULL; ai.fd_rdir = NULL; /* jail code cruft XXX JH */ - if (soreserve(so, head->so_snd.ssb_hiwat, head->so_rcv.ssb_hiwat, NULL) || - /* Directly call function since we're already at protocol level. */ + + /* + * Reserve space and call pru_attach. We can directl call the + * function since we're already in the protocol thread. + */ + if (soreserve(so, head->so_snd.ssb_hiwat, + head->so_rcv.ssb_hiwat, NULL) || (*so->so_proto->pr_usrreqs->pru_attach)(so, 0, &ai)) { - sodealloc(so); + so->so_head = NULL; + sofree(so); /* remove implied pcb ref */ return (NULL); } + KKASSERT(so->so_refs == 2); /* attach + our base ref */ + sofree(so); KKASSERT(so->so_port != NULL); so->so_rcv.ssb_lowat = head->so_rcv.ssb_lowat; so->so_snd.ssb_lowat = head->so_snd.ssb_lowat; @@ -341,27 +366,29 @@ sonewconn(struct socket *head, int connstatus) (SSB_AUTOSIZE | SSB_AUTOLOWAT); so->so_snd.ssb_flags |= head->so_snd.ssb_flags & (SSB_AUTOSIZE | SSB_AUTOLOWAT); + lwkt_gettoken(&head->so_rcv.ssb_token); if (connstatus) { TAILQ_INSERT_TAIL(&head->so_comp, so, so_list); - so->so_state |= SS_COMP; + sosetstate(so, SS_COMP); head->so_qlen++; } else { if (head->so_incqlen > head->so_qlimit) { sp = TAILQ_FIRST(&head->so_incomp); TAILQ_REMOVE(&head->so_incomp, sp, so_list); head->so_incqlen--; - sp->so_state &= ~SS_INCOMP; + soclrstate(sp, SS_INCOMP); sp->so_head = NULL; soaborta(sp); } TAILQ_INSERT_TAIL(&head->so_incomp, so, so_list); - so->so_state |= SS_INCOMP; + sosetstate(so, SS_INCOMP); head->so_incqlen++; } + lwkt_reltoken(&head->so_rcv.ssb_token); if (connstatus) { sorwakeup(head); wakeup((caddr_t)&head->so_timeo); - so->so_state |= connstatus; + sosetstate(so, connstatus); } return (so); } @@ -378,14 +405,14 @@ sonewconn(struct socket *head, int connstatus) void socantsendmore(struct socket *so) { - so->so_state |= SS_CANTSENDMORE; + sosetstate(so, SS_CANTSENDMORE); sowwakeup(so); } void socantrcvmore(struct socket *so) { - so->so_state |= SS_CANTRCVMORE; + sosetstate(so, SS_CANTRCVMORE); sorwakeup(so); } diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c index e378300338..192c5cc778 100644 --- a/sys/kern/uipc_syscalls.c +++ b/sys/kern/uipc_syscalls.c @@ -225,6 +225,7 @@ soaccept_predicate(struct netmsg *msg0) msg->nm_netmsg.nm_lmsg.ms_error = head->so_error; return (TRUE); } + lwkt_gettoken(&head->so_rcv.ssb_token); if (!TAILQ_EMPTY(&head->so_comp)) { /* Abuse nm_so field as copy in/copy out parameter. XXX JH */ msg->nm_so = TAILQ_FIRST(&head->so_comp); @@ -232,8 +233,10 @@ soaccept_predicate(struct netmsg *msg0) head->so_qlen--; msg->nm_netmsg.nm_lmsg.ms_error = 0; + lwkt_reltoken(&head->so_rcv.ssb_token); return (TRUE); } + lwkt_reltoken(&head->so_rcv.ssb_token); if (head->so_state & SS_CANTRCVMORE) { msg->nm_netmsg.nm_lmsg.ms_error = ECONNABORTED; return (TRUE); @@ -313,7 +316,7 @@ kern_accept(int s, int fflags, struct sockaddr **name, int *namelen, int *res) /* connection has been removed from the listen queue */ KNOTE(&head->so_rcv.ssb_kq.ki_note, 0); - so->so_state &= ~SS_COMP; + soclrstate(so, SS_COMP); so->so_head = NULL; if (head->so_sigio != NULL) fsetown(fgetown(head->so_sigio), &so->so_sigio); @@ -517,7 +520,7 @@ kern_connect(int s, int fflags, struct sockaddr *sa) } bad: if (!interrupted) - so->so_state &= ~SS_ISCONNECTING; + soclrstate(so, SS_ISCONNECTING); if (error == ERESTART) error = EINTR; done: @@ -1880,8 +1883,8 @@ sys_sctp_peeloff(struct sctp_peeloff_args *uap) */ goto noconnection; } - so->so_state &= ~SS_COMP; - so->so_state &= ~SS_NOFDREF; + soreference(so); /* reference needed */ + soclrstate(so, SS_NOFDREF | SS_COMP); /* when clearing NOFDREF */ so->so_head = NULL; if (head->so_sigio != NULL) fsetown(fgetown(head->so_sigio), &so->so_sigio); diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c index 5ec5ea55a3..17e77196c6 100644 --- a/sys/kern/uipc_usrreq.c +++ b/sys/kern/uipc_usrreq.c @@ -56,9 +56,10 @@ #include #include #include + #include #include - +#include static MALLOC_DEFINE(M_UNPCB, "unpcb", "unpcb struct"); static unp_gen_t unp_gencnt; @@ -66,6 +67,8 @@ static u_int unp_count; static struct unp_head unp_shead, unp_dhead; +static struct lwkt_token unp_token = LWKT_TOKEN_MP_INITIALIZER(unp_token); + /* * Unix communications domain. * @@ -100,26 +103,41 @@ static int unp_internalize (struct mbuf *, struct thread *); static int unp_listen (struct unpcb *, struct thread *); static void unp_fp_externalize(struct lwp *lp, struct file *fp, int fd); +/* + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. + */ static int uipc_abort(struct socket *so) { - struct unpcb *unp = so->so_pcb; + struct unpcb *unp; + int error; - if (unp == NULL) - return EINVAL; - unp_drop(unp, ECONNABORTED); - unp_detach(unp); - sofree(so); - return 0; + lwkt_gettoken(&unp_token); + unp = so->so_pcb; + if (unp) { + unp_drop(unp, ECONNABORTED); + unp_detach(unp); + error = 0; + } else { + error = EINVAL; + } + lwkt_reltoken(&unp_token); + + return error; } static int uipc_accept(struct socket *so, struct sockaddr **nam) { - struct unpcb *unp = so->so_pcb; + struct unpcb *unp; - if (unp == NULL) + lwkt_gettoken(&unp_token); + unp = so->so_pcb; + if (unp == NULL) { + lwkt_reltoken(&unp_token); return EINVAL; + } /* * Pass back name of connected socket, @@ -131,48 +149,76 @@ uipc_accept(struct socket *so, struct sockaddr **nam) } else { *nam = dup_sockaddr((struct sockaddr *)&sun_noname); } + lwkt_reltoken(&unp_token); return 0; } static int uipc_attach(struct socket *so, int proto, struct pru_attach_info *ai) { - struct unpcb *unp = so->so_pcb; + struct unpcb *unp; + int error; + + lwkt_gettoken(&unp_token); + unp = so->so_pcb; + if (unp) + error = EISCONN; + else + error = unp_attach(so, ai); + lwkt_reltoken(&unp_token); - if (unp != NULL) - return EISCONN; - return unp_attach(so, ai); + return error; } static int uipc_bind(struct socket *so, struct sockaddr *nam, struct thread *td) { - struct unpcb *unp = so->so_pcb; + struct unpcb *unp; + int error; - if (unp == NULL) - return EINVAL; - return unp_bind(unp, nam, td); + lwkt_gettoken(&unp_token); + unp = so->so_pcb; + if (unp) + error = unp_bind(unp, nam, td); + else + error = EINVAL; + lwkt_reltoken(&unp_token); + + return error; } static int uipc_connect(struct socket *so, struct sockaddr *nam, struct thread *td) { - struct unpcb *unp = so->so_pcb; + struct unpcb *unp; + int error; - if (unp == NULL) - return EINVAL; - return unp_connect(so, nam, td); + lwkt_gettoken(&unp_token); + unp = so->so_pcb; + if (unp) + error = unp_connect(so, nam, td); + else + error = EINVAL; + lwkt_reltoken(&unp_token); + + return error; } static int uipc_connect2(struct socket *so1, struct socket *so2) { - struct unpcb *unp = so1->so_pcb; + struct unpcb *unp; + int error; - if (unp == NULL) - return EINVAL; + lwkt_gettoken(&unp_token); + unp = so1->so_pcb; + if (unp) + error = unp_connect2(so1, so2); + else + error = EINVAL; + lwkt_reltoken(&unp_token); - return unp_connect2(so1, so2); + return error; } /* control is EOPNOTSUPP */ @@ -180,69 +226,102 @@ uipc_connect2(struct socket *so1, struct socket *so2) static int uipc_detach(struct socket *so) { - struct unpcb *unp = so->so_pcb; + struct unpcb *unp; + int error; - if (unp == NULL) - return EINVAL; + lwkt_gettoken(&unp_token); + unp = so->so_pcb; + if (unp) { + unp_detach(unp); + error = 0; + } else { + error = EINVAL; + } + lwkt_reltoken(&unp_token); - unp_detach(unp); - return 0; + return error; } static int uipc_disconnect(struct socket *so) { - struct unpcb *unp = so->so_pcb; + struct unpcb *unp; + int error; - if (unp == NULL) - return EINVAL; - unp_disconnect(unp); - return 0; + lwkt_gettoken(&unp_token); + unp = so->so_pcb; + if (unp) { + unp_disconnect(unp); + error = 0; + } else { + error = EINVAL; + } + lwkt_reltoken(&unp_token); + + return error; } static int uipc_listen(struct socket *so, struct thread *td) { - struct unpcb *unp = so->so_pcb; + struct unpcb *unp; + int error; + lwkt_gettoken(&unp_token); + unp = so->so_pcb; if (unp == NULL || unp->unp_vnode == NULL) - return EINVAL; - return unp_listen(unp, td); + error = EINVAL; + else + error = unp_listen(unp, td); + lwkt_reltoken(&unp_token); + + return error; } static int uipc_peeraddr(struct socket *so, struct sockaddr **nam) { - struct unpcb *unp = so->so_pcb; + struct unpcb *unp; + int error; - if (unp == NULL) - return EINVAL; - if (unp->unp_conn && unp->unp_conn->unp_addr) + lwkt_gettoken(&unp_token); + unp = so->so_pcb; + if (unp == NULL) { + error = EINVAL; + } else if (unp->unp_conn && unp->unp_conn->unp_addr) { *nam = dup_sockaddr((struct sockaddr *)unp->unp_conn->unp_addr); - else { + error = 0; + } else { /* * XXX: It seems that this test always fails even when * connection is established. So, this else clause is * added as workaround to return PF_LOCAL sockaddr. */ *nam = dup_sockaddr((struct sockaddr *)&sun_noname); + error = 0; } - return 0; + lwkt_reltoken(&unp_token); + + return error; } static int uipc_rcvd(struct socket *so, int flags) { - struct unpcb *unp = so->so_pcb; + struct unpcb *unp; struct socket *so2; - if (unp == NULL) + lwkt_gettoken(&unp_token); + unp = so->so_pcb; + if (unp == NULL) { + lwkt_reltoken(&unp_token); return EINVAL; + } + switch (so->so_type) { case SOCK_DGRAM: panic("uipc_rcvd DGRAM?"); /*NOTREACHED*/ - case SOCK_STREAM: case SOCK_SEQPACKET: if (unp->unp_conn == NULL) @@ -260,10 +339,12 @@ uipc_rcvd(struct socket *so, int flags) sowwakeup(so2); } break; - default: panic("uipc_rcvd unknown socktype"); + /*NOTREACHED*/ } + lwkt_reltoken(&unp_token); + return 0; } @@ -273,10 +354,13 @@ static int uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam, struct mbuf *control, struct thread *td) { - int error = 0; - struct unpcb *unp = so->so_pcb; + struct unpcb *unp; struct socket *so2; + int error = 0; + lwkt_gettoken(&unp_token); + + unp = so->so_pcb; if (unp == NULL) { error = EINVAL; goto release; @@ -397,6 +481,8 @@ uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam, unp_dispose(control); release: + lwkt_reltoken(&unp_token); + if (control) m_freem(control); if (m) @@ -410,10 +496,14 @@ release: static int uipc_sense(struct socket *so, struct stat *sb) { - struct unpcb *unp = so->so_pcb; + struct unpcb *unp; - if (unp == NULL) + lwkt_gettoken(&unp_token); + unp = so->so_pcb; + if (unp == NULL) { + lwkt_reltoken(&unp_token); return EINVAL; + } sb->st_blksize = so->so_snd.ssb_hiwat; sb->st_dev = NOUDEV; if (unp->unp_ino == 0) { /* make up a non-zero inode number */ @@ -422,31 +512,49 @@ uipc_sense(struct socket *so, struct stat *sb) spin_unlock(&unp_ino_spin); } sb->st_ino = unp->unp_ino; + lwkt_reltoken(&unp_token); + return (0); } static int uipc_shutdown(struct socket *so) { - struct unpcb *unp = so->so_pcb; + struct unpcb *unp; + int error; - if (unp == NULL) - return EINVAL; - socantsendmore(so); - unp_shutdown(unp); - return 0; + lwkt_gettoken(&unp_token); + unp = so->so_pcb; + if (unp) { + socantsendmore(so); + unp_shutdown(unp); + error = 0; + } else { + error = EINVAL; + } + lwkt_reltoken(&unp_token); + + return error; } static int uipc_sockaddr(struct socket *so, struct sockaddr **nam) { - struct unpcb *unp = so->so_pcb; + struct unpcb *unp; + int error; - if (unp == NULL) - return EINVAL; - if (unp->unp_addr) - *nam = dup_sockaddr((struct sockaddr *)unp->unp_addr); - return 0; + lwkt_gettoken(&unp_token); + unp = so->so_pcb; + if (unp) { + if (unp->unp_addr) + *nam = dup_sockaddr((struct sockaddr *)unp->unp_addr); + error = 0; + } else { + error = EINVAL; + } + lwkt_reltoken(&unp_token); + + return error; } struct pr_usrreqs uipc_usrreqs = { @@ -474,9 +582,12 @@ struct pr_usrreqs uipc_usrreqs = { int uipc_ctloutput(struct socket *so, struct sockopt *sopt) { - struct unpcb *unp = so->so_pcb; + struct unpcb *unp; int error = 0; + lwkt_gettoken(&unp_token); + unp = so->so_pcb; + switch (sopt->sopt_dir) { case SOPT_GET: switch (sopt->sopt_name) { @@ -503,6 +614,8 @@ uipc_ctloutput(struct socket *so, struct sockopt *sopt) error = EOPNOTSUPP; break; } + lwkt_reltoken(&unp_token); + return (error); } @@ -551,6 +664,7 @@ unp_attach(struct socket *so, struct pru_attach_info *ai) struct unpcb *unp; int error; + lwkt_gettoken(&unp_token); if (so->so_snd.ssb_hiwat == 0 || so->so_rcv.ssb_hiwat == 0) { switch (so->so_type) { @@ -569,11 +683,13 @@ unp_attach(struct socket *so, struct pru_attach_info *ai) panic("unp_attach"); } if (error) - return (error); + goto failed; } unp = kmalloc(sizeof(*unp), M_UNPCB, M_NOWAIT|M_ZERO); - if (unp == NULL) - return (ENOBUFS); + if (unp == NULL) { + error = ENOBUFS; + goto failed; + } unp->unp_gencnt = ++unp_gencnt; unp_count++; LIST_INIT(&unp->unp_refs); @@ -582,13 +698,21 @@ unp_attach(struct socket *so, struct pru_attach_info *ai) LIST_INSERT_HEAD(so->so_type == SOCK_DGRAM ? &unp_dhead : &unp_shead, unp, unp_link); so->so_pcb = (caddr_t)unp; + soreference(so); so->so_port = sync_soport(so, NULL, NULL); - return (0); + error = 0; +failed: + lwkt_reltoken(&unp_token); + return error; } static void unp_detach(struct unpcb *unp) { + struct socket *so; + + lwkt_gettoken(&unp_token); + LIST_REMOVE(unp, unp_link); unp->unp_gencnt = ++unp_gencnt; --unp_count; @@ -602,7 +726,12 @@ unp_detach(struct unpcb *unp) while (!LIST_EMPTY(&unp->unp_refs)) unp_drop(LIST_FIRST(&unp->unp_refs), ECONNRESET); soisdisconnected(unp->unp_socket); - unp->unp_socket->so_pcb = NULL; + so = unp->unp_socket; + soreference(so); /* for delayed sorflush */ + so->so_pcb = NULL; + unp->unp_socket = NULL; + sofree(so); /* remove pcb ref */ + if (unp_rights) { /* * Normally the receive buffer is flushed later, @@ -611,9 +740,12 @@ unp_detach(struct unpcb *unp) * of those descriptor references after the garbage collector * gets them (resulting in a "panic: closef: count < 0"). */ - sorflush(unp->unp_socket); + sorflush(so); unp_gc(); } + sofree(so); + lwkt_reltoken(&unp_token); + if (unp->unp_addr) kfree(unp->unp_addr, M_SONAME); kfree(unp, M_UNPCB); @@ -630,11 +762,16 @@ unp_bind(struct unpcb *unp, struct sockaddr *nam, struct thread *td) struct nlookupdata nd; char buf[SOCK_MAXADDRLEN]; - if (unp->unp_vnode != NULL) - return (EINVAL); + lwkt_gettoken(&unp_token); + if (unp->unp_vnode != NULL) { + error = EINVAL; + goto failed; + } namelen = soun->sun_len - offsetof(struct sockaddr_un, sun_path); - if (namelen <= 0) - return (EINVAL); + if (namelen <= 0) { + error = EINVAL; + goto failed; + } strncpy(buf, soun->sun_path, namelen); buf[namelen] = 0; /* null-terminate the string */ error = nlookup_init(&nd, buf, UIO_SYSSPACE, @@ -658,6 +795,8 @@ unp_bind(struct unpcb *unp, struct sockaddr *nam, struct thread *td) } done: nlookup_done(&nd); +failed: + lwkt_reltoken(&unp_token); return (error); } @@ -673,11 +812,13 @@ unp_connect(struct socket *so, struct sockaddr *nam, struct thread *td) struct nlookupdata nd; char buf[SOCK_MAXADDRLEN]; - KKASSERT(p); + lwkt_gettoken(&unp_token); len = nam->sa_len - offsetof(struct sockaddr_un, sun_path); - if (len <= 0) - return EINVAL; + if (len <= 0) { + error = EINVAL; + goto failed; + } strncpy(buf, soun->sun_path, len); buf[len] = 0; @@ -689,7 +830,7 @@ unp_connect(struct socket *so, struct sockaddr *nam, struct thread *td) error = cache_vget(&nd.nl_nch, nd.nl_cred, LK_EXCLUSIVE, &vp); nlookup_done(&nd); if (error) - return (error); + goto failed; if (vp->v_type != VSOCK) { error = ENOTSOCK; @@ -747,21 +888,27 @@ unp_connect(struct socket *so, struct sockaddr *nam, struct thread *td) error = unp_connect2(so, so2); bad: vput(vp); +failed: + lwkt_reltoken(&unp_token); return (error); } int unp_connect2(struct socket *so, struct socket *so2) { - struct unpcb *unp = so->so_pcb; + struct unpcb *unp; struct unpcb *unp2; - if (so2->so_type != so->so_type) + lwkt_gettoken(&unp_token); + unp = so->so_pcb; + if (so2->so_type != so->so_type) { + lwkt_reltoken(&unp_token); return (EPROTOTYPE); + } unp2 = so2->so_pcb; unp->unp_conn = unp2; - switch (so->so_type) { + switch (so->so_type) { case SOCK_DGRAM: LIST_INSERT_HEAD(&unp2->unp_refs, unp, unp_reflink); soisconnected(so); @@ -777,23 +924,29 @@ unp_connect2(struct socket *so, struct socket *so2) default: panic("unp_connect2"); } + lwkt_reltoken(&unp_token); return (0); } static void unp_disconnect(struct unpcb *unp) { - struct unpcb *unp2 = unp->unp_conn; + struct unpcb *unp2; - if (unp2 == NULL) + lwkt_gettoken(&unp_token); + + unp2 = unp->unp_conn; + if (unp2 == NULL) { + lwkt_reltoken(&unp_token); return; + } unp->unp_conn = NULL; switch (unp->unp_socket->so_type) { case SOCK_DGRAM: LIST_REMOVE(unp, unp_reflink); - unp->unp_socket->so_state &= ~SS_ISCONNECTED; + soclrstate(unp->unp_socket, SS_ISCONNECTED); break; case SOCK_STREAM: case SOCK_SEQPACKET: @@ -802,14 +955,16 @@ unp_disconnect(struct unpcb *unp) soisdisconnected(unp2->unp_socket); break; } + lwkt_reltoken(&unp_token); } #ifdef notdef void unp_abort(struct unpcb *unp) { - + lwkt_gettoken(&unp_token); unp_detach(unp); + lwkt_reltoken(&unp_token); } #endif @@ -854,6 +1009,8 @@ unp_pcblist(SYSCTL_HANDLER_ARGS) if (req->newptr != NULL) return EPERM; + lwkt_gettoken(&unp_token); + /* * OK, now we're committed to doing something. */ @@ -892,7 +1049,9 @@ unp_pcblist(SYSCTL_HANDLER_ARGS) error = SYSCTL_OUT(req, &xu, sizeof xu); } } + lwkt_reltoken(&unp_token); kfree(unp_list, M_TEMP); + return error; } @@ -931,7 +1090,8 @@ unp_drop(struct unpcb *unp, int err) void unp_drain(void) { - + lwkt_gettoken(&unp_token); + lwkt_reltoken(&unp_token); } #endif @@ -950,6 +1110,8 @@ unp_externalize(struct mbuf *rights) / sizeof (struct file *); int f; + lwkt_gettoken(&unp_token); + /* * if the new FD's will not fit, then we free them all */ @@ -964,6 +1126,7 @@ unp_externalize(struct mbuf *rights) *rp++ = 0; unp_discard(fp, NULL); } + lwkt_reltoken(&unp_token); return (EMSGSIZE); } @@ -1006,6 +1169,8 @@ unp_externalize(struct mbuf *rights) */ cm->cmsg_len = CMSG_LEN(newfds * sizeof(int)); rights->m_len = cm->cmsg_len; + + lwkt_reltoken(&unp_token); return (0); } @@ -1015,6 +1180,8 @@ unp_fp_externalize(struct lwp *lp, struct file *fp, int fd) struct file *fx; int error; + lwkt_gettoken(&unp_token); + if (lp) { KKASSERT(fd >= 0); if (fp->f_flag & FREVOKED) { @@ -1035,6 +1202,8 @@ unp_fp_externalize(struct lwp *lp, struct file *fp, int fd) unp_rights--; spin_unlock(&unp_spin); fdrop(fp); + + lwkt_reltoken(&unp_token); } @@ -1058,13 +1227,17 @@ unp_internalize(struct mbuf *control, struct thread *td) struct cmsgcred *cmcred; int oldfds; u_int newlen; + int error; KKASSERT(p); + lwkt_gettoken(&unp_token); + fdescp = p->p_fd; if ((cm->cmsg_type != SCM_RIGHTS && cm->cmsg_type != SCM_CREDS) || cm->cmsg_level != SOL_SOCKET || CMSG_ALIGN(cm->cmsg_len) != control->m_len) { - return (EINVAL); + error = EINVAL; + goto done; } /* @@ -1080,15 +1253,18 @@ unp_internalize(struct mbuf *control, struct thread *td) CMGROUP_MAX); for (i = 0; i < cmcred->cmcred_ngroups; i++) cmcred->cmcred_groups[i] = p->p_ucred->cr_groups[i]; - return(0); + error = 0; + goto done; } /* * cmsghdr may not be aligned, do not allow calculation(s) to * go negative. */ - if (cm->cmsg_len < CMSG_LEN(0)) - return(EINVAL); + if (cm->cmsg_len < CMSG_LEN(0)) { + error = EINVAL; + goto done; + } oldfds = (cm->cmsg_len - CMSG_LEN(0)) / sizeof (int); @@ -1100,10 +1276,14 @@ unp_internalize(struct mbuf *control, struct thread *td) for (i = 0; i < oldfds; i++) { fd = *fdp++; if ((unsigned)fd >= fdescp->fd_nfiles || - fdescp->fd_files[fd].fp == NULL) - return (EBADF); - if (fdescp->fd_files[fd].fp->f_type == DTYPE_KQUEUE) - return (EOPNOTSUPP); + fdescp->fd_files[fd].fp == NULL) { + error = EBADF; + goto done; + } + if (fdescp->fd_files[fd].fp->f_type == DTYPE_KQUEUE) { + error = EOPNOTSUPP; + goto done; + } } /* * Now replace the integer FDs with pointers to @@ -1112,14 +1292,20 @@ unp_internalize(struct mbuf *control, struct thread *td) * enough, return E2BIG. */ newlen = CMSG_LEN(oldfds * sizeof(struct file *)); - if (newlen > MCLBYTES) - return (E2BIG); + if (newlen > MCLBYTES) { + error = E2BIG; + goto done; + } if (newlen - control->m_len > M_TRAILINGSPACE(control)) { - if (control->m_flags & M_EXT) - return (E2BIG); + if (control->m_flags & M_EXT) { + error = E2BIG; + goto done; + } MCLGET(control, MB_WAIT); - if (!(control->m_flags & M_EXT)) - return (ENOBUFS); + if (!(control->m_flags & M_EXT)) { + error = ENOBUFS; + goto done; + } /* copy the data to the cluster */ memcpy(mtod(control, char *), cm, cm->cmsg_len); @@ -1166,7 +1352,10 @@ unp_internalize(struct mbuf *control, struct thread *td) spin_unlock(&unp_spin); } } - return (0); + error = 0; +done: + lwkt_reltoken(&unp_token); + return error; } /* @@ -1201,6 +1390,8 @@ unp_gc(void) unp_gcing = TRUE; spin_unlock(&unp_spin); + lwkt_gettoken(&unp_token); + /* * before going through all this, set all FDs to * be NOT defered and NOT externally accessible @@ -1271,6 +1462,9 @@ unp_gc(void) for (i = info.index, fpp = info.extra_ref; --i >= 0; ++fpp) closef(*fpp, NULL); } while (info.index == info.maxindex); + + lwkt_reltoken(&unp_token); + kfree((caddr_t)info.extra_ref, M_FILE); unp_gcing = FALSE; } @@ -1416,6 +1610,7 @@ unp_revoke_gc(struct file *fx) struct unp_revoke_gc_info info; int i; + lwkt_gettoken(&unp_token); info.fx = fx; do { info.fcount = 0; @@ -1423,6 +1618,7 @@ unp_revoke_gc(struct file *fx) for (i = 0; i < info.fcount; ++i) unp_fp_externalize(NULL, info.fary[i], -1); } while (info.fcount == REVOKE_GC_MAXFILES); + lwkt_reltoken(&unp_token); } /* @@ -1505,8 +1701,10 @@ unp_revoke_gc_check(struct file *fps, void *vinfo) void unp_dispose(struct mbuf *m) { + lwkt_gettoken(&unp_token); if (m) unp_scan(m, unp_discard, NULL); + lwkt_reltoken(&unp_token); } static int @@ -1515,8 +1713,10 @@ unp_listen(struct unpcb *unp, struct thread *td) struct proc *p = td->td_proc; KKASSERT(p); + lwkt_gettoken(&unp_token); cru2x(p->p_ucred, &unp->unp_peercred); unp->unp_flags |= UNP_HAVEPCCACHED; + lwkt_reltoken(&unp_token); return (0); } diff --git a/sys/net/ip_mroute/ip_mroute.c b/sys/net/ip_mroute/ip_mroute.c index 96a91bd2a7..c7ab5393cc 100644 --- a/sys/net/ip_mroute/ip_mroute.c +++ b/sys/net/ip_mroute/ip_mroute.c @@ -100,6 +100,9 @@ SYSCTL_OPAQUE(_net_inet_ip, OID_AUTO, viftable, CTLFLAG_RD, static u_char nexpire[MFCTBLSIZ]; +struct lwkt_token mroute_token = LWKT_TOKEN_MP_INITIALIZER(mroute_token); + + static struct callout expire_upcalls_ch; static struct callout tbf_reprocess_q_ch; #define EXPIRE_TIMEOUT (hz / 4) /* 4x / second */ @@ -519,16 +522,17 @@ get_sg_cnt(struct sioc_sg_req *req) { struct mfc *rt; - crit_enter(); + lwkt_gettoken(&mroute_token); rt = mfc_find(req->src.s_addr, req->grp.s_addr); - crit_exit(); if (rt == NULL) { req->pktcnt = req->bytecnt = req->wrong_if = 0xffffffff; + lwkt_reltoken(&mroute_token); return EADDRNOTAVAIL; } req->pktcnt = rt->mfc_pkt_cnt; req->bytecnt = rt->mfc_byte_cnt; req->wrong_if = rt->mfc_wrong_if; + lwkt_reltoken(&mroute_token); return 0; } @@ -610,7 +614,7 @@ X_ip_mrouter_done(void) struct mfc *rt; struct rtdetq *rte; - crit_enter(); + lwkt_gettoken(&mroute_token); /* * For each phyint in use, disable promiscuous reception of all IP @@ -677,7 +681,7 @@ X_ip_mrouter_done(void) ip_mrouter = NULL; - crit_exit(); + lwkt_reltoken(&mroute_token); if (mrtdebug) log(LOG_DEBUG, "ip_mrouter_done\n"); @@ -816,14 +820,14 @@ add_vif(struct vifctl *vifcp) return EOPNOTSUPP; /* Enable promiscuous reception of all IP multicasts from the if */ - crit_enter(); + lwkt_gettoken(&mroute_token); error = if_allmulti(ifp, 1); - crit_exit(); + lwkt_reltoken(&mroute_token); if (error) return error; } - crit_enter(); + lwkt_gettoken(&mroute_token); /* define parameters for the tbf structure */ vifp->v_tbf = v_tbf; GET_TIME(vifp->v_tbf->tbf_last_pkt_t); @@ -846,11 +850,12 @@ add_vif(struct vifctl *vifcp) vifp->v_pkt_out = 0; vifp->v_bytes_in = 0; vifp->v_bytes_out = 0; - crit_exit(); /* Adjust numvifs up if the vifi is higher than numvifs */ if (numvifs <= vifcp->vifc_vifi) numvifs = vifcp->vifc_vifi + 1; + lwkt_reltoken(&mroute_token); + if (mrtdebug) log(LOG_DEBUG, "add_vif #%d, lcladdr %lx, %s %lx, thresh %x, rate %d\n", vifcp->vifc_vifi, @@ -877,7 +882,7 @@ del_vif(vifi_t vifi) if (vifp->v_lcl_addr.s_addr == INADDR_ANY) return EADDRNOTAVAIL; - crit_enter(); + lwkt_gettoken(&mroute_token); if (!(vifp->v_flags & (VIFF_TUNNEL | VIFF_REGISTER))) if_allmulti(vifp->v_ifp, 0); @@ -914,7 +919,7 @@ del_vif(vifi_t vifi) break; numvifs = vifi; - crit_exit(); + lwkt_reltoken(&mroute_token); return 0; } @@ -980,16 +985,16 @@ add_mfc(struct mfcctl2 *mfccp) (u_long)ntohl(mfccp->mfcc_mcastgrp.s_addr), mfccp->mfcc_parent); - crit_enter(); + lwkt_gettoken(&mroute_token); update_mfc_params(rt, mfccp); - crit_exit(); + lwkt_reltoken(&mroute_token); return 0; } /* * Find the entry for which the upcall was made and update */ - crit_enter(); + lwkt_gettoken(&mroute_token); hash = MFCHASH(mfccp->mfcc_origin.s_addr, mfccp->mfcc_mcastgrp.s_addr); for (rt = mfctable[hash], nstl = 0; rt; rt = rt->mfc_next) { @@ -1051,7 +1056,7 @@ add_mfc(struct mfcctl2 *mfccp) if (rt == NULL) { /* no upcall, so make a new entry */ rt = kmalloc(sizeof(*rt), M_MRTABLE, M_INTWAIT | M_NULLOK); if (rt == NULL) { - crit_exit(); + lwkt_reltoken(&mroute_token); return ENOBUFS; } @@ -1065,7 +1070,7 @@ add_mfc(struct mfcctl2 *mfccp) mfctable[hash] = rt; } } - crit_exit(); + lwkt_reltoken(&mroute_token); return 0; } @@ -1089,7 +1094,7 @@ del_mfc(struct mfcctl2 *mfccp) log(LOG_DEBUG,"del_mfc orig %lx mcastgrp %lx\n", (u_long)ntohl(origin.s_addr), (u_long)ntohl(mcastgrp.s_addr)); - crit_enter(); + lwkt_gettoken(&mroute_token); hash = MFCHASH(origin.s_addr, mcastgrp.s_addr); for (nptr = &mfctable[hash]; (rt = *nptr) != NULL; nptr = &rt->mfc_next) @@ -1098,7 +1103,7 @@ del_mfc(struct mfcctl2 *mfccp) rt->mfc_stall == NULL) break; if (rt == NULL) { - crit_exit(); + lwkt_reltoken(&mroute_token); return EADDRNOTAVAIL; } @@ -1109,11 +1114,9 @@ del_mfc(struct mfcctl2 *mfccp) */ list = rt->mfc_bw_meter; rt->mfc_bw_meter = NULL; + lwkt_reltoken(&mroute_token); kfree(rt, M_MRTABLE); - - crit_exit(); - free_bw_list(list); return 0; @@ -1212,14 +1215,15 @@ X_ip_mforward(struct ip *ip, struct ifnet *ifp, struct mbuf *m, /* * Determine forwarding vifs from the forwarding cache table */ - crit_enter(); + lwkt_gettoken(&mroute_token); ++mrtstat.mrts_mfc_lookups; rt = mfc_find(ip->ip_src.s_addr, ip->ip_dst.s_addr); /* Entry exists, so forward if necessary */ if (rt != NULL) { - crit_exit(); - return ip_mdq(m, ifp, rt, -1); + int ipres = ip_mdq(m, ifp, rt, -1); + lwkt_reltoken(&mroute_token); + return ipres; } else { /* * If we don't have a route for packet's origin, @@ -1246,7 +1250,7 @@ X_ip_mforward(struct ip *ip, struct ifnet *ifp, struct mbuf *m, */ rte = kmalloc((sizeof *rte), M_MRTABLE, M_INTWAIT | M_NULLOK); if (rte == NULL) { - crit_exit(); + lwkt_reltoken(&mroute_token); return ENOBUFS; } @@ -1255,7 +1259,7 @@ X_ip_mforward(struct ip *ip, struct ifnet *ifp, struct mbuf *m, mb0 = m_pullup(mb0, hlen); if (mb0 == NULL) { kfree(rte, M_MRTABLE); - crit_exit(); + lwkt_reltoken(&mroute_token); return ENOBUFS; } @@ -1314,7 +1318,7 @@ fail1: fail: kfree(rte, M_MRTABLE); m_freem(mb0); - crit_exit(); + lwkt_reltoken(&mroute_token); return ENOBUFS; } @@ -1356,7 +1360,7 @@ fail: non_fatal: kfree(rte, M_MRTABLE); m_freem(mb0); - crit_exit(); + lwkt_reltoken(&mroute_token); return 0; } @@ -1368,7 +1372,7 @@ non_fatal: rte->ifp = ifp; rte->next = NULL; - crit_exit(); + lwkt_reltoken(&mroute_token); return 0; } } @@ -1383,7 +1387,7 @@ expire_upcalls(void *unused) struct mfc *mfc, **nptr; int i; - crit_enter(); + lwkt_gettoken(&mroute_token); for (i = 0; i < MFCTBLSIZ; i++) { if (nexpire[i] == 0) continue; @@ -1432,7 +1436,7 @@ expire_upcalls(void *unused) } } callout_reset(&expire_upcalls_ch, EXPIRE_TIMEOUT, expire_upcalls, NULL); - crit_exit(); + lwkt_reltoken(&mroute_token); } /* @@ -1810,7 +1814,7 @@ tbf_queue(struct vif *vifp, struct mbuf *m) { struct tbf *t = vifp->v_tbf; - crit_enter(); + lwkt_gettoken(&mroute_token); if (t->tbf_t == NULL) /* Queue was empty */ t->tbf_q = m; @@ -1828,7 +1832,7 @@ tbf_queue(struct vif *vifp, struct mbuf *m) t->tbf_q_len++; - crit_exit(); + lwkt_reltoken(&mroute_token); } /* @@ -1839,7 +1843,7 @@ tbf_process_q(struct vif *vifp) { struct tbf *t = vifp->v_tbf; - crit_enter(); + lwkt_gettoken(&mroute_token); /* loop through the queue at the interface and send as many packets * as possible @@ -1861,7 +1865,7 @@ tbf_process_q(struct vif *vifp) m->m_nextpkt = NULL; tbf_send_packet(vifp, m); } - crit_exit(); + lwkt_reltoken(&mroute_token); } static void @@ -1889,7 +1893,7 @@ tbf_dq_sel(struct vif *vifp, struct ip *ip) struct mbuf **np; struct tbf *t = vifp->v_tbf; - crit_enter(); + lwkt_gettoken(&mroute_token); p = priority(vifp, ip); @@ -1905,21 +1909,21 @@ tbf_dq_sel(struct vif *vifp, struct ip *ip) /* It's impossible for the queue to be empty, but check anyways. */ if (--t->tbf_q_len == 0) t->tbf_t = NULL; - crit_exit(); mrtstat.mrts_drop_sel++; + lwkt_reltoken(&mroute_token); return 1; } np = &m->m_nextpkt; last = m; } - crit_exit(); + lwkt_reltoken(&mroute_token); return 0; } static void tbf_send_packet(struct vif *vifp, struct mbuf *m) { - crit_enter(); + lwkt_gettoken(&mroute_token); if (vifp->v_flags & VIFF_TUNNEL) /* If tunnel options */ ip_output(m, NULL, &vifp->v_route, IP_FORWARDING, NULL, NULL); @@ -1945,7 +1949,7 @@ tbf_send_packet(struct vif *vifp, struct mbuf *m) log(LOG_DEBUG, "phyint_send on vif %d err %d\n", (int)(vifp - viftable), error); } - crit_exit(); + lwkt_reltoken(&mroute_token); } /* determine the current time and then @@ -1959,7 +1963,7 @@ tbf_update_tokens(struct vif *vifp) u_long tm; struct tbf *t = vifp->v_tbf; - crit_enter(); + lwkt_gettoken(&mroute_token); GET_TIME(tp); @@ -1980,7 +1984,7 @@ tbf_update_tokens(struct vif *vifp) if (t->tbf_n_tok > MAX_BKT_SIZE) t->tbf_n_tok = MAX_BKT_SIZE; - crit_exit(); + lwkt_reltoken(&mroute_token); } static int @@ -2032,17 +2036,17 @@ X_ip_rsvp_vif(struct socket *so, struct sockopt *sopt) if (error) return error; - crit_enter(); + lwkt_gettoken(&mroute_token); if (vifi < 0 || vifi >= numvifs) { /* Error if vif is invalid */ - crit_exit(); + lwkt_reltoken(&mroute_token); return EADDRNOTAVAIL; } if (sopt->sopt_name == IP_RSVP_VIF_ON) { /* Check if socket is available. */ if (viftable[vifi].v_rsvpd != NULL) { - crit_exit(); + lwkt_reltoken(&mroute_token); return EADDRINUSE; } @@ -2070,7 +2074,7 @@ X_ip_rsvp_vif(struct socket *so, struct sockopt *sopt) rsvp_on--; } } - crit_exit(); + lwkt_reltoken(&mroute_token); return 0; } @@ -2083,7 +2087,7 @@ X_ip_rsvp_force_done(struct socket *so) if (so->so_type != SOCK_RAW || so->so_proto->pr_protocol != IPPROTO_RSVP) return; - crit_enter(); + lwkt_gettoken(&mroute_token); /* The socket may be attached to more than one vif...this * is perfectly legal. @@ -2101,7 +2105,7 @@ X_ip_rsvp_force_done(struct socket *so) } } - crit_exit(); + lwkt_reltoken(&mroute_token); } static void @@ -2137,7 +2141,7 @@ X_rsvp_input(struct mbuf *m, ...) return; } - crit_enter(); + lwkt_gettoken(&mroute_token); if (rsvpdebug) kprintf("rsvp_input: check vifs\n"); @@ -2174,7 +2178,7 @@ X_rsvp_input(struct mbuf *m, ...) kprintf("rsvp_input: No socket defined for vif %d\n",vifi); m_freem(m); } - crit_exit(); + lwkt_reltoken(&mroute_token); return; } rsvp_src.sin_addr = ip->ip_src; @@ -2212,7 +2216,7 @@ X_rsvp_input(struct mbuf *m, ...) } #endif /* !ALTQ */ - crit_exit(); + lwkt_reltoken(&mroute_token); } /* @@ -2277,10 +2281,10 @@ add_bw_upcall(struct bw_upcall *req) /* * Find if we have already same bw_meter entry */ - crit_enter(); + lwkt_gettoken(&mroute_token); mfc = mfc_find(req->bu_src.s_addr, req->bu_dst.s_addr); if (mfc == NULL) { - crit_exit(); + lwkt_reltoken(&mroute_token); return EADDRNOTAVAIL; } for (x = mfc->mfc_bw_meter; x != NULL; x = x->bm_mfc_next) { @@ -2289,11 +2293,11 @@ add_bw_upcall(struct bw_upcall *req) (x->bm_threshold.b_packets == req->bu_threshold.b_packets) && (x->bm_threshold.b_bytes == req->bu_threshold.b_bytes) && (x->bm_flags & BW_METER_USER_FLAGS) == flags) { - crit_exit(); + lwkt_reltoken(&mroute_token); return 0; /* XXX Already installed */ } } - crit_exit(); + lwkt_reltoken(&mroute_token); /* Allocate the new bw_meter entry */ x = kmalloc(sizeof(*x), M_BWMETER, M_INTWAIT); @@ -2311,12 +2315,12 @@ add_bw_upcall(struct bw_upcall *req) x->bm_time_hash = BW_METER_BUCKETS; /* Add the new bw_meter entry to the front of entries for this MFC */ - crit_enter(); + lwkt_gettoken(&mroute_token); x->bm_mfc = mfc; x->bm_mfc_next = mfc->mfc_bw_meter; mfc->mfc_bw_meter = x; schedule_bw_meter(x, &now); - crit_exit(); + lwkt_reltoken(&mroute_token); return 0; } @@ -2345,11 +2349,11 @@ del_bw_upcall(struct bw_upcall *req) if (!(mrt_api_config & MRT_MFC_BW_UPCALL)) return EOPNOTSUPP; - crit_enter(); + lwkt_gettoken(&mroute_token); /* Find the corresponding MFC entry */ mfc = mfc_find(req->bu_src.s_addr, req->bu_dst.s_addr); if (mfc == NULL) { - crit_exit(); + lwkt_reltoken(&mroute_token); return EADDRNOTAVAIL; } else if (req->bu_flags & BW_UPCALL_DELETE_ALL) { /* @@ -2359,7 +2363,7 @@ del_bw_upcall(struct bw_upcall *req) list = mfc->mfc_bw_meter; mfc->mfc_bw_meter = NULL; - crit_exit(); + lwkt_reltoken(&mroute_token); free_bw_list(list); return 0; } else { /* Delete a single bw_meter entry */ @@ -2383,14 +2387,13 @@ del_bw_upcall(struct bw_upcall *req) prev->bm_mfc_next = x->bm_mfc_next; /* remove from middle*/ else x->bm_mfc->mfc_bw_meter = x->bm_mfc_next;/* new head of list */ - crit_exit(); - unschedule_bw_meter(x); + lwkt_reltoken(&mroute_token); /* Free the bw_meter entry */ kfree(x, M_BWMETER); return 0; } else { - crit_exit(); + lwkt_reltoken(&mroute_token); return EINVAL; } } @@ -2405,7 +2408,7 @@ bw_meter_receive_packet(struct bw_meter *x, int plen, struct timeval *nowp) { struct timeval delta; - crit_enter(); + lwkt_gettoken(&mroute_token); delta = *nowp; BW_TIMEVALDECR(&delta, &x->bm_start_time); @@ -2486,7 +2489,7 @@ bw_meter_receive_packet(struct bw_meter *x, int plen, struct timeval *nowp) x->bm_flags &= ~BW_METER_UPCALL_DELIVERED; } } - crit_exit(); + lwkt_reltoken(&mroute_token); } /* @@ -2498,7 +2501,7 @@ bw_meter_prepare_upcall(struct bw_meter *x, struct timeval *nowp) struct timeval delta; struct bw_upcall *u; - crit_enter(); + lwkt_gettoken(&mroute_token); /* * Compute the measured time interval @@ -2534,7 +2537,7 @@ bw_meter_prepare_upcall(struct bw_meter *x, struct timeval *nowp) if (x->bm_flags & BW_METER_LEQ) u->bu_flags |= BW_UPCALL_LEQ; - crit_exit(); + lwkt_reltoken(&mroute_token); } /* @@ -2614,12 +2617,11 @@ schedule_bw_meter(struct bw_meter *x, struct timeval *nowp) /* * Reset the bw_meter entry */ - crit_enter(); + lwkt_gettoken(&mroute_token); x->bm_start_time = *nowp; x->bm_measured.b_packets = 0; x->bm_measured.b_bytes = 0; x->bm_flags &= ~BW_METER_UPCALL_DELIVERED; - crit_exit(); /* * Compute the timeout hash value and insert the entry @@ -2628,6 +2630,8 @@ schedule_bw_meter(struct bw_meter *x, struct timeval *nowp) x->bm_time_next = bw_meter_timers[time_hash]; bw_meter_timers[time_hash] = x; x->bm_time_hash = time_hash; + + lwkt_reltoken(&mroute_token); } /* @@ -2690,7 +2694,7 @@ bw_meter_process(void) if (last_tv_sec == now.tv_sec) return; /* nothing to do */ - crit_enter(); + lwkt_gettoken(&mroute_token); loops = now.tv_sec - last_tv_sec; last_tv_sec = now.tv_sec; if (loops > BW_METER_BUCKETS) @@ -2756,10 +2760,9 @@ bw_meter_process(void) schedule_bw_meter(x, &now); } } - crit_exit(); - /* Send all upcalls that are pending delivery */ bw_upcalls_send(); + lwkt_reltoken(&mroute_token); } /* @@ -3255,7 +3258,7 @@ ip_mroute_modevent(module_t mod, int type, void *unused) { switch (type) { case MOD_LOAD: - crit_enter(); + lwkt_gettoken(&mroute_token); /* XXX Protect against multiple loading */ ip_mcast_src = X_ip_mcast_src; ip_mforward = X_ip_mforward; @@ -3268,14 +3271,14 @@ ip_mroute_modevent(module_t mod, int type, void *unused) legal_vif_num = X_legal_vif_num; mrt_ioctl = X_mrt_ioctl; rsvp_input_p = X_rsvp_input; - crit_exit(); + lwkt_reltoken(&mroute_token); break; case MOD_UNLOAD: if (ip_mrouter) return EINVAL; - crit_enter(); + lwkt_gettoken(&mroute_token); ip_mcast_src = NULL; ip_mforward = NULL; ip_mrouter_done = NULL; @@ -3287,7 +3290,7 @@ ip_mroute_modevent(module_t mod, int type, void *unused) legal_vif_num = NULL; mrt_ioctl = NULL; rsvp_input_p = NULL; - crit_exit(); + lwkt_reltoken(&mroute_token); break; } return 0; diff --git a/sys/net/raw_cb.c b/sys/net/raw_cb.c index 0046d2a6e3..b92c1ef5cd 100644 --- a/sys/net/raw_cb.c +++ b/sys/net/raw_cb.c @@ -61,6 +61,9 @@ static u_long raw_recvspace = RAWRCVQ; /* * Allocate a control block and a nominal amount * of buffer space for the socket. + * + * The so->so_pcb has already been assigned by the caller, and the + * caller has also already bumped the socket refs. */ int raw_attach(struct socket *so, int proto, struct rlimit *rl) @@ -95,7 +98,7 @@ raw_detach(struct rawcb *rp) struct socket *so = rp->rcb_socket; so->so_pcb = NULL; - sofree(so); + sofree(so); /* remove pcb ref */ LIST_REMOVE(rp, list); kfree(rp, M_PCB); } diff --git a/sys/net/raw_usrreq.c b/sys/net/raw_usrreq.c index 54be725b42..1e87404230 100644 --- a/sys/net/raw_usrreq.c +++ b/sys/net/raw_usrreq.c @@ -44,6 +44,8 @@ #include #include +#include + #include /* @@ -133,6 +135,10 @@ raw_ctlinput(int cmd, struct sockaddr *arg, void *dummy) /* INCOMPLETE */ } +/* + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. + */ static int raw_uabort(struct socket *so) { @@ -141,7 +147,6 @@ raw_uabort(struct socket *so) if (rp == NULL) return EINVAL; raw_disconnect(rp); - sofree(so); soisdisconnected(so); return 0; } @@ -198,8 +203,11 @@ raw_udisconnect(struct socket *so) if (rp->rcb_faddr == NULL) { return ENOTCONN; } + soreference(so); raw_disconnect(rp); soisdisconnected(so); + sofree(so); + return 0; } diff --git a/sys/net/rtsock.c b/sys/net/rtsock.c index 1bd97f69ba..4dea0fd614 100644 --- a/sys/net/rtsock.c +++ b/sys/net/rtsock.c @@ -81,7 +81,9 @@ #include #include #include + #include +#include #include #include @@ -160,6 +162,7 @@ rts_attach(struct socket *so, int proto, struct pru_attach_info *ai) */ crit_enter(); so->so_pcb = rp; + soreference(so); /* so_pcb assignment */ error = raw_attach(so, proto, ai->sb_rlimit); rp = sotorawcb(so); if (error) { diff --git a/sys/netbt/hci_socket.c b/sys/netbt/hci_socket.c index ae785e7bed..9bc0a1a832 100644 --- a/sys/netbt/hci_socket.c +++ b/sys/netbt/hci_socket.c @@ -52,7 +52,9 @@ #include #include #include + #include +#include #include #include @@ -556,14 +558,20 @@ bad: /* * Implementation of usrreqs. + * + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. */ static int hci_sabort (struct socket *so) { + int error; + /* struct hci_pcb *pcb = (struct hci_pcb *)so->so_pcb; */ soisdisconnected(so); - return hci_sdetach(so); + error = hci_sdetach(so); + return (error); } static int @@ -577,6 +585,8 @@ hci_sdetach(struct socket *so) hci_cmdwait_flush(so); so->so_pcb = NULL; + sofree(so); /* remove pcb ref */ + LIST_REMOVE(pcb, hp_next); kfree(pcb, M_PCB); @@ -599,7 +609,7 @@ hci_sdisconnect (struct socket *so) * this socket (which is permitted) you get a broken pipe when you * try to write any data. */ - so->so_state &= ~SS_ISCONNECTED; + soclrstate(so, SS_ISCONNECTED); return 0; } @@ -628,6 +638,7 @@ hci_sattach (struct socket *so, int proto, struct pru_attach_info *ai) if (pcb == NULL) return ENOMEM; + soreference(so); so->so_pcb = pcb; pcb->hp_socket = so; diff --git a/sys/netbt/l2cap_socket.c b/sys/netbt/l2cap_socket.c index 4eabfdf320..355479f683 100644 --- a/sys/netbt/l2cap_socket.c +++ b/sys/netbt/l2cap_socket.c @@ -241,15 +241,21 @@ l2cap_sdetach(struct socket *so) return l2cap_detach((struct l2cap_channel **)&so->so_pcb); } +/* + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. + */ static int l2cap_sabort (struct socket *so) { struct l2cap_channel *pcb = so->so_pcb; + int error; l2cap_disconnect(pcb, 0); soisdisconnected(so); - return l2cap_sdetach(so); + error = l2cap_sdetach(so); + return error; } static int diff --git a/sys/netbt/rfcomm_socket.c b/sys/netbt/rfcomm_socket.c index 7d5c02b1de..78664fa579 100644 --- a/sys/netbt/rfcomm_socket.c +++ b/sys/netbt/rfcomm_socket.c @@ -251,14 +251,20 @@ rfcomm_sdetach(struct socket *so) return rfcomm_detach((struct rfcomm_dlc **)&so->so_pcb); } +/* + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. + */ static int rfcomm_sabort (struct socket *so) { struct rfcomm_dlc *pcb = (struct rfcomm_dlc *) so->so_pcb; + int error; rfcomm_disconnect(pcb, 0); soisdisconnected(so); - return rfcomm_sdetach(so); + error = rfcomm_sdetach(so); + return error; } static int diff --git a/sys/netbt/sco_socket.c b/sys/netbt/sco_socket.c index 56953923e3..09cfb2f289 100644 --- a/sys/netbt/sco_socket.c +++ b/sys/netbt/sco_socket.c @@ -228,15 +228,21 @@ sco_sdetach(struct socket *so) return sco_detach((struct sco_pcb **)&so->so_pcb); } +/* + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. + */ static int sco_sabort (struct socket *so) { struct sco_pcb *pcb = (struct sco_pcb *)so->so_pcb; + int error; sco_disconnect(pcb, 0); soisdisconnected(so); + error = sco_sdetach(so); - return sco_sdetach(so); + return error; } static int diff --git a/sys/netgraph/ksocket/ng_ksocket.c b/sys/netgraph/ksocket/ng_ksocket.c index 3cc3c737f7..edcddd2bbf 100644 --- a/sys/netgraph/ksocket/ng_ksocket.c +++ b/sys/netgraph/ksocket/ng_ksocket.c @@ -732,7 +732,7 @@ ng_ksocket_rcvmsg(node_p node, struct ng_mesg *msg, if ((so->so_state & SS_ISCONNECTING) != 0) ERROUT(EALREADY); if ((error = soconnect(so, sa, td)) != 0) { - so->so_state &= ~SS_ISCONNECTING; + soclrstate(so, SS_ISCONNECTING); ERROUT(error); } if ((so->so_state & SS_ISCONNECTING) != 0) { @@ -1006,7 +1006,7 @@ ng_ksocket_incoming(struct socket *so, void *arg, int waitflag) if (priv->flags & KSF_CONNECTING) { if ((error = so->so_error) != 0) { so->so_error = 0; - so->so_state &= ~SS_ISCONNECTING; + soclrstate(so, SS_ISCONNECTING); } if (!(so->so_state & SS_ISCONNECTING)) { NG_MKMESSAGE(response, NGM_KSOCKET_COOKIE, @@ -1132,16 +1132,20 @@ ng_ksocket_check_accept(priv_p priv) struct socket *const head = priv->so; int error; + lwkt_gettoken(&head->so_rcv.ssb_token); if ((error = head->so_error) != 0) { head->so_error = 0; + lwkt_reltoken(&head->so_rcv.ssb_token); return error; } if (TAILQ_EMPTY(&head->so_comp)) { if (head->so_state & SS_CANTRCVMORE) - return ECONNABORTED; - return EWOULDBLOCK; + error = ECONNABORTED; + else + error = EWOULDBLOCK; } - return 0; + lwkt_reltoken(&head->so_rcv.ssb_token); + return error; } /* @@ -1160,15 +1164,19 @@ ng_ksocket_finish_accept(priv_p priv, struct ng_mesg **rptr) priv_p priv2; int len; + lwkt_gettoken(&head->so_rcv.ssb_token); so = TAILQ_FIRST(&head->so_comp); - if (so == NULL) /* Should never happen */ + if (so == NULL) { /* Should never happen */ + lwkt_reltoken(&head->so_rcv.ssb_token); return; + } TAILQ_REMOVE(&head->so_comp, so, so_list); head->so_qlen--; + lwkt_reltoken(&head->so_rcv.ssb_token); /* XXX KNOTE(&head->so_rcv.ssb_sel.si_note, 0); */ - so->so_state &= ~SS_COMP; + soclrstate(so, SS_COMP); so->so_head = NULL; soaccept(so, &sa); diff --git a/sys/netgraph/socket/ng_socket.c b/sys/netgraph/socket/ng_socket.c index a11f1bda16..c2d63173eb 100644 --- a/sys/netgraph/socket/ng_socket.c +++ b/sys/netgraph/socket/ng_socket.c @@ -476,6 +476,7 @@ ng_attach_common(struct socket *so, int type) pcbp->type = type; /* Link the pcb and the socket */ + soreference(so); so->so_pcb = (caddr_t) pcbp; pcbp->ng_socket = so; @@ -494,6 +495,7 @@ static void ng_detach_common(struct ngpcb *pcbp, int which) { struct ngsock *sockdata; + struct socket *so; if (pcbp->sockdata) { sockdata = pcbp->sockdata; @@ -511,8 +513,11 @@ ng_detach_common(struct ngpcb *pcbp, int which) if ((--sockdata->refs == 0) && (sockdata->node != NULL)) ng_rmnode(sockdata->node); } - pcbp->ng_socket->so_pcb = NULL; + so = pcbp->ng_socket; + so->so_pcb = NULL; pcbp->ng_socket = NULL; + sofree(so); /* remove pcb ref */ + LIST_REMOVE(pcbp, socks); FREE(pcbp, M_PCB); } diff --git a/sys/netgraph7/bluetooth/socket/ng_btsocket_rfcomm.c b/sys/netgraph7/bluetooth/socket/ng_btsocket_rfcomm.c index 0f43a2d7f1..6d7f34bb90 100644 --- a/sys/netgraph7/bluetooth/socket/ng_btsocket_rfcomm.c +++ b/sys/netgraph7/bluetooth/socket/ng_btsocket_rfcomm.c @@ -706,6 +706,8 @@ ng_btsocket_rfcomm_detach(struct socket *so) mtx_lock(&pcb->pcb_mtx); + so->so_pcb = NULL; + switch (pcb->state) { case NG_BTSOCKET_RFCOMM_DLC_W4_CONNECT: case NG_BTSOCKET_RFCOMM_DLC_CONFIGURING: @@ -748,7 +750,7 @@ ng_btsocket_rfcomm_detach(struct socket *so) FREE(pcb, M_NETGRAPH_BTSOCKET_RFCOMM); soisdisconnected(so); - so->so_pcb = NULL; + sofree(so); /* for so_pcb = NULL */ } /* ng_btsocket_rfcomm_detach */ /* @@ -1296,7 +1298,7 @@ ng_btsocket_rfcomm_session_create(ng_btsocket_rfcomm_session_p *sp, SOCKBUF_LOCK(&l2so->so_snd); l2so->so_snd.sb_flags |= SB_UPCALL; SOCKBUF_UNLOCK(&l2so->so_snd); - l2so->so_state |= SS_NBIO; + sosetstate(l2so, SS_NBIO); s->l2so = l2so; mtx_lock(&s->session_mtx); @@ -1380,7 +1382,7 @@ bad: SOCKBUF_LOCK(&l2so->so_snd); l2so->so_snd.sb_flags &= ~SB_UPCALL; SOCKBUF_UNLOCK(&l2so->so_snd); - l2so->so_state &= ~SS_NBIO; + soclrstate(l2so, SS_NBIO); mtx_destroy(&s->session_mtx); bzero(s, sizeof(*s)); @@ -1434,7 +1436,7 @@ ng_btsocket_rfcomm_session_accept(ng_btsocket_rfcomm_session_p s0) l2so->so_head = NULL; SOCK_LOCK(l2so); soref(l2so); - l2so->so_state |= SS_NBIO; + sosetstate(l2so, SS_NBIO); SOCK_UNLOCK(l2so); ACCEPT_UNLOCK(); diff --git a/sys/netgraph7/ng_ksocket.c b/sys/netgraph7/ng_ksocket.c index 8680bbe1f8..b2b84bbc2d 100644 --- a/sys/netgraph7/ng_ksocket.c +++ b/sys/netgraph7/ng_ksocket.c @@ -626,7 +626,7 @@ ng_ksocket_connect(hook_p hook) priv->so->so_snd.sb_flags |= SB_UPCALL; SOCKBUF_UNLOCK(&priv->so->so_snd); SOCK_LOCK(priv->so); - priv->so->so_state |= SS_NBIO; + sosetstate(priv->so, SS_NBIO); SOCK_UNLOCK(priv->so); /* * --Original comment-- @@ -761,7 +761,7 @@ ng_ksocket_rcvmsg(node_p node, item_p item, hook_p lasthook) if ((so->so_state & SS_ISCONNECTING) != 0) ERROUT(EALREADY); if ((error = soconnect(so, sa, td)) != 0) { - so->so_state &= ~SS_ISCONNECTING; + soclrstate(so, SS_ISCONNECTING); ERROUT(error); } if ((so->so_state & SS_ISCONNECTING) != 0) { @@ -1050,7 +1050,7 @@ ng_ksocket_incoming2(node_p node, hook_p hook, void *arg1, int arg2) if (priv->flags & KSF_CONNECTING) { if ((error = so->so_error) != 0) { so->so_error = 0; - so->so_state &= ~SS_ISCONNECTING; + soclrstate(so, SS_ISCONNECTING); } if (!(so->so_state & SS_ISCONNECTING)) { NG_MKMESSAGE(response, NGM_KSOCKET_COOKIE, @@ -1211,7 +1211,7 @@ ng_ksocket_finish_accept(priv_p priv) so->so_head = NULL; SOCK_LOCK(so); soref(so); - so->so_state |= SS_NBIO; + sosetstate(so, SS_NBIO); SOCK_UNLOCK(so); ACCEPT_UNLOCK(); diff --git a/sys/netgraph7/ng_socket.c b/sys/netgraph7/ng_socket.c index 237f97a50a..ed065c13cf 100644 --- a/sys/netgraph7/ng_socket.c +++ b/sys/netgraph7/ng_socket.c @@ -824,9 +824,9 @@ ngs_connect(hook_p hook) if ((priv->datasock) && (priv->datasock->ng_socket)) { if (NG_NODE_NUMHOOKS(node) == 1) - priv->datasock->ng_socket->so_state |= SS_ISCONNECTED; + sosetstate(priv->datasock->ng_socket, SS_ISCONNECTED); else - priv->datasock->ng_socket->so_state &= ~SS_ISCONNECTED; + soclrstate(priv->datasock->ng_socket, SS_ISCONNECTED); } return (0); } @@ -983,9 +983,9 @@ ngs_disconnect(hook_p hook) if ((priv->datasock) && (priv->datasock->ng_socket)) { if (NG_NODE_NUMHOOKS(node) == 1) - priv->datasock->ng_socket->so_state |= SS_ISCONNECTED; + sosetstate(priv->datasock->ng_socket, SS_ISCONNECTED); else - priv->datasock->ng_socket->so_state &= ~SS_ISCONNECTED; + soclrstate(priv->datasock->ng_socket, SS_ISCONNECTED); } if ((priv->flags & NGS_FLAG_NOLINGER) && diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c index 1ae4c092cd..d9747040c7 100644 --- a/sys/netinet/in_pcb.c +++ b/sys/netinet/in_pcb.c @@ -84,7 +84,9 @@ #include #include #include + #include +#include #include @@ -217,6 +219,7 @@ in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo) if (ip6_auto_flowlabel) inp->inp_flags |= IN6P_AUTOFLOWLABEL; #endif + soreference(so); so->so_pcb = inp; LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list); pcbinfo->ipi_count++; @@ -648,7 +651,7 @@ in_pcbdetach(struct inpcb *inp) inp->inp_gencnt = ++ipi->ipi_gencnt; in_pcbremlists(inp); so->so_pcb = NULL; - sofree(so); + sofree(so); /* remove pcb ref */ if (inp->inp_options) m_free(inp->inp_options); if (inp->inp_route.ro_rt) diff --git a/sys/netinet/ip_divert.c b/sys/netinet/ip_divert.c index bec5dc2f3a..4018d48ccd 100644 --- a/sys/netinet/ip_divert.c +++ b/sys/netinet/ip_divert.c @@ -126,6 +126,8 @@ static u_long div_recvspace = DIVRCVQ; /* XXX sysctl ? */ static struct mbuf *ip_divert(struct mbuf *, int, int); +static struct lwkt_token div_token = LWKT_TOKEN_MP_INITIALIZER(div_token); + /* * Initialize divert connection block queue. */ @@ -307,7 +309,7 @@ div_packet(struct mbuf *m, int incoming, int port) * saving/testing the socket pointer is not MPSAFE. So we still * need to hold BGL here. */ - get_mplock(); + lwkt_gettoken(&div_token); LIST_FOREACH(inp, &divcbinfo.pcblisthead, inp_list) { if (inp->inp_flags & INP_PLACEMARKER) continue; @@ -315,18 +317,18 @@ div_packet(struct mbuf *m, int incoming, int port) sa = inp->inp_socket; } if (sa) { - if (ssb_appendaddr(&sa->so_rcv, (struct sockaddr *)&divsrc, m, - NULL) == 0) + lwkt_gettoken(&sa->so_rcv.ssb_token); + if (ssb_appendaddr(&sa->so_rcv, (struct sockaddr *)&divsrc, m, NULL) == 0) m_freem(m); else sorwakeup(sa); - rel_mplock(); + lwkt_reltoken(&sa->so_rcv.ssb_token); } else { - rel_mplock(); m_freem(m); ipstat.ips_noproto++; ipstat.ips_delivered--; } + lwkt_reltoken(&div_token); } #ifdef SMP @@ -476,9 +478,12 @@ div_attach(struct socket *so, int proto, struct pru_attach_info *ai) error = soreserve(so, div_sendspace, div_recvspace, ai->sb_rlimit); if (error) return error; + lwkt_gettoken(&div_token); error = in_pcballoc(so, &divcbinfo); - if (error) + if (error) { + lwkt_reltoken(&div_token); return error; + } inp = (struct inpcb *)so->so_pcb; inp->inp_ip_p = proto; inp->inp_vflag |= INP_IPV4; @@ -488,7 +493,8 @@ div_attach(struct socket *so, int proto, struct pru_attach_info *ai) * we always know "where" to send the packet. */ so->so_port = cpu0_soport(so, NULL, NULL); - so->so_state |= SS_ISCONNECTED; + sosetstate(so, SS_ISCONNECTED); + lwkt_reltoken(&div_token); return 0; } @@ -504,19 +510,33 @@ div_detach(struct socket *so) return 0; } +/* + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. + */ static int div_abort(struct socket *so) { + int error; + soisdisconnected(so); - return div_detach(so); + error = div_detach(so); + + return error; } static int div_disconnect(struct socket *so) { + int error; + if (!(so->so_state & SS_ISCONNECTED)) return ENOTCONN; - return div_abort(so); + soreference(so); + error = div_abort(so); + sofree(so); + + return error; } static int diff --git a/sys/netinet/raw_ip.c b/sys/netinet/raw_ip.c index 80b1b9bcf7..20be825f0d 100644 --- a/sys/netinet/raw_ip.c +++ b/sys/netinet/raw_ip.c @@ -50,7 +50,9 @@ #include #include #include + #include +#include #include @@ -83,6 +85,9 @@ struct inpcbinfo ripcbinfo; ip_fw_ctl_t *ip_fw_ctl_ptr; ip_dn_ctl_t *ip_dn_ctl_ptr; +static struct lwkt_token raw_token = LWKT_TOKEN_MP_INITIALIZER(raw_token); + + /* * hooks for multicast routing. They all default to NULL, * so leave them not initialized and rely on BSS being set to 0. @@ -156,6 +161,7 @@ rip_input(struct mbuf *m, ...) __va_end(ap); ripsrc.sin_addr = ip->ip_src; + lwkt_gettoken(&raw_token); LIST_FOREACH(inp, &ripcbinfo.pcblisthead, inp_list) { if (inp->inp_flags & INP_PLACEMARKER) continue; @@ -190,18 +196,21 @@ rip_input(struct mbuf *m, ...) } else #endif /*FAST_IPSEC*/ if (n) { + lwkt_gettoken(&last->inp_socket->so_rcv.ssb_token); if (last->inp_flags & INP_CONTROLOPTS || last->inp_socket->so_options & SO_TIMESTAMP) ip_savecontrol(last, &opts, ip, n); if (ssb_appendaddr(&last->inp_socket->so_rcv, - (struct sockaddr *)&ripsrc, n, - opts) == 0) { + (struct sockaddr *)&ripsrc, n, + opts) == 0) { /* should notify about lost packet */ m_freem(n); if (opts) m_freem(opts); - } else + } else { sorwakeup(last->inp_socket); + } + lwkt_reltoken(&last->inp_socket->so_rcv.ssb_token); opts = 0; } } @@ -232,18 +241,22 @@ rip_input(struct mbuf *m, ...) if (last->inp_flags & INP_CONTROLOPTS || last->inp_socket->so_options & SO_TIMESTAMP) ip_savecontrol(last, &opts, ip, m); + lwkt_gettoken(&last->inp_socket->so_rcv.ssb_token); if (ssb_appendaddr(&last->inp_socket->so_rcv, (struct sockaddr *)&ripsrc, m, opts) == 0) { m_freem(m); if (opts) m_freem(opts); - } else + } else { sorwakeup(last->inp_socket); + } + lwkt_reltoken(&last->inp_socket->so_rcv.ssb_token); } else { m_freem(m); ipstat.ips_noproto++; ipstat.ips_delivered--; } + lwkt_reltoken(&raw_token); } /* @@ -538,15 +551,15 @@ rip_attach(struct socket *so, int proto, struct pru_attach_info *ai) error = soreserve(so, rip_sendspace, rip_recvspace, ai->sb_rlimit); if (error) return error; - crit_enter(); + lwkt_gettoken(&raw_token); error = in_pcballoc(so, &ripcbinfo); - crit_exit(); - if (error) - return error; - inp = (struct inpcb *)so->so_pcb; - inp->inp_vflag |= INP_IPV4; - inp->inp_ip_p = proto; - inp->inp_ip_ttl = ip_defttl; + if (error == 0) { + inp = (struct inpcb *)so->so_pcb; + inp->inp_vflag |= INP_IPV4; + inp->inp_ip_p = proto; + inp->inp_ip_ttl = ip_defttl; + } + lwkt_reltoken(&raw_token); return 0; } @@ -568,21 +581,36 @@ rip_detach(struct socket *so) return 0; } +/* + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. + */ static int rip_abort(struct socket *so) { + int error; + soisdisconnected(so); - if (so->so_state & SS_NOFDREF) - return rip_detach(so); - return 0; + if (so->so_state & SS_NOFDREF) /* XXX not sure why this test */ + error = rip_detach(so); + else + error = 0; + + return error; } static int rip_disconnect(struct socket *so) { + int error; + if ((so->so_state & SS_ISCONNECTED) == 0) return ENOTCONN; - return rip_abort(so); + soreference(so); + error = rip_abort(so); + sofree(so); + + return error; } static int diff --git a/sys/netinet/sctp_indata.c b/sys/netinet/sctp_indata.c index be68768444..e1d5c13bfe 100644 --- a/sys/netinet/sctp_indata.c +++ b/sys/netinet/sctp_indata.c @@ -459,6 +459,7 @@ sctp_deliver_data(struct sctp_tcb *stcb, struct sctp_association *asoc, } goto skip; } + lwkt_gettoken(&stcb->sctp_socket->so_rcv.ssb_token); if (!sctp_sbappendaddr_nocheck(&stcb->sctp_socket->so_rcv, to, chk->data, control, stcb->asoc.my_vtag, stcb->sctp_ep)) { @@ -479,13 +480,16 @@ sctp_deliver_data(struct sctp_tcb *stcb, struct sctp_association *asoc, } free_it = 1; } + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); } else { /* append to a already started message. */ + lwkt_gettoken(&stcb->sctp_socket->so_rcv.ssb_token); if (sctp_sbspace(&stcb->sctp_socket->so_rcv) >= (long)chk->send_size) { ssb_append(&stcb->sctp_socket->so_rcv, chk->data); free_it = 1; } + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); } skip: if (hold_locks == 0) @@ -684,6 +688,7 @@ sctp_service_reassembly(struct sctp_tcb *stcb, struct sctp_association *asoc, in SCTP_INP_WUNLOCK(stcb->sctp_ep); return; } + lwkt_gettoken(&stcb->sctp_socket->so_rcv.ssb_token); if (!sctp_sbappendaddr_nocheck(&stcb->sctp_socket->so_rcv, to, chk->data, control, stcb->asoc.my_vtag, stcb->sctp_ep)) { @@ -697,8 +702,10 @@ sctp_service_reassembly(struct sctp_tcb *stcb, struct sctp_association *asoc, in stcb->sctp_socket); if (hold_locks == 0) SCTP_INP_WUNLOCK(stcb->sctp_ep); + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); return; } + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); if ((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL) == 0) { if (sctp_add_to_socket_q(stcb->sctp_ep, stcb)) { stcb->asoc.my_rwnd_control_len += @@ -711,7 +718,9 @@ sctp_service_reassembly(struct sctp_tcb *stcb, struct sctp_association *asoc, in } else { if (sctp_sbspace(&stcb->sctp_socket->so_rcv) >= (long)chk->send_size) { + lwkt_gettoken(&stcb->sctp_socket->so_rcv.ssb_token); ssb_append(&stcb->sctp_socket->so_rcv, chk->data); + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); cntDel++; } else { /* out of space in the sb */ @@ -1996,6 +2005,7 @@ sctp_process_a_data_chunk(struct sctp_tcb *stcb, struct sctp_association *asoc, SCTP_TCB_UNLOCK(stcb); SCTP_INP_WLOCK(stcb->sctp_ep); SCTP_TCB_LOCK(stcb); + lwkt_gettoken(&stcb->sctp_socket->so_rcv.ssb_token); if (!sctp_sbappendaddr_nocheck(&stcb->sctp_socket->so_rcv, to, dmbuf, control, stcb->asoc.my_vtag, stcb->sctp_ep)) { if (control) { @@ -2004,8 +2014,10 @@ sctp_process_a_data_chunk(struct sctp_tcb *stcb, struct sctp_association *asoc, CMSG_LEN(sizeof(struct sctp_sndrcvinfo)); } sctp_m_freem(dmbuf); + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); goto failed_express_del; } + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); if ((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL) == 0) { if (sctp_add_to_socket_q(stcb->sctp_ep, stcb)) { stcb->asoc.my_rwnd_control_len += sizeof(struct mbuf); diff --git a/sys/netinet/sctp_input.c b/sys/netinet/sctp_input.c index fbe20f8860..750d1128f7 100644 --- a/sys/netinet/sctp_input.c +++ b/sys/netinet/sctp_input.c @@ -591,7 +591,7 @@ sctp_handle_shutdown(struct sctp_shutdown_chunk *cp, #if defined(__FreeBSD__) && __FreeBSD_version >= 502115 stcb->sctp_ep->sctp_socket->so_rcv.sb_state |= SBS_CANTSENDMORE; #else - stcb->sctp_ep->sctp_socket->so_state |= SS_CANTSENDMORE; + sosetstate(stcb->sctp_ep->sctp_socket, SS_CANTSENDMORE); #endif } /* reset time */ diff --git a/sys/netinet/sctp_pcb.c b/sys/netinet/sctp_pcb.c index bca9e666a8..96b30320ce 100644 --- a/sys/netinet/sctp_pcb.c +++ b/sys/netinet/sctp_pcb.c @@ -2237,13 +2237,11 @@ sctp_inpcb_free(struct sctp_inpcb *inp, int immediate) #endif int cnt; - crit_enter(); SCTP_ASOC_CREATE_LOCK(inp); SCTP_INP_WLOCK(inp); if (inp->sctp_flags & SCTP_PCB_FLAGS_SOCKET_ALLGONE) { /* been here before */ - crit_exit(); kprintf("Endpoint was all gone (dup free)?\n"); SCTP_INP_WUNLOCK(inp); SCTP_ASOC_CREATE_UNLOCK(inp); @@ -2333,7 +2331,6 @@ sctp_inpcb_free(struct sctp_inpcb *inp, int immediate) /* now is there some left in our SHUTDOWN state? */ if (cnt_in_sd) { inp->sctp_flags |= SCTP_PCB_FLAGS_SOCKET_GONE; - crit_exit(); SCTP_INP_WUNLOCK(inp); SCTP_ASOC_CREATE_UNLOCK(inp); return; @@ -2359,7 +2356,6 @@ sctp_inpcb_free(struct sctp_inpcb *inp, int immediate) #ifdef IPSEC #ifdef __OpenBSD__ /* XXX IPsec cleanup here */ - crit_enter(); if (ip_pcb->inp_tdb_in) TAILQ_REMOVE(&ip_pcb->inp_tdb_in->tdb_inp_in, ip_pcb, inp_tdb_in_next); @@ -2378,7 +2374,6 @@ sctp_inpcb_free(struct sctp_inpcb *inp, int immediate) ipsp_reffree(ip_pcb->inp_ipsec_localauth); if (ip_pcb->inp_ipsec_remoteauth) ipsp_reffree(ip_pcb->inp_ipsec_remoteauth); - crit_exit(); #else ipsec4_delete_pcbpolicy(ip_pcb); #endif @@ -2387,12 +2382,8 @@ sctp_inpcb_free(struct sctp_inpcb *inp, int immediate) ACCEPT_LOCK(); SOCK_LOCK(so); #endif - so->so_pcb = 0; -#if defined(__FreeBSD__) && __FreeBSD_version > 500000 - sotryfree(so); -#else + so->so_pcb = NULL; sofree(so); -#endif } if (ip_pcb->inp_options) { @@ -2516,7 +2507,6 @@ sctp_inpcb_free(struct sctp_inpcb *inp, int immediate) sctppcbinfo.ipi_count_ep--; SCTP_INP_INFO_WUNLOCK(); - crit_exit(); } @@ -3319,11 +3309,9 @@ sctp_free_assoc(struct sctp_inpcb *inp, struct sctp_tcb *stcb) struct sctp_socket_q_list *sq; /* first, lets purge the entry from the hash table. */ - crit_enter(); if (stcb->asoc.state == 0) { kprintf("Freeing already free association:%p - huh??\n", stcb); - crit_exit(); return; } asoc = &stcb->asoc; @@ -3627,7 +3615,6 @@ sctp_free_assoc(struct sctp_inpcb *inp, struct sctp_tcb *stcb) if (inp->sctp_flags & SCTP_PCB_FLAGS_SOCKET_GONE) { sctp_inpcb_free(inp, 0); } - crit_exit(); } @@ -3923,11 +3910,9 @@ int sctp_insert_laddr(struct sctpladdr *list, struct ifaddr *ifa) { struct sctp_laddr *laddr; - crit_enter(); laddr = (struct sctp_laddr *)SCTP_ZONE_GET(sctppcbinfo.ipi_zone_laddr); if (laddr == NULL) { /* out of memory? */ - crit_exit(); return (EINVAL); } sctppcbinfo.ipi_count_laddr++; @@ -3937,7 +3922,6 @@ sctp_insert_laddr(struct sctpladdr *list, struct ifaddr *ifa) { /* insert it */ LIST_INSERT_HEAD(list, laddr, sctp_nxt_addr); - crit_exit(); return (0); } @@ -3947,13 +3931,11 @@ sctp_insert_laddr(struct sctpladdr *list, struct ifaddr *ifa) { void sctp_remove_laddr(struct sctp_laddr *laddr) { - crit_enter(); /* remove from the list */ LIST_REMOVE(laddr, sctp_nxt_addr); SCTP_ZONE_FREE(sctppcbinfo.ipi_zone_laddr, laddr); sctppcbinfo.ipi_count_laddr--; sctppcbinfo.ipi_gencnt_laddr++; - crit_exit(); } /* @@ -5065,9 +5047,7 @@ sctp_initiate_iterator(asoc_func af, uint32_t pcb_state, uint32_t asoc_state, SCTP_INP_INFO_WLOCK(); LIST_INSERT_HEAD(&sctppcbinfo.iteratorhead, it, sctp_nxt_itr); SCTP_INP_INFO_WUNLOCK(); - crit_enter(); sctp_iterator_timer(it); - crit_exit(); return (0); } @@ -5089,7 +5069,6 @@ callout_init(struct callout *c) void callout_reset(struct callout *c, int to_ticks, void (*ftn)(void *), void *arg) { - crit_enter(); if (c->c_flags & CALLOUT_PENDING) callout_stop(c); @@ -5111,19 +5090,16 @@ callout_reset(struct callout *c, int to_ticks, void (*ftn)(void *), void *arg) c->c_time = ticks + to_ticks; TAILQ_INSERT_TAIL(&sctppcbinfo.callqueue, c, tqe); #endif - crit_exit(); } int callout_stop(struct callout *c) { - crit_enter(); /* * Don't attempt to delete a callout that's not on the queue. */ if (!(c->c_flags & CALLOUT_PENDING)) { c->c_flags &= ~CALLOUT_ACTIVE; - crit_exit(); return (0); } c->c_flags &= ~(CALLOUT_ACTIVE | CALLOUT_PENDING| CALLOUT_FIRED); @@ -5134,7 +5110,6 @@ callout_stop(struct callout *c) TAILQ_REMOVE(&sctppcbinfo.callqueue, c, tqe); c->c_func = NULL; #endif - crit_exit(); return (1); } @@ -5146,7 +5121,6 @@ sctp_fasttim(void) struct calloutlist locallist; int inited = 0; - crit_enter(); /* run through and subtract and mark all callouts */ c = TAILQ_FIRST(&sctppcbinfo.callqueue); while (c) { @@ -5173,14 +5147,11 @@ sctp_fasttim(void) /* now validate that it did not get canceled */ if (c->c_flags & CALLOUT_FIRED) { c->c_flags &= ~CALLOUT_PENDING; - crit_exit(); (*c->c_func)(c->c_arg); - crit_enter(); } c = TAILQ_FIRST(&locallist); } } - crit_exit(); } #endif #endif /* _SCTP_NEEDS_CALLOUT_ */ diff --git a/sys/netinet/sctp_peeloff.c b/sys/netinet/sctp_peeloff.c index 77395c9f3b..63f053f076 100644 --- a/sys/netinet/sctp_peeloff.c +++ b/sys/netinet/sctp_peeloff.c @@ -187,12 +187,14 @@ sctp_get_peeloff(struct socket *head, caddr_t assoc_id, int *error) SCTP_PCB_FLAGS_IN_TCPPOOL | /* Turn on Blocking IO */ (SCTP_PCB_COPY_FLAGS & inp->sctp_flags)); n_inp->sctp_socket = newso; - newso->so_state |= SS_ISCONNECTED; + sosetstate(newso, SS_ISCONNECTED); /* We remove it right away */ #if defined(__FreeBSD__) || defined(__APPLE__) || defined(__DragonFly__) SOCK_LOCK(head); + lwkt_gettoken(&head->so_rcv.ssb_token); TAILQ_REMOVE(&head->so_comp, newso, so_list); head->so_qlen--; + lwkt_reltoken(&head->so_rcv.ssb_token); SOCK_UNLOCK(head); #else diff --git a/sys/netinet/sctp_usrreq.c b/sys/netinet/sctp_usrreq.c index cf09ad2094..20fce83800 100644 --- a/sys/netinet/sctp_usrreq.c +++ b/sys/netinet/sctp_usrreq.c @@ -697,19 +697,24 @@ SYSCTL_INT(_net_inet_sctp, OID_AUTO, debug, CTLFLAG_RW, #endif /* SCTP_DEBUG */ #endif +/* + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. + */ static int sctp_abort(struct socket *so) { struct sctp_inpcb *inp; + int error; inp = (struct sctp_inpcb *)so->so_pcb; - if (inp == 0) - return EINVAL; /* ??? possible? panic instead? */ - - crit_enter(); - sctp_inpcb_free(inp, 1); - crit_exit(); - return 0; + if (inp) { + sctp_inpcb_free(inp, 1); + error = 0; + } else { + error = EINVAL; + } + return error; } static int @@ -1077,7 +1082,7 @@ sctp_shutdown(struct socket *so) #if defined(__FreeBSD__) && __FreeBSD_version >= 502115 so->so_rcv.sb_state &= ~SBS_CANTRCVMORE; #else - so->so_state &= ~SS_CANTRCVMORE; + soclrstate(so, SS_CANTRCVMORE); #endif /* This proc will wakeup for read and do nothing (I hope) */ crit_exit(); diff --git a/sys/netinet/sctputil.c b/sys/netinet/sctputil.c index 493a7a2317..3167f03a45 100644 --- a/sys/netinet/sctputil.c +++ b/sys/netinet/sctputil.c @@ -2189,13 +2189,16 @@ sctp_notify_assoc_change(u_int32_t event, struct sctp_tcb *stcb, SCTP_TCB_UNLOCK(stcb); SCTP_INP_WLOCK(stcb->sctp_ep); SCTP_TCB_LOCK(stcb); + lwkt_gettoken(&stcb->sctp_socket->so_rcv.ssb_token); if (!sctp_sbappendaddr_nocheck(&stcb->sctp_socket->so_rcv, to, m_notify, NULL, stcb->asoc.my_vtag, stcb->sctp_ep)) { /* not enough room */ sctp_m_freem(m_notify); SCTP_INP_WUNLOCK(stcb->sctp_ep); + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); return; } + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); if (((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL) == 0) && ((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_CONNECTED) == 0)){ if (sctp_add_to_socket_q(stcb->sctp_ep, stcb)) { @@ -2281,13 +2284,16 @@ sctp_notify_peer_addr_change(struct sctp_tcb *stcb, uint32_t state, SCTP_TCB_UNLOCK(stcb); SCTP_INP_WLOCK(stcb->sctp_ep); SCTP_TCB_LOCK(stcb); + lwkt_gettoken(&stcb->sctp_socket->so_rcv.ssb_token); if (!sctp_sbappendaddr_nocheck(&stcb->sctp_socket->so_rcv, to, m_notify, NULL, stcb->asoc.my_vtag, stcb->sctp_ep)) { /* not enough room */ sctp_m_freem(m_notify); SCTP_INP_WUNLOCK(stcb->sctp_ep); + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); return; } + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); if (((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL) == 0) && ((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_CONNECTED) == 0)){ if (sctp_add_to_socket_q(stcb->sctp_ep, stcb)) { @@ -2382,13 +2388,16 @@ sctp_notify_send_failed(struct sctp_tcb *stcb, u_int32_t error, SCTP_TCB_UNLOCK(stcb); SCTP_INP_WLOCK(stcb->sctp_ep); SCTP_TCB_LOCK(stcb); + lwkt_gettoken(&stcb->sctp_socket->so_rcv.ssb_token); if (!sctp_sbappendaddr_nocheck(&stcb->sctp_socket->so_rcv, to, m_notify, NULL, stcb->asoc.my_vtag, stcb->sctp_ep)) { /* not enough room */ sctp_m_freem(m_notify); + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); SCTP_INP_WUNLOCK(stcb->sctp_ep); return; } + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); if (((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL) == 0) && ((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_CONNECTED) == 0)){ if (sctp_add_to_socket_q(stcb->sctp_ep, stcb)) { @@ -2458,13 +2467,16 @@ sctp_notify_adaption_layer(struct sctp_tcb *stcb, SCTP_TCB_UNLOCK(stcb); SCTP_INP_WLOCK(stcb->sctp_ep); SCTP_TCB_LOCK(stcb); + lwkt_gettoken(&stcb->sctp_socket->so_rcv.ssb_token); if (!sctp_sbappendaddr_nocheck(&stcb->sctp_socket->so_rcv, to, m_notify, NULL, stcb->asoc.my_vtag, stcb->sctp_ep)) { /* not enough room */ sctp_m_freem(m_notify); + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); SCTP_INP_WUNLOCK(stcb->sctp_ep); return; } + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); if (((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL) == 0) && ((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_CONNECTED) == 0)){ if (sctp_add_to_socket_q(stcb->sctp_ep, stcb)) { @@ -2534,13 +2546,16 @@ sctp_notify_partial_delivery_indication(struct sctp_tcb *stcb, SCTP_TCB_UNLOCK(stcb); SCTP_INP_WLOCK(stcb->sctp_ep); SCTP_TCB_LOCK(stcb); + lwkt_gettoken(&stcb->sctp_socket->so_rcv.ssb_token); if (!sctp_sbappendaddr_nocheck(&stcb->sctp_socket->so_rcv, to, m_notify, NULL, stcb->asoc.my_vtag, stcb->sctp_ep)) { /* not enough room */ sctp_m_freem(m_notify); + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); SCTP_INP_WUNLOCK(stcb->sctp_ep); return; } + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); if (((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL) == 0) && ((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_CONNECTED) == 0)){ if (sctp_add_to_socket_q(stcb->sctp_ep, stcb)) { @@ -2619,13 +2634,16 @@ sctp_notify_shutdown_event(struct sctp_tcb *stcb) SCTP_TCB_UNLOCK(stcb); SCTP_INP_WLOCK(stcb->sctp_ep); SCTP_TCB_LOCK(stcb); + lwkt_gettoken(&stcb->sctp_socket->so_rcv.ssb_token); if (!sctp_sbappendaddr_nocheck(&stcb->sctp_socket->so_rcv, to, m_notify, NULL, stcb->asoc.my_vtag, stcb->sctp_ep)) { /* not enough room */ sctp_m_freem(m_notify); + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); SCTP_INP_WUNLOCK(stcb->sctp_ep); return; } + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); if (((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL) == 0) && ((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_CONNECTED) == 0)){ if (sctp_add_to_socket_q(stcb->sctp_ep, stcb)) { @@ -2717,13 +2735,16 @@ sctp_notify_stream_reset(struct sctp_tcb *stcb, SCTP_TCB_UNLOCK(stcb); SCTP_INP_WLOCK(stcb->sctp_ep); SCTP_TCB_LOCK(stcb); + lwkt_gettoken(&stcb->sctp_socket->so_rcv.ssb_token); if (!sctp_sbappendaddr_nocheck(&stcb->sctp_socket->so_rcv, to, m_notify, NULL, stcb->asoc.my_vtag, stcb->sctp_ep)) { /* not enough room */ sctp_m_freem(m_notify); + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); SCTP_INP_WUNLOCK(stcb->sctp_ep); return; } + lwkt_reltoken(&stcb->sctp_socket->so_rcv.ssb_token); if (((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL) == 0) && ((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_CONNECTED) == 0)){ if (sctp_add_to_socket_q(stcb->sctp_ep, stcb)) { @@ -3385,6 +3406,7 @@ sctp_get_first_vtag_from_sb(struct socket *so) u_int32_t retval; retval = 0; + lwkt_gettoken(&so->so_rcv.ssb_token); if (so->so_rcv.ssb_mb) { /* grubbing time */ this = so->so_rcv.ssb_mb; @@ -3421,6 +3443,7 @@ sctp_get_first_vtag_from_sb(struct socket *so) } } + lwkt_reltoken(&so->so_rcv.ssb_token); return (retval); } @@ -3442,6 +3465,8 @@ sctp_grub_through_socket_buffer(struct sctp_inpcb *inp, struct socket *old, } SOCKBUF_LOCK(old_sb); SOCKBUF_LOCK(new_sb); + lwkt_gettoken(&old_sb->ssb_token); + lwkt_gettoken(&new_sb->ssb_token); if (inp->sctp_vtag_first == asoc->my_vtag) { /* First one must be moved */ @@ -3501,6 +3526,8 @@ sctp_grub_through_socket_buffer(struct sctp_inpcb *inp, struct socket *old, */ inp->sctp_vtag_first = sctp_get_first_vtag_from_sb(old); } + lwkt_reltoken(&new_sb->ssb_token); + lwkt_reltoken(&old_sb->ssb_token); SOCKBUF_UNLOCK(old_sb); SOCKBUF_UNLOCK(new_sb); } diff --git a/sys/netinet/tcp_input.c b/sys/netinet/tcp_input.c index 30695214d8..f97379a3c3 100644 --- a/sys/netinet/tcp_input.c +++ b/sys/netinet/tcp_input.c @@ -88,6 +88,8 @@ #include #include +#include + #include /* before tcp_seq.h, for tcp_random18() */ #include @@ -468,10 +470,13 @@ present: KASSERT(LIST_EMPTY(&tp->t_segq) || LIST_FIRST(&tp->t_segq)->tqe_th->th_seq != tp->rcv_nxt, ("segment not coalesced")); - if (so->so_state & SS_CANTRCVMORE) + if (so->so_state & SS_CANTRCVMORE) { m_freem(q->tqe_m); - else + } else { + lwkt_gettoken(&so->so_rcv.ssb_token); ssb_appendstream(&so->so_rcv, q->tqe_m); + lwkt_reltoken(&so->so_rcv.ssb_token); + } kfree(q, M_TSEGQ); atomic_add_int(&tcp_reass_qsize, -1); ND6_HINT(tp); @@ -1364,6 +1369,7 @@ after_listen: * being avoided (which is the default), * so force an ack. */ + lwkt_gettoken(&so->so_rcv.ssb_token); if (newsize) { tp->t_flags |= TF_RXRESIZED; if (!ssb_reserve(&so->so_rcv, newsize, @@ -1377,6 +1383,7 @@ after_listen: } m_adj(m, drop_hdrlen); /* delayed header drop */ ssb_appendstream(&so->so_rcv, m); + lwkt_reltoken(&so->so_rcv.ssb_token); } sorwakeup(so); /* @@ -2395,7 +2402,7 @@ step6: so->so_oobmark = so->so_rcv.ssb_cc + (tp->rcv_up - tp->rcv_nxt) - 1; if (so->so_oobmark == 0) - so->so_state |= SS_RCVATMARK; + sosetstate(so, SS_RCVATMARK); sohasoutofband(so); tp->t_oobflags &= ~(TCPOOB_HAVEDATA | TCPOOB_HADDATA); } @@ -2457,10 +2464,13 @@ dodata: /* XXX */ tcpstat.tcps_rcvpack++; tcpstat.tcps_rcvbyte += tlen; ND6_HINT(tp); - if (so->so_state & SS_CANTRCVMORE) + if (so->so_state & SS_CANTRCVMORE) { m_freem(m); - else + } else { + lwkt_gettoken(&so->so_rcv.ssb_token); ssb_appendstream(&so->so_rcv, m); + lwkt_reltoken(&so->so_rcv.ssb_token); + } sorwakeup(so); } else { if (!(tp->t_flags & TF_DUPSEG)) { @@ -3041,8 +3051,11 @@ tcp_mss(struct tcpcb *tp, int offer) bufsize = roundup(bufsize, mss); if (bufsize > sb_max) bufsize = sb_max; - if (bufsize > so->so_rcv.ssb_hiwat) + if (bufsize > so->so_rcv.ssb_hiwat) { + lwkt_gettoken(&so->so_rcv.ssb_token); ssb_reserve(&so->so_rcv, bufsize, so, NULL); + lwkt_reltoken(&so->so_rcv.ssb_token); + } } /* diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c index 15cf85aa03..ed7a9e84bb 100644 --- a/sys/netinet/tcp_subr.c +++ b/sys/netinet/tcp_subr.c @@ -947,6 +947,7 @@ no_valid_rt: inp->inp_ppcb = NULL; soisdisconnected(so); + /* note: pcb detached later on */ tcp_destroy_timermsg(tp); if (tp->t_flags & TF_SYNCACHE) diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c index 7f46ae01de..366d270d77 100644 --- a/sys/netinet/tcp_usrreq.c +++ b/sys/netinet/tcp_usrreq.c @@ -91,6 +91,7 @@ #include #include +#include #include #include @@ -171,7 +172,7 @@ tcp_usr_attach(struct socket *so, int proto, struct pru_attach_info *ai) struct tcpcb *tp = 0; TCPDEBUG0; - crit_enter(); + soreference(so); inp = so->so_pcb; TCPDEBUG1(); if (inp) { @@ -187,8 +188,8 @@ tcp_usr_attach(struct socket *so, int proto, struct pru_attach_info *ai) so->so_linger = TCP_LINGERTIME; tp = sototcpcb(so); out: + sofree(so); /* from ref above */ TCPDEBUG2(PRU_ATTACH); - crit_exit(); return error; } @@ -207,17 +208,14 @@ tcp_usr_detach(struct socket *so) struct tcpcb *tp; TCPDEBUG0; - crit_enter(); inp = so->so_pcb; /* * If the inp is already detached it may have been due to an async * close. Just return as if no error occured. */ - if (inp == NULL) { - crit_exit(); + if (inp == NULL) return 0; - } /* * It's possible for the tcpcb (tp) to disconnect from the inp due @@ -229,7 +227,6 @@ tcp_usr_detach(struct socket *so) tp = tcp_disconnect(tp); TCPDEBUG2(PRU_DETACH); } - crit_exit(); return error; } @@ -241,19 +238,16 @@ tcp_usr_detach(struct socket *so) #define COMMON_START(so, inp, ignore_error) \ TCPDEBUG0; \ \ - crit_enter(); \ inp = so->so_pcb; \ do { \ if (inp == NULL) { \ - crit_exit(); \ return (ignore_error ? 0 : EINVAL); \ } \ tp = intotcpcb(inp); \ TCPDEBUG1(); \ } while(0) -#define COMMON_END(req) out: TCPDEBUG2(req); crit_exit(); return error; goto out - +#define COMMON_END(req) out: TCPDEBUG2(req); return error; goto out /* * Give the socket an address. @@ -570,16 +564,14 @@ tcp_usr_accept(struct socket *so, struct sockaddr **nam) struct tcpcb *tp = NULL; TCPDEBUG0; - crit_enter(); inp = so->so_pcb; if (so->so_state & SS_ISDISCONNECTED) { error = ECONNABORTED; goto out; } - if (inp == 0) { - crit_exit(); + if (inp == 0) return (EINVAL); - } + tp = intotcpcb(inp); TCPDEBUG1(); in_setpeeraddr(so, nam); @@ -595,17 +587,14 @@ tcp6_usr_accept(struct socket *so, struct sockaddr **nam) struct tcpcb *tp = NULL; TCPDEBUG0; - crit_enter(); inp = so->so_pcb; if (so->so_state & SS_ISDISCONNECTED) { error = ECONNABORTED; goto out; } - if (inp == 0) { - crit_exit(); + if (inp == 0) return (EINVAL); - } tp = intotcpcb(inp); TCPDEBUG1(); in6_mapped_peeraddr(so, nam); @@ -664,7 +653,6 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, #endif TCPDEBUG0; - crit_enter(); inp = so->so_pcb; if (inp == NULL) { @@ -769,7 +757,8 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, } /* - * Abort the TCP. + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. */ static int tcp_usr_abort(struct socket *so) @@ -1228,12 +1217,10 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) struct tcpcb *tp; error = 0; - crit_enter(); /* XXX */ inp = so->so_pcb; - if (inp == NULL) { - crit_exit(); + if (inp == NULL) return (ECONNRESET); - } + if (sopt->sopt_level != IPPROTO_TCP) { #ifdef INET6 if (INP_CHECK_SOCKAF(so, AF_INET6)) @@ -1241,7 +1228,6 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) else #endif /* INET6 */ error = ip_ctloutput(so, sopt); - crit_exit(); return (error); } tp = intotcpcb(inp); @@ -1343,7 +1329,6 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) soopt_from_kbuf(sopt, &optval, sizeof optval); break; } - crit_exit(); return (error); } @@ -1379,8 +1364,10 @@ tcp_attach(struct socket *so, struct pru_attach_info *ai) #endif if (so->so_snd.ssb_hiwat == 0 || so->so_rcv.ssb_hiwat == 0) { + lwkt_gettoken(&so->so_rcv.ssb_token); error = soreserve(so, tcp_sendspace, tcp_recvspace, ai->sb_rlimit); + lwkt_reltoken(&so->so_rcv.ssb_token); if (error) return (error); } @@ -1400,17 +1387,18 @@ tcp_attach(struct socket *so, struct pru_attach_info *ai) #endif inp->inp_vflag |= INP_IPV4; tp = tcp_newtcpcb(inp); - if (tp == 0) { - int nofd = so->so_state & SS_NOFDREF; /* XXX */ - - so->so_state &= ~SS_NOFDREF; /* don't free the socket yet */ + if (tp == NULL) { + /* + * Make sure the socket is destroyed by the pcbdetach. + */ + soreference(so); #ifdef INET6 if (isipv6) in6_pcbdetach(inp); else #endif in_pcbdetach(inp); - so->so_state |= nofd; + sofree(so); /* from ref above */ return (ENOBUFS); } tp->t_state = TCPS_CLOSED; @@ -1431,16 +1419,18 @@ tcp_disconnect(struct tcpcb *tp) { struct socket *so = tp->t_inpcb->inp_socket; - if (tp->t_state < TCPS_ESTABLISHED) + if (tp->t_state < TCPS_ESTABLISHED) { tp = tcp_close(tp); - else if ((so->so_options & SO_LINGER) && so->so_linger == 0) + } else if ((so->so_options & SO_LINGER) && so->so_linger == 0) { tp = tcp_drop(tp, 0); - else { + } else { + lwkt_gettoken(&so->so_rcv.ssb_token); soisdisconnecting(so); sbflush(&so->so_rcv.sb); tp = tcp_usrclosed(tp); if (tp) tcp_output(tp); + lwkt_reltoken(&so->so_rcv.ssb_token); } return (tp); } diff --git a/sys/netinet/udp_usrreq.c b/sys/netinet/udp_usrreq.c index bce1ecfe46..e1a5781535 100644 --- a/sys/netinet/udp_usrreq.c +++ b/sys/netinet/udp_usrreq.c @@ -84,9 +84,11 @@ #include #include #include -#include #include +#include +#include + #include #include @@ -661,13 +663,11 @@ udp_ctlinput(int cmd, struct sockaddr *sa, void *vip) else if ((unsigned)cmd >= PRC_NCMDS || inetctlerrmap[cmd] == 0) return; if (ip) { - crit_enter(); uh = (struct udphdr *)((caddr_t)ip + (ip->ip_hl << 2)); inp = in_pcblookup_hash(&udbinfo, faddr, uh->uh_dport, ip->ip_src, uh->uh_sport, 0, NULL); if (inp != NULL && inp->inp_socket != NULL) (*notify)(inp, inetctlerrmap[cmd]); - crit_exit(); } else if (PRC_IS_REDIRECT(cmd)) { struct netmsg_udp_notify nmsg; @@ -706,7 +706,6 @@ udp_getcred(SYSCTL_HANDLER_ARGS) error = SYSCTL_IN(req, addrs, sizeof addrs); if (error) return (error); - crit_enter(); inp = in_pcblookup_hash(&udbinfo, addrs[1].sin_addr, addrs[1].sin_port, addrs[0].sin_addr, addrs[0].sin_port, 1, NULL); if (inp == NULL || inp->inp_socket == NULL) { @@ -715,7 +714,6 @@ udp_getcred(SYSCTL_HANDLER_ARGS) } error = SYSCTL_OUT(req, inp->inp_socket->so_cred, sizeof(struct ucred)); out: - crit_exit(); return (error); } @@ -880,19 +878,25 @@ u_long udp_recvspace = 40 * (1024 + SYSCTL_INT(_net_inet_udp, UDPCTL_RECVSPACE, recvspace, CTLFLAG_RW, &udp_recvspace, 0, "Maximum incoming UDP datagram size"); +/* + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. + */ static int udp_abort(struct socket *so) { struct inpcb *inp; + int error; inp = so->so_pcb; - if (inp == NULL) - return EINVAL; /* ??? possible? panic instead? */ - soisdisconnected(so); - crit_enter(); - in_pcbdetach(inp); - crit_exit(); - return 0; + if (inp) { + soisdisconnected(so); + in_pcbdetach(inp); + error = 0; + } else { + error = EINVAL; + } + return error; } static int @@ -908,9 +912,7 @@ udp_attach(struct socket *so, int proto, struct pru_attach_info *ai) error = soreserve(so, udp_sendspace, udp_recvspace, ai->sb_rlimit); if (error) return error; - crit_enter(); error = in_pcballoc(so, &udbinfo); - crit_exit(); if (error) return error; so->so_port = udp_soport_attach(so); @@ -931,9 +933,7 @@ udp_bind(struct socket *so, struct sockaddr *nam, struct thread *td) inp = so->so_pcb; if (inp == NULL) return EINVAL; - crit_enter(); error = in_pcbbind(inp, nam, td); - crit_exit(); if (error == 0) { if (sin->sin_addr.s_addr != INADDR_ANY) inp->inp_flags |= INP_WASBOUND_NOTANY; @@ -1079,9 +1079,7 @@ udp_detach(struct socket *so) inp = so->so_pcb; if (inp == NULL) return EINVAL; - crit_enter(); in_pcbdetach(inp); - crit_exit(); return 0; } @@ -1097,10 +1095,10 @@ udp_disconnect(struct socket *so) if (inp->inp_faddr.s_addr == INADDR_ANY) return ENOTCONN; - crit_enter(); + soreference(so); in_pcbdisconnect(inp); - crit_exit(); - so->so_state &= ~SS_ISCONNECTED; /* XXX */ + soclrstate(so, SS_ISCONNECTED); /* XXX */ + sofree(so); ro = &inp->inp_route; if (ro->ro_rt != NULL) diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c index 8f1b8207bc..a69fee265b 100644 --- a/sys/netinet6/in6_pcb.c +++ b/sys/netinet6/in6_pcb.c @@ -685,7 +685,7 @@ in6_pcbdetach(struct inpcb *inp) inp->inp_gencnt = ++ipi->ipi_gencnt; in_pcbremlists(inp); so->so_pcb = NULL; - sofree(so); + sofree(so); /* remove pcb ref */ if (inp->in6p_options) m_freem(inp->in6p_options); diff --git a/sys/netinet6/ipsec.c b/sys/netinet6/ipsec.c index 6a771b753e..b502ce08d5 100644 --- a/sys/netinet6/ipsec.c +++ b/sys/netinet6/ipsec.c @@ -292,12 +292,15 @@ ipsec4_getpolicybysock(struct mbuf *m, u_int dir, struct socket *so, int *error) if (currsp == NULL) panic("ipsec4_getpolicybysock: currsp is NULL."); + lwkt_gettoken(&key_token); + /* when privilieged socket */ if (pcbsp->priv) { switch (currsp->policy) { case IPSEC_POLICY_BYPASS: currsp->refcnt++; *error = 0; + lwkt_reltoken(&key_token); return currsp; case IPSEC_POLICY_ENTRUST: @@ -310,6 +313,7 @@ ipsec4_getpolicybysock(struct mbuf *m, u_int dir, struct socket *so, int *error) kprintf("DP ipsec4_getpolicybysock called " "to allocate SP:%p\n", kernsp)); *error = 0; + lwkt_reltoken(&key_token); return kernsp; } @@ -323,17 +327,20 @@ ipsec4_getpolicybysock(struct mbuf *m, u_int dir, struct socket *so, int *error) } ip4_def_policy.refcnt++; *error = 0; + lwkt_reltoken(&key_token); return &ip4_def_policy; case IPSEC_POLICY_IPSEC: currsp->refcnt++; *error = 0; + lwkt_reltoken(&key_token); return currsp; default: ipseclog((LOG_ERR, "ipsec4_getpolicybysock: " "Invalid policy for PCB %d\n", currsp->policy)); *error = EINVAL; + lwkt_reltoken(&key_token); return NULL; } /* NOTREACHED */ @@ -349,6 +356,7 @@ ipsec4_getpolicybysock(struct mbuf *m, u_int dir, struct socket *so, int *error) kprintf("DP ipsec4_getpolicybysock called " "to allocate SP:%p\n", kernsp)); *error = 0; + lwkt_reltoken(&key_token); return kernsp; } @@ -359,6 +367,7 @@ ipsec4_getpolicybysock(struct mbuf *m, u_int dir, struct socket *so, int *error) "Illegal policy for non-priviliged defined %d\n", currsp->policy)); *error = EINVAL; + lwkt_reltoken(&key_token); return NULL; case IPSEC_POLICY_ENTRUST: @@ -371,17 +380,20 @@ ipsec4_getpolicybysock(struct mbuf *m, u_int dir, struct socket *so, int *error) } ip4_def_policy.refcnt++; *error = 0; + lwkt_reltoken(&key_token); return &ip4_def_policy; case IPSEC_POLICY_IPSEC: currsp->refcnt++; *error = 0; + lwkt_reltoken(&key_token); return currsp; default: ipseclog((LOG_ERR, "ipsec4_getpolicybysock: " "Invalid policy for PCB %d\n", currsp->policy)); *error = EINVAL; + lwkt_reltoken(&key_token); return NULL; } /* NOTREACHED */ diff --git a/sys/netinet6/raw_ip6.c b/sys/netinet6/raw_ip6.c index 0204ad4bdb..c795a9e502 100644 --- a/sys/netinet6/raw_ip6.c +++ b/sys/netinet6/raw_ip6.c @@ -79,7 +79,9 @@ #include #include #include + #include +#include #include #include @@ -602,22 +604,35 @@ rip6_detach(struct socket *so) return 0; } +/* + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. + */ static int rip6_abort(struct socket *so) { + int error; + soisdisconnected(so); - return rip6_detach(so); + error = rip6_detach(so); + + return error; } static int rip6_disconnect(struct socket *so) { struct inpcb *inp = so->so_pcb; + int error; if (!(so->so_state & SS_ISCONNECTED)) return ENOTCONN; inp->in6p_faddr = kin6addr_any; - return rip6_abort(so); + soreference(so); + error = rip6_abort(so); + sofree(so); + + return error; } static int diff --git a/sys/netinet6/sctp6_usrreq.c b/sys/netinet6/sctp6_usrreq.c index eccd6da353..d6a0a6d981 100644 --- a/sys/netinet6/sctp6_usrreq.c +++ b/sys/netinet6/sctp6_usrreq.c @@ -724,20 +724,26 @@ SYSCTL_PROC(_net_inet6_sctp6, OID_AUTO, getcred, CTLTYPE_OPAQUE|CTLFLAG_RW, #endif -/* This is the same as the sctp_abort() could be made common */ +/* + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. + */ static int sctp6_abort(struct socket *so) { struct sctp_inpcb *inp; + int error; inp = (struct sctp_inpcb *)so->so_pcb; - if (inp == NULL) - return EINVAL; /* ??? possible? panic instead? */ - soisdisconnected(so); - crit_enter(); - sctp_inpcb_free(inp, 1); - crit_exit(); - return 0; + if (inp) { + soisdisconnected(so); + sctp_inpcb_free(inp, 1); + error = 0; + } else { + error = EINVAL; + } + + return error; } static int diff --git a/sys/netinet6/udp6_usrreq.c b/sys/netinet6/udp6_usrreq.c index 795aea862d..a1d50eae76 100644 --- a/sys/netinet6/udp6_usrreq.c +++ b/sys/netinet6/udp6_usrreq.c @@ -83,7 +83,9 @@ #include #include #include + #include +#include #include #include @@ -523,19 +525,26 @@ SYSCTL_PROC(_net_inet6_udp6, OID_AUTO, getcred, CTLTYPE_OPAQUE|CTLFLAG_RW, 0, 0, udp6_getcred, "S,ucred", "Get the ucred of a UDP6 connection"); +/* + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. + */ static int udp6_abort(struct socket *so) { struct inpcb *inp; + int error; inp = so->so_pcb; - if (inp == NULL) - return EINVAL; /* ??? possible? panic instead? */ - soisdisconnected(so); - crit_enter(); - in6_pcbdetach(inp); - crit_exit(); - return 0; + if (inp) { + soisdisconnected(so); + in6_pcbdetach(inp); + error = 0; + } else { + error = EINVAL; + } + + return error; } static int @@ -712,7 +721,7 @@ udp6_disconnect(struct socket *so) crit_enter(); in6_pcbdisconnect(inp); crit_exit(); - so->so_state &= ~SS_ISCONNECTED; /* XXX */ + soclrstate(so, SS_ISCONNECTED); /* XXX */ return 0; } diff --git a/sys/netproto/atalk/ddp_usrreq.c b/sys/netproto/atalk/ddp_usrreq.c index ae6c86a5c2..25646dde2f 100644 --- a/sys/netproto/atalk/ddp_usrreq.c +++ b/sys/netproto/atalk/ddp_usrreq.c @@ -50,9 +50,7 @@ ddp_attach(struct socket *so, int proto, struct pru_attach_info *ai) return( EINVAL); } - crit_enter(); error = at_pcballoc( so ); - crit_exit(); if (error) { return (error); } @@ -68,9 +66,7 @@ ddp_detach(struct socket *so) if ( ddp == NULL ) { return( EINVAL); } - crit_enter(); at_pcbdetach( so, ddp ); - crit_exit(); return(0); } @@ -84,9 +80,7 @@ ddp_bind(struct socket *so, struct sockaddr *nam, struct thread *td) if ( ddp == NULL ) { return( EINVAL); } - crit_enter(); error = at_pcbsetaddr(ddp, nam, td); - crit_exit(); return (error); } @@ -105,9 +99,7 @@ ddp_connect(struct socket *so, struct sockaddr *nam, struct thread *td) return(EISCONN); } - crit_enter(); error = at_pcbconnect( ddp, nam, td ); - crit_exit(); if ( error == 0 ) soisconnected( so ); return(error); @@ -127,11 +119,11 @@ ddp_disconnect(struct socket *so) return(ENOTCONN); } - crit_enter(); + soreference(so); at_pcbdisconnect( ddp ); ddp->ddp_fsat.sat_addr.s_node = ATADDR_ANYNODE; - crit_exit(); soisdisconnected( so ); + sofree(sp); /* soref above */ return(0); } @@ -169,9 +161,7 @@ ddp_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *addr, return(EISCONN); } - crit_enter(); error = at_pcbconnect(ddp, addr, td); - crit_exit(); if ( error ) { return(error); } @@ -181,29 +171,32 @@ ddp_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *addr, } } - crit_enter(); error = ddp_output( m, so ); if ( addr ) { at_pcbdisconnect( ddp ); } - crit_exit(); return(error); } +/* + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. + */ static int ddp_abort(struct socket *so) { - struct ddpcb *ddp; + struct ddpcb *ddp; + int error; - ddp = sotoddpcb( so ); - if ( ddp == NULL ) { - return(EINVAL); + ddp = sotoddpcb(so); + if (ddb) { + soisdisconnected( so ); + at_pcbdetach( so, ddp ); + error = 0; + } else { + error = EINVAL; } - soisdisconnected( so ); - crit_enter(); - at_pcbdetach( so, ddp ); - crit_exit(); - return(0); + return error; } @@ -435,9 +428,9 @@ at_pcballoc( struct socket *so ) static void at_pcbdetach( struct socket *so, struct ddpcb *ddp) { - soisdisconnected( so ); - so->so_pcb = 0; - sofree( so ); + soisdisconnected(so); + so->so_pcb = NULL; + sofree(so); /* remove ddp from ddp_ports list */ if ( ddp->ddp_lsat.sat_port != ATADDR_ANYPORT && diff --git a/sys/netproto/atm/atm_aal5.c b/sys/netproto/atm/atm_aal5.c index c46d98c2c3..3d52712f26 100644 --- a/sys/netproto/atm/atm_aal5.c +++ b/sys/netproto/atm/atm_aal5.c @@ -518,6 +518,8 @@ out: * 0 request processed * error error processing request - reason indicated * + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. */ static int atm_aal5_abort(struct socket *so) @@ -526,7 +528,6 @@ atm_aal5_abort(struct socket *so) so->so_error = ECONNABORTED; err = atm_sock_detach(so); - ATM_OUTRO(); } diff --git a/sys/netproto/atm/atm_socket.c b/sys/netproto/atm/atm_socket.c index eb84bcb163..4c763c8455 100644 --- a/sys/netproto/atm/atm_socket.c +++ b/sys/netproto/atm/atm_socket.c @@ -150,7 +150,7 @@ atm_sock_detach(struct socket *so) * Break links and free control blocks */ so->so_pcb = NULL; - sofree(so); + sofree(so); /* remove pcb ref */ atm_free((caddr_t)atp); diff --git a/sys/netproto/ipsec/keysock.c b/sys/netproto/ipsec/keysock.c index 2cca895d8f..673353f3e2 100644 --- a/sys/netproto/ipsec/keysock.c +++ b/sys/netproto/ipsec/keysock.c @@ -355,9 +355,9 @@ static int key_abort(struct socket *so) { int error; - crit_enter(); + error = raw_usrreqs.pru_abort(so); - crit_exit(); + return error; } diff --git a/sys/netproto/ipx/ipx_pcb.c b/sys/netproto/ipx/ipx_pcb.c index 61c147e4f2..352f0b2078 100644 --- a/sys/netproto/ipx/ipx_pcb.c +++ b/sys/netproto/ipx/ipx_pcb.c @@ -261,8 +261,9 @@ ipx_pcbdetach(struct ipxpcb *ipxp) { struct socket *so = ipxp->ipxp_socket; - so->so_pcb = 0; + so->so_pcb = NULL; sofree(so); + if (ipxp->ipxp_route.ro_rt != NULL) rtfree(ipxp->ipxp_route.ro_rt); remque(ipxp); diff --git a/sys/netproto/ipx/ipx_usrreq.c b/sys/netproto/ipx/ipx_usrreq.c index fd84623628..9aaee0ed64 100644 --- a/sys/netproto/ipx/ipx_usrreq.c +++ b/sys/netproto/ipx/ipx_usrreq.c @@ -188,8 +188,10 @@ ipx_abort(struct ipxpcb *ipxp) { struct socket *so = ipxp->ipxp_socket; + soreference(so); ipx_pcbdisconnect(ipxp); soisdisconnected(so); + sofree(so); } /* @@ -213,8 +215,10 @@ ipx_drop(struct ipxpcb *ipxp, int errno) tcp_output(tp); }*/ so->so_error = errno; + soreference(so); ipx_pcbdisconnect(ipxp); soisdisconnected(so); + sofree(so); } static int @@ -443,16 +447,18 @@ ipx_ctloutput(struct socket *so, struct sockopt *sopt) return (error); } +/* + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. + */ static int ipx_usr_abort(struct socket *so) { struct ipxpcb *ipxp = sotoipxpcb(so); - crit_enter(); ipx_pcbdetach(ipxp); - crit_exit(); - sofree(so); soisdisconnected(so); + return (0); } @@ -464,9 +470,7 @@ ipx_attach(struct socket *so, int proto, struct pru_attach_info *ai) if (ipxp != NULL) return (EINVAL); - crit_enter(); error = ipx_pcballoc(so, &ipxpcb); - crit_exit(); if (error == 0) error = soreserve(so, ipxsendspace, ipxrecvspace, ai->sb_rlimit); @@ -489,9 +493,7 @@ ipx_connect(struct socket *so, struct sockaddr *nam, struct thread *td) if (!ipx_nullhost(ipxp->ipxp_faddr)) return (EISCONN); - crit_enter(); error = ipx_pcbconnect(ipxp, nam, td); - crit_exit(); if (error == 0) soisconnected(so); return (error); @@ -504,9 +506,7 @@ ipx_detach(struct socket *so) if (ipxp == NULL) return (ENOTCONN); - crit_enter(); ipx_pcbdetach(ipxp); - crit_exit(); return (0); } @@ -517,10 +517,11 @@ ipx_disconnect(struct socket *so) if (ipx_nullhost(ipxp->ipxp_faddr)) return (ENOTCONN); - crit_enter(); + soreference(so); ipx_pcbdisconnect(ipxp); - crit_exit(); soisdisconnected(so); + sofree(so); + return (0); } @@ -541,7 +542,6 @@ ipx_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam, struct ipxpcb *ipxp = sotoipxpcb(so); struct ipx_addr laddr; - crit_enter(); if (nam != NULL) { laddr = ipxp->ipxp_laddr; if (!ipx_nullhost(ipxp->ipxp_faddr)) { @@ -568,7 +568,6 @@ ipx_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam, ipxp->ipxp_laddr = laddr; } send_release: - crit_exit(); if (m != NULL) m_freem(m); return (error); @@ -598,9 +597,7 @@ ripx_attach(struct socket *so, int proto, struct pru_attach_info *ai) if ((error = priv_check_cred(ai->p_ucred, PRIV_ROOT, NULL_CRED_OKAY)) != 0) return (error); - crit_enter(); error = ipx_pcballoc(so, &ipxrawpcb); - crit_exit(); if (error) return (error); error = soreserve(so, ipxsendspace, ipxrecvspace, ai->sb_rlimit); diff --git a/sys/netproto/ipx/spx_usrreq.c b/sys/netproto/ipx/spx_usrreq.c index 477fffa08d..9eed8e4bcd 100644 --- a/sys/netproto/ipx/spx_usrreq.c +++ b/sys/netproto/ipx/spx_usrreq.c @@ -601,7 +601,7 @@ present: if (so->so_rcv.ssb_cc) so->so_oobmark = so->so_rcv.ssb_cc; else - so->so_state |= SS_RCVATMARK; + sosetstate(so, SS_RCVATMARK); } nq = q; q = q->si_prev; @@ -633,7 +633,7 @@ present: m_chtype(m, MT_OOBDATA); spx_newchecks[1]++; so->so_oobmark = 0; - so->so_state &= ~SS_RCVATMARK; + soclrstate(so, SS_RCVATMARK); } if (packetp == 0) { m->m_data += SPINC; @@ -1293,6 +1293,10 @@ spx_ctloutput(struct socket *so, struct sockopt *sopt) return (error); } +/* + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. + */ static int spx_usr_abort(struct socket *so) { @@ -1302,9 +1306,8 @@ spx_usr_abort(struct socket *so) ipxp = sotoipxpcb(so); cb = ipxtospxpcb(ipxp); - crit_enter(); spx_drop(cb, ECONNABORTED); - crit_exit(); + return (0); } diff --git a/sys/netproto/key/key.c b/sys/netproto/key/key.c index a00d8dfc73..d1488fd59f 100644 --- a/sys/netproto/key/key.c +++ b/sys/netproto/key/key.c @@ -153,6 +153,8 @@ static LIST_HEAD(_acqtree, secacq) acqtree; /* acquiring list */ #endif static LIST_HEAD(_spacqtree, secspacq) spacqtree; /* SP acquiring list */ +struct lwkt_token key_token = LWKT_TOKEN_MP_INITIALIZER(key_token); + struct key_cb key_cb; /* search order for SAs */ @@ -526,7 +528,7 @@ key_allocsp(struct secpolicyindex *spidx, u_int dir) } /* get a SP entry */ - crit_enter(); + lwkt_gettoken(&key_token); KEYDEBUG(KEYDEBUG_IPSEC_DATA, kprintf("*** objects\n"); kdebug_secpolicyindex(spidx)); @@ -542,7 +544,7 @@ key_allocsp(struct secpolicyindex *spidx, u_int dir) goto found; } - crit_exit(); + lwkt_reltoken(&key_token); return NULL; found: @@ -553,7 +555,7 @@ found: microtime(&tv); sp->lastused = tv.tv_sec; sp->refcnt++; - crit_exit(); + lwkt_reltoken(&key_token); KEYDEBUG(KEYDEBUG_IPSEC_STAMP, kprintf("DP key_allocsp cause refcnt++:%d SP:%p\n", sp->refcnt, sp)); @@ -582,7 +584,7 @@ key_gettunnel(struct sockaddr *osrc, struct sockaddr *odst, return NULL; } - crit_enter(); + lwkt_gettoken(&key_token); LIST_FOREACH(sp, &sptree[dir], chain) { if (sp->state == IPSEC_SPSTATE_DEAD) continue; @@ -622,14 +624,14 @@ key_gettunnel(struct sockaddr *osrc, struct sockaddr *odst, goto found; } } - crit_exit(); + lwkt_reltoken(&key_token); return NULL; found: microtime(&tv); sp->lastused = tv.tv_sec; sp->refcnt++; - crit_exit(); + lwkt_reltoken(&key_token); return sp; } @@ -659,6 +661,8 @@ key_checkrequest(struct ipsecrequest *isr, struct secasindex *saidx) panic("key_checkrequest: Invalid policy defined.\n"); } + lwkt_gettoken(&key_token); + /* get current level */ level = ipsec_get_reqlevel(isr); @@ -706,17 +710,21 @@ key_checkrequest(struct ipsecrequest *isr, struct secasindex *saidx) isr->sav = key_allocsa_policy(saidx); /* When there is SA. */ - if (isr->sav != NULL) + if (isr->sav != NULL) { + lwkt_reltoken(&key_token); return 0; + } /* there is no SA */ if ((error = key_acquire(saidx, isr->sp)) != 0) { /* XXX What should I do ? */ ipseclog((LOG_DEBUG, "key_checkrequest: error %d returned " "from key_acquire.\n", error)); + lwkt_reltoken(&key_token); return error; } + lwkt_reltoken(&key_token); return level == IPSEC_LEVEL_REQUIRE ? ENOENT : 0; } @@ -945,7 +953,7 @@ key_allocsa(u_int family, caddr_t src, caddr_t dst, u_int proto, * IPsec tunnel packet is received. But ESP tunnel mode is * encrypted so we can't check internal IP header. */ - crit_enter(); + lwkt_gettoken(&key_token); LIST_FOREACH(sah, &sahtree, chain) { /* * search a valid state list for inbound packet. @@ -1044,12 +1052,12 @@ key_allocsa(u_int family, caddr_t src, caddr_t dst, u_int proto, } /* not found */ - crit_exit(); + lwkt_reltoken(&key_token); return NULL; found: sav->refcnt++; - crit_exit(); + lwkt_reltoken(&key_token); KEYDEBUG(KEYDEBUG_IPSEC_STAMP, kprintf("DP allocsa cause refcnt++:%d SA:%p\n", sav->refcnt, sav)); @@ -1067,6 +1075,7 @@ key_freesp(struct secpolicy *sp) if (sp == NULL) panic("key_freesp: NULL pointer is passed.\n"); + lwkt_gettoken(&key_token); sp->refcnt--; KEYDEBUG(KEYDEBUG_IPSEC_STAMP, kprintf("DP freesp cause refcnt--:%d SP:%p\n", @@ -1074,8 +1083,7 @@ key_freesp(struct secpolicy *sp) if (sp->refcnt == 0) key_delsp(sp); - - return; + lwkt_reltoken(&key_token); } /* @@ -1089,6 +1097,7 @@ key_freeso(struct socket *so) if (so == NULL) panic("key_freeso: NULL pointer is passed.\n"); + lwkt_gettoken(&key_token); switch (so->so_proto->pr_domain->dom_family) { #ifdef INET case PF_INET: @@ -1097,7 +1106,7 @@ key_freeso(struct socket *so) /* Does it have a PCB ? */ if (pcb == NULL) - return; + break; key_freesp_so(&pcb->inp_sp->sp_in); key_freesp_so(&pcb->inp_sp->sp_out); } @@ -1111,7 +1120,7 @@ key_freeso(struct socket *so) /* Does it have a PCB ? */ if (pcb == NULL) - return; + break; key_freesp_so(&pcb->inp_sp->sp_in); key_freesp_so(&pcb->inp_sp->sp_out); #else @@ -1119,7 +1128,7 @@ key_freeso(struct socket *so) /* Does it have a PCB ? */ if (pcb == NULL) - return; + break; key_freesp_so(&pcb->in6p_sp->sp_in); key_freesp_so(&pcb->in6p_sp->sp_out); #endif @@ -1129,10 +1138,9 @@ key_freeso(struct socket *so) default: ipseclog((LOG_DEBUG, "key_freeso: unknown address family=%d.\n", so->so_proto->pr_domain->dom_family)); - return; + break; } - - return; + lwkt_reltoken(&key_token); } static void @@ -1171,6 +1179,7 @@ key_freesav(struct secasvar *sav) if (sav == NULL) panic("key_freesav: NULL pointer is passed.\n"); + lwkt_gettoken(&key_token); sav->refcnt--; KEYDEBUG(KEYDEBUG_IPSEC_STAMP, kprintf("DP freesav cause refcnt--:%d SA:%p SPI %u\n", @@ -1179,7 +1188,7 @@ key_freesav(struct secasvar *sav) if (sav->refcnt == 0) key_delsav(sav); - return; + lwkt_reltoken(&key_token); } /* %%% SPD management */ @@ -1198,7 +1207,6 @@ key_delsp(struct secpolicy *sp) if (sp->refcnt > 0) return; /* can't free */ - crit_enter(); /* remove from SP index */ if (__LIST_CHAINED(sp)) LIST_REMOVE(sp, chain); @@ -1222,10 +1230,6 @@ key_delsp(struct secpolicy *sp) } keydb_delsecpolicy(sp); - - crit_exit(); - - return; } /* @@ -1290,12 +1294,13 @@ key_newsp(void) { struct secpolicy *newsp = NULL; + lwkt_gettoken(&key_token); newsp = keydb_newsecpolicy(); - if (!newsp) - return newsp; - - newsp->refcnt = 1; - newsp->req = NULL; + if (newsp) { + newsp->refcnt = 1; + newsp->req = NULL; + } + lwkt_reltoken(&key_token); return newsp; } @@ -1321,7 +1326,9 @@ key_msg2sp(struct sadb_x_policy *xpl0, size_t len, int *error) return NULL; } + lwkt_gettoken(&key_token); if ((newsp = key_newsp()) == NULL) { + lwkt_reltoken(&key_token); *error = ENOBUFS; return NULL; } @@ -1349,6 +1356,7 @@ key_msg2sp(struct sadb_x_policy *xpl0, size_t len, int *error) ipseclog((LOG_DEBUG, "key_msg2sp: Invalid msg length.\n")); key_freesp(newsp); + lwkt_reltoken(&key_token); *error = EINVAL; return NULL; } @@ -1363,6 +1371,7 @@ key_msg2sp(struct sadb_x_policy *xpl0, size_t len, int *error) ipseclog((LOG_DEBUG, "key_msg2sp: " "invalid ipsecrequest length.\n")); key_freesp(newsp); + lwkt_reltoken(&key_token); *error = EINVAL; return NULL; } @@ -1373,6 +1382,7 @@ key_msg2sp(struct sadb_x_policy *xpl0, size_t len, int *error) ipseclog((LOG_DEBUG, "key_msg2sp: No more memory.\n")); key_freesp(newsp); + lwkt_reltoken(&key_token); *error = ENOBUFS; return NULL; } @@ -1391,6 +1401,7 @@ key_msg2sp(struct sadb_x_policy *xpl0, size_t len, int *error) "key_msg2sp: invalid proto type=%u\n", xisr->sadb_x_ipsecrequest_proto)); key_freesp(newsp); + lwkt_reltoken(&key_token); *error = EPROTONOSUPPORT; return NULL; } @@ -1406,6 +1417,7 @@ key_msg2sp(struct sadb_x_policy *xpl0, size_t len, int *error) "key_msg2sp: invalid mode=%u\n", xisr->sadb_x_ipsecrequest_mode)); key_freesp(newsp); + lwkt_reltoken(&key_token); *error = EINVAL; return NULL; } @@ -1436,6 +1448,7 @@ key_msg2sp(struct sadb_x_policy *xpl0, size_t len, int *error) u_int32_t reqid; if ((reqid = key_newreqid()) == 0) { key_freesp(newsp); + lwkt_reltoken(&key_token); *error = ENOBUFS; return NULL; } @@ -1452,6 +1465,7 @@ key_msg2sp(struct sadb_x_policy *xpl0, size_t len, int *error) ipseclog((LOG_DEBUG, "key_msg2sp: invalid level=%u\n", xisr->sadb_x_ipsecrequest_level)); key_freesp(newsp); + lwkt_reltoken(&key_token); *error = EINVAL; return NULL; } @@ -1469,6 +1483,7 @@ key_msg2sp(struct sadb_x_policy *xpl0, size_t len, int *error) ipseclog((LOG_DEBUG, "key_msg2sp: invalid request " "address length.\n")); key_freesp(newsp); + lwkt_reltoken(&key_token); *error = EINVAL; return NULL; } @@ -1484,6 +1499,7 @@ key_msg2sp(struct sadb_x_policy *xpl0, size_t len, int *error) ipseclog((LOG_DEBUG, "key_msg2sp: invalid request " "address length.\n")); key_freesp(newsp); + lwkt_reltoken(&key_token); *error = EINVAL; return NULL; } @@ -1502,6 +1518,7 @@ key_msg2sp(struct sadb_x_policy *xpl0, size_t len, int *error) if (tlen < 0) { ipseclog((LOG_DEBUG, "key_msg2sp: becoming tlen < 0.\n")); key_freesp(newsp); + lwkt_reltoken(&key_token); *error = EINVAL; return NULL; } @@ -1514,10 +1531,11 @@ key_msg2sp(struct sadb_x_policy *xpl0, size_t len, int *error) default: ipseclog((LOG_DEBUG, "key_msg2sp: invalid policy type.\n")); key_freesp(newsp); + lwkt_reltoken(&key_token); *error = EINVAL; return NULL; } - + lwkt_reltoken(&key_token); *error = 0; return newsp; } @@ -1550,12 +1568,14 @@ key_sp2msg(struct secpolicy *sp) if (sp == NULL) panic("key_sp2msg: NULL pointer was passed.\n"); + lwkt_gettoken(&key_token); tlen = key_getspreqmsglen(sp); m = key_alloc_mbuf(tlen); if (!m || m->m_next) { /*XXX*/ if (m) m_freem(m); + lwkt_reltoken(&key_token); return NULL; } @@ -1597,7 +1617,7 @@ key_sp2msg(struct secpolicy *sp) + isr->saidx.dst.ss_len); } } - + lwkt_reltoken(&key_token); return m; } @@ -2200,6 +2220,7 @@ key_spdacquire(struct secpolicy *sp) if (sp->policy != IPSEC_POLICY_IPSEC) panic("key_spdacquire: policy mismatched. IPsec is expected.\n"); + lwkt_gettoken(&key_token); /* get a entry to check whether sent message or not. */ if ((newspacq = key_getspacq(&sp->spidx)) != NULL) { if (key_blockacq_count < newspacq->count) { @@ -2208,12 +2229,15 @@ key_spdacquire(struct secpolicy *sp) } else { /* increment counter and do nothing. */ newspacq->count++; + lwkt_reltoken(&key_token); return 0; } } else { /* make new entry for blocking to send SADB_ACQUIRE. */ - if ((newspacq = key_newspacq(&sp->spidx)) == NULL) + if ((newspacq = key_newspacq(&sp->spidx)) == NULL) { + lwkt_reltoken(&key_token); return ENOBUFS; + } /* add to acqtree */ LIST_INSERT_HEAD(&spacqtree, newspacq, chain); @@ -2234,9 +2258,12 @@ key_spdacquire(struct secpolicy *sp) mtod(result, struct sadb_msg *)->sadb_msg_len = PFKEY_UNIT64(result->m_pkthdr.len); - return key_sendup_mbuf(NULL, m, KEY_SENDUP_REGISTERED); + error = key_sendup_mbuf(NULL, m, KEY_SENDUP_REGISTERED); + lwkt_reltoken(&key_token); + return error; fail: + lwkt_reltoken(&key_token); if (result) m_freem(result); return error; @@ -2443,7 +2470,6 @@ key_spdexpire(struct secpolicy *sp) struct sadb_lifetime *lt; /* XXX: Why do we lock ? */ - crit_enter(); /* sanity check */ if (sp == NULL) @@ -2536,7 +2562,6 @@ key_spdexpire(struct secpolicy *sp) fail: if (result) m_freem(result); - crit_exit(); return error; } @@ -2582,8 +2607,6 @@ key_delsah(struct secashead *sah) if (sah == NULL) panic("key_delsah: NULL pointer is passed.\n"); - crit_enter(); - /* searching all SA registerd in the secindex. */ for (stateidx = 0; stateidx < _ARRAYLEN(saorder_state_any); @@ -2614,10 +2637,8 @@ key_delsah(struct secashead *sah) } /* don't delete sah only if there are savs. */ - if (zombie) { - crit_exit(); + if (zombie) return; - } if (sah->sa_route.ro_rt) { RTFREE(sah->sa_route.ro_rt); @@ -2630,7 +2651,6 @@ key_delsah(struct secashead *sah) KFREE(sah); - crit_exit(); return; } @@ -3728,11 +3748,13 @@ key_ismyaddr(struct sockaddr *sa) struct sockaddr_in *sin; struct in_ifaddr_container *iac; #endif + int res; /* sanity check */ if (sa == NULL) panic("key_ismyaddr: NULL pointer is passed.\n"); + lwkt_gettoken(&key_token); switch (sa->sa_family) { #ifdef INET case AF_INET: @@ -3744,18 +3766,24 @@ key_ismyaddr(struct sockaddr *sa) sin->sin_len == ia->ia_addr.sin_len && sin->sin_addr.s_addr == ia->ia_addr.sin_addr.s_addr) { + lwkt_reltoken(&key_token); return 1; } } + res = 0; break; #endif #ifdef INET6 case AF_INET6: - return key_ismyaddr6((struct sockaddr_in6 *)sa); + res = key_ismyaddr6((struct sockaddr_in6 *)sa); + break; #endif + default: + res = 0; + break; } - - return 0; + lwkt_reltoken(&key_token); + return res; } #ifdef INET6 @@ -4099,7 +4127,7 @@ key_timehandler(void *__dummy) microtime(&tv); - crit_enter(); + lwkt_gettoken(&key_token); /* SPD */ { @@ -4349,7 +4377,7 @@ key_timehandler(void *__dummy) callout_reset(&key_timehandler_ch, hz, key_timehandler, NULL); #endif /* IPSEC_DEBUG2 */ - crit_exit(); + lwkt_reltoken(&key_token); return; } @@ -6262,6 +6290,7 @@ key_freereg(struct socket *so) * check all type of SA, because there is a potential that * one socket is registered to multiple type of SA. */ + lwkt_gettoken(&key_token); for (i = 0; i <= SADB_SATYPE_MAX; i++) { LIST_FOREACH(reg, ®tree[i], chain) { if (reg->so == so @@ -6272,8 +6301,7 @@ key_freereg(struct socket *so) } } } - - return; + lwkt_reltoken(&key_token); } /* @@ -6295,9 +6323,6 @@ key_expire(struct secasvar *sav) int error = -1; struct sadb_lifetime *lt; - /* XXX: Why do we lock ? */ - crit_enter(); - /* sanity check */ if (sav == NULL) panic("key_expire: NULL pointer is passed.\n"); @@ -6393,13 +6418,11 @@ key_expire(struct secasvar *sav) mtod(result, struct sadb_msg *)->sadb_msg_len = PFKEY_UNIT64(result->m_pkthdr.len); - crit_exit(); return key_sendup_mbuf(NULL, result, KEY_SENDUP_REGISTERED); fail: if (result) m_freem(result); - crit_exit(); return error; } @@ -6905,11 +6928,17 @@ key_parse(struct mbuf *m, struct socket *so) goto senderror; } - return (*key_typesw[msg->sadb_msg_type])(so, m, &mh); + lwkt_gettoken(&key_token); + error = (*key_typesw[msg->sadb_msg_type])(so, m, &mh); + lwkt_reltoken(&key_token); + return error; senderror: msg->sadb_msg_errno = error; - return key_sendup_mbuf(so, m, target); + lwkt_gettoken(&key_token); + error = key_sendup_mbuf(so, m, target); + lwkt_reltoken(&key_token); + return error; } static int @@ -7274,6 +7303,7 @@ key_sa_routechange(struct sockaddr *dst) struct secashead *sah; struct route *ro; + lwkt_gettoken(&key_token); LIST_FOREACH(sah, &sahtree, chain) { ro = &sah->sa_route; if (ro->ro_rt && dst->sa_len == ro->ro_dst.sa_len @@ -7282,8 +7312,7 @@ key_sa_routechange(struct sockaddr *dst) ro->ro_rt = NULL; } } - - return; + lwkt_reltoken(&key_token); } static void @@ -7305,7 +7334,6 @@ key_sa_chgstate(struct secasvar *sav, u_int8_t state) void key_sa_stir_iv(struct secasvar *sav) { - if (!sav->iv) panic("key_sa_stir_iv called with sav == NULL"); key_randomfill(sav->iv, sav->ivlen); diff --git a/sys/netproto/key/key.h b/sys/netproto/key/key.h index aada28c891..2c904122f7 100644 --- a/sys/netproto/key/key.h +++ b/sys/netproto/key/key.h @@ -37,6 +37,7 @@ #ifdef _KERNEL extern struct key_cb key_cb; +extern struct lwkt_token key_token; struct secpolicy; struct secpolicyindex; diff --git a/sys/netproto/key/keydb.c b/sys/netproto/key/keydb.c index 2f8f580d87..f3a9f63031 100644 --- a/sys/netproto/key/keydb.c +++ b/sys/netproto/key/keydb.c @@ -52,6 +52,7 @@ #include #include "keydb.h" +#include "key.h" #include #include @@ -119,20 +120,20 @@ keydb_newsecasvar(void) void keydb_refsecasvar(struct secasvar *p) { - crit_enter(); + lwkt_gettoken(&key_token); p->refcnt++; - crit_exit(); + lwkt_reltoken(&key_token); } void keydb_freesecasvar(struct secasvar *p) { - crit_enter(); + lwkt_gettoken(&key_token); p->refcnt--; /* negative refcnt will cause panic intentionally */ if (p->refcnt <= 0) keydb_delsecasvar(p); - crit_exit(); + lwkt_reltoken(&key_token); } static void diff --git a/sys/netproto/key/keysock.c b/sys/netproto/key/keysock.c index 10ed19aa37..37d925fccf 100644 --- a/sys/netproto/key/keysock.c +++ b/sys/netproto/key/keysock.c @@ -112,10 +112,10 @@ key_output(struct mbuf *m, struct socket *so, ...) } /*XXX giant lock*/ - crit_enter(); + lwkt_gettoken(&key_token); error = key_parse(m, so); m = NULL; - crit_exit(); + lwkt_reltoken(&key_token); end: if (m) m_freem(m); @@ -153,13 +153,16 @@ key_sendup0(struct rawcb *rp, struct mbuf *m, int promisc) pfkeystat.in_msgtype[pmsg->sadb_msg_type]++; } - if (!ssb_appendaddr(&rp->rcb_socket->so_rcv, (struct sockaddr *)&key_src, - m, NULL)) { + lwkt_gettoken(&key_token); + if (!ssb_appendaddr(&rp->rcb_socket->so_rcv, + (struct sockaddr *)&key_src, m, NULL)) { pfkeystat.in_nomem++; m_freem(m); error = ENOBUFS; - } else + } else { error = 0; + } + lwkt_reltoken(&key_token); sorwakeup(rp->rcb_socket); return error; } @@ -285,6 +288,8 @@ key_sendup_mbuf(struct socket *so, struct mbuf *m, int target) pfkeystat.in_msgtype[msg->sadb_msg_type]++; } + lwkt_gettoken(&key_token); + LIST_FOREACH(rp, &rawcb_list, list) { if (rp->rcb_proto.sp_family != PF_KEY) @@ -335,16 +340,19 @@ key_sendup_mbuf(struct socket *so, struct mbuf *m, int target) if ((n = m_copy(m, 0, (int)M_COPYALL)) == NULL) { m_freem(m); pfkeystat.in_nomem++; + lwkt_reltoken(&key_token); return ENOBUFS; } if ((error = key_sendup0(rp, n, 0)) != 0) { + lwkt_reltoken(&key_token); m_freem(m); return error; } n = NULL; } + lwkt_reltoken(&key_token); if (so) { error = key_sendup0(sotorawcb(so), m, 0); @@ -365,9 +373,10 @@ key_abort(struct socket *so) { int error; - crit_enter(); + lwkt_gettoken(&key_token); error = raw_usrreqs.pru_abort(so); - crit_exit(); + lwkt_reltoken(&key_token); + return error; } @@ -392,14 +401,14 @@ key_attach(struct socket *so, int proto, struct pru_attach_info *ai) * Probably we should try to do more of this work beforehand and * eliminate the critical section. */ - crit_enter(); + lwkt_gettoken(&key_token); so->so_pcb = (caddr_t)kp; error = raw_usrreqs.pru_attach(so, proto, ai); kp = (struct keycb *)sotorawcb(so); if (error) { kfree(kp, M_PCB); so->so_pcb = (caddr_t) 0; - crit_exit(); + lwkt_reltoken(&key_token); return error; } @@ -413,7 +422,7 @@ key_attach(struct socket *so, int proto, struct pru_attach_info *ai) soisconnected(so); so->so_options |= SO_USELOOPBACK; - crit_exit(); + lwkt_reltoken(&key_token); return 0; } @@ -426,9 +435,9 @@ key_bind(struct socket *so, struct sockaddr *nam, struct thread *td) { int error; - crit_enter(); + lwkt_gettoken(&key_token); error = raw_usrreqs.pru_bind(so, nam, td); /* xxx just EINVAL */ - crit_exit(); + lwkt_reltoken(&key_token); return error; } @@ -441,9 +450,9 @@ key_connect(struct socket *so, struct sockaddr *nam, struct thread *td) { int error; - crit_enter(); + lwkt_gettoken(&key_token); error = raw_usrreqs.pru_connect(so, nam, td); /* XXX just EINVAL */ - crit_exit(); + lwkt_reltoken(&key_token); return error; } @@ -457,7 +466,7 @@ key_detach(struct socket *so) struct keycb *kp = (struct keycb *)sotorawcb(so); int error; - crit_enter(); + lwkt_gettoken(&key_token); if (kp != 0) { if (kp->kp_raw.rcb_proto.sp_protocol == PF_KEY) /* XXX: AF_KEY */ @@ -467,7 +476,7 @@ key_detach(struct socket *so) key_freereg(so); } error = raw_usrreqs.pru_detach(so); - crit_exit(); + lwkt_reltoken(&key_token); return error; } @@ -480,9 +489,9 @@ key_disconnect(struct socket *so) { int error; - crit_enter(); + lwkt_gettoken(&key_token); error = raw_usrreqs.pru_disconnect(so); - crit_exit(); + lwkt_reltoken(&key_token); return error; } @@ -495,9 +504,9 @@ key_peeraddr(struct socket *so, struct sockaddr **nam) { int error; - crit_enter(); + lwkt_gettoken(&key_token); error = raw_usrreqs.pru_peeraddr(so, nam); - crit_exit(); + lwkt_reltoken(&key_token); return error; } @@ -511,9 +520,9 @@ key_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam, { int error; - crit_enter(); + lwkt_gettoken(&key_token); error = raw_usrreqs.pru_send(so, flags, m, nam, control, td); - crit_exit(); + lwkt_reltoken(&key_token); return error; } @@ -526,9 +535,9 @@ key_shutdown(struct socket *so) { int error; - crit_enter(); + lwkt_gettoken(&key_token); error = raw_usrreqs.pru_shutdown(so); - crit_exit(); + lwkt_reltoken(&key_token); return error; } @@ -541,9 +550,9 @@ key_sockaddr(struct socket *so, struct sockaddr **nam) { int error; - crit_enter(); + lwkt_gettoken(&key_token); error = raw_usrreqs.pru_sockaddr(so, nam); - crit_exit(); + lwkt_reltoken(&key_token); return error; } diff --git a/sys/netproto/natm/natm.c b/sys/netproto/natm/natm.c index 9756b5766b..2d4e12f0f2 100644 --- a/sys/netproto/natm/natm.c +++ b/sys/netproto/natm/natm.c @@ -139,9 +139,9 @@ natm_usr_detach(struct socket *so) /* * we turn on 'drain' *before* we sofree. */ - npcb_free(npcb, NPCB_DESTROY); /* drain */ so->so_pcb = NULL; - sofree(so); + npcb_free(npcb, NPCB_DESTROY); /* drain */ + sofree(so); /* remove pcb ref */ out: crit_exit(); return (error); @@ -400,10 +400,18 @@ natm_usr_control(struct socket *so, u_long cmd, caddr_t arg, return (error); } +/* + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. + */ static int natm_usr_abort(struct socket *so) { - return natm_usr_shutdown(so); + int error; + + error = natm_usr_shutdown(so); + + return error; } static int @@ -500,9 +508,9 @@ natm_usrreq(struct socket *so, int req, struct mbuf *m, struct mbuf *nam, * we turn on 'drain' *before* we sofree. */ - npcb_free(npcb, NPCB_DESTROY); /* drain */ so->so_pcb = NULL; - sofree(so); + npcb_free(npcb, NPCB_DESTROY); /* drain */ + sofree(so); /* remove pcb ref */ break; diff --git a/sys/netproto/ncp/ncp_sock.c b/sys/netproto/ncp/ncp_sock.c index efb0201980..f28a75962a 100644 --- a/sys/netproto/ncp/ncp_sock.c +++ b/sys/netproto/ncp/ncp_sock.c @@ -102,7 +102,7 @@ ncp_soconnect(struct socket *so,struct sockaddr *target, struct thread *td) { tsleep((caddr_t)&so->so_timeo, 0, "ncpcon", 2 * hz); if ((so->so_state & SS_ISCONNECTING) && so->so_error == 0 /*&& rep &&*/) { - so->so_state &= ~SS_ISCONNECTING; + soclrstate(so, SS_ISCONNECTING); crit_exit(); goto bad; } diff --git a/sys/netproto/ns/idp_usrreq.c b/sys/netproto/ns/idp_usrreq.c index d8b70f2ea9..1432f0d12d 100644 --- a/sys/netproto/ns/idp_usrreq.c +++ b/sys/netproto/ns/idp_usrreq.c @@ -123,9 +123,12 @@ idp_abort(struct nspcb *nsp) { struct socket *so = nsp->nsp_socket; + soreference(so); ns_pcbdisconnect(nsp); soisdisconnected(so); + sofree(so); } + /* * Drop connection, reporting * the specified error. @@ -145,8 +148,10 @@ idp_drop(struct nspcb *nsp, int errno) tcp_output(tp); }*/ so->so_error = errno; + soreference(so); ns_pcbdisconnect(nsp); soisdisconnected(so); + sofree(so); } int noIdpRoute; @@ -384,6 +389,10 @@ idp_ctloutput(int req, struct socket *so, int level, int name, * IDP_USRREQ PROCEDURES */ +/* + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. + */ static int idp_usr_abort(struct socket *so) { @@ -392,12 +401,12 @@ idp_usr_abort(struct socket *so) if (nsp) { ns_pcbdetach(nsp); - sofree(so); soisdisconnected(so); error = 0; } else { error = EINVAL; } + return(error); } @@ -493,8 +502,10 @@ idp_usr_disconnect(struct socket *so) error = ENOTCONN; } else { error = 0; + soreference(so); ns_pcbdisconnect(nsp); soisdisconnected(so); + sofree(so); } } else { error = EINVAL; diff --git a/sys/netproto/ns/ns_pcb.c b/sys/netproto/ns/ns_pcb.c index e7c650bf9d..9128110c3b 100644 --- a/sys/netproto/ns/ns_pcb.c +++ b/sys/netproto/ns/ns_pcb.c @@ -234,7 +234,7 @@ ns_pcbdetach(struct nspcb *nsp) { struct socket *so = nsp->nsp_socket; - so->so_pcb = 0; + so->so_pcb = NULL; sofree(so); if (nsp->nsp_route.ro_rt) rtfree(nsp->nsp_route.ro_rt); diff --git a/sys/netproto/ns/spp_usrreq.c b/sys/netproto/ns/spp_usrreq.c index f768b21fb8..b748a42395 100644 --- a/sys/netproto/ns/spp_usrreq.c +++ b/sys/netproto/ns/spp_usrreq.c @@ -540,7 +540,7 @@ present: if (so->so_rcv.ssb_cc) so->so_oobmark = so->so_rcv.ssb_cc; else - so->so_state |= SS_RCVATMARK; + sosetstate(so, SS_RCVATMARK); } nq = q; q = q->si_prev; @@ -572,7 +572,7 @@ present: m_chtype(m, MT_OOBDATA); spp_newchecks[1]++; so->so_oobmark = 0; - so->so_state &= ~SS_RCVATMARK; + soclrstate(so, SS_RCVATMARK); } if (packetp == 0) { m->m_data += SPINC; @@ -1275,6 +1275,10 @@ spp_ctloutput(int req, struct socket *so, int level, * SPP_USRREQ PROCEDURES */ +/* + * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort() + * will sofree() it when we return. + */ static int spp_usr_abort(struct socket *so) { @@ -1289,6 +1293,7 @@ spp_usr_abort(struct socket *so) } else { error = EINVAL; } + return(error); } @@ -1719,7 +1724,7 @@ spp_close(struct sppcb *cb) } kfree(cb->s_idp, M_IDP); kfree(cb, M_SPPCB); - nsp->nsp_pcb = 0; + nsp->nsp_pcb = NULL; soisdisconnected(so); ns_pcbdetach(nsp); sppstat.spps_closed++; diff --git a/sys/netproto/smb/smb_trantcp.c b/sys/netproto/smb/smb_trantcp.c index 1f8c551a61..25086b9d6e 100644 --- a/sys/netproto/smb/smb_trantcp.c +++ b/sys/netproto/smb/smb_trantcp.c @@ -179,7 +179,7 @@ nb_connect_in(struct nbpcb *nbp, struct sockaddr_in *to, struct thread *td) tsleep(&so->so_timeo, 0, "nbcon", 2 * hz); if ((so->so_state & SS_ISCONNECTING) && so->so_error == 0 && (error = nb_intr(nbp, td)) != 0) { - so->so_state &= ~SS_ISCONNECTING; + soclrstate(so, SS_ISCONNECTING); crit_exit(); goto bad; } diff --git a/sys/sys/socketvar.h b/sys/sys/socketvar.h index 1c996786d0..afe7bb97cd 100644 --- a/sys/sys/socketvar.h +++ b/sys/sys/socketvar.h @@ -47,6 +47,9 @@ #ifndef _SYS_EVENT_H_ #include /* for struct kqinfo */ #endif +#ifndef _SYS_THREAD_H_ +#include /* for struct lwkt_token */ +#endif #ifndef _SYS_SOCKBUF_H_ #include #endif @@ -71,6 +74,7 @@ struct signalsockbuf { long ssb_lowat; /* low water mark */ u_long ssb_hiwat; /* high water mark / max actual char count */ u_long ssb_mbmax; /* max chars of mbufs to use */ + struct lwkt_token ssb_token; /* frontend/backend serializer */ }; #define ssb_cc sb.sb_cc /* commonly used fields */ @@ -139,6 +143,7 @@ struct socket { struct ucred *so_cred; /* user credentials */ /* NB: generation count must not be first; easiest to make it last. */ void *so_emuldata; /* private data for emulators */ + int so_refs; /* shutdown refs */ struct so_accf { struct accept_filter *so_accept_filter; void *so_accept_filter_arg; /* saved filter args */ @@ -150,6 +155,11 @@ struct socket { /* * Socket state bits. + * + * NOTE: The following states are interlocked with so_refs: + * + * SS_NOFDREF so_refs while not set + * (so_pcb) so_refs while set */ #define SS_NOFDREF 0x0001 /* no file table ref any more */ #define SS_ISCONNECTED 0x0002 /* socket connected to a peer */ @@ -159,7 +169,7 @@ struct socket { #define SS_CANTRCVMORE 0x0020 /* can't receive more data from peer */ #define SS_RCVATMARK 0x0040 /* at mark on input */ -#define SS_ABORTING 0x0100 /* so_abort() in progress */ +#define SS_UNUSED0100 0x0100 #define SS_ASYNC 0x0200 /* async i/o notify */ #define SS_ISCONFIRMING 0x0400 /* deciding to accept connection req */ @@ -389,7 +399,6 @@ int soconnect (struct socket *so, struct sockaddr *nam, struct thread *td); int soconnect2 (struct socket *so1, struct socket *so2); int socreate (int dom, struct socket **aso, int type, int proto, struct thread *td); -void sodealloc (struct socket *so); int sodisconnect (struct socket *so); void sofree (struct socket *so); int sogetopt (struct socket *so, struct sockopt *sopt); diff --git a/sys/sys/socketvar2.h b/sys/sys/socketvar2.h index 899a7b2c4e..a944192a42 100644 --- a/sys/sys/socketvar2.h +++ b/sys/sys/socketvar2.h @@ -55,6 +55,9 @@ * Acquire a lock on a signalsockbuf, sleep if the lock is already held. * The sleep is interruptable unless SSB_NOINTR is set in the ssb. * + * We also acquire the token on success. This token is used to interlock + * frontend/backend operations until the sockbuf itself can be made mpsafe. + * * Returns 0 on success, non-zero if the lock could not be acquired. */ static __inline int @@ -70,8 +73,10 @@ ssb_lock(struct signalsockbuf *ssb, int wf) return _ssb_lock(ssb); return EWOULDBLOCK; } - if (atomic_cmpset_int(&ssb->ssb_flags, flags, flags | SSB_LOCK)) + if (atomic_cmpset_int(&ssb->ssb_flags, flags, flags|SSB_LOCK)) { + lwkt_gettoken(&ssb->ssb_token); return(0); + } } } @@ -86,6 +91,7 @@ ssb_unlock(struct signalsockbuf *ssb) uint32_t flags; KKASSERT(ssb->ssb_flags & SSB_LOCK); + lwkt_reltoken(&ssb->ssb_token); for (;;) { flags = ssb->ssb_flags; cpu_ccfence(); @@ -98,4 +104,22 @@ ssb_unlock(struct signalsockbuf *ssb) } } +static __inline void +sosetstate(struct socket *so, short state) +{ + atomic_set_short(&so->so_state, state); +} + +static __inline void +soclrstate(struct socket *so, short state) +{ + atomic_clear_short(&so->so_state, state); +} + +static __inline void +soreference(struct socket *so) +{ + atomic_add_int(&so->so_refs, 1); +} + #endif diff --git a/sys/vfs/fifofs/fifo_vnops.c b/sys/vfs/fifofs/fifo_vnops.c index 7020de53df..abae13003e 100644 --- a/sys/vfs/fifofs/fifo_vnops.c +++ b/sys/vfs/fifofs/fifo_vnops.c @@ -51,6 +51,7 @@ #include #include +#include #include "fifo.h" @@ -193,7 +194,7 @@ fifo_open(struct vop_open_args *ap) } fip->fi_readers = fip->fi_writers = 0; wso->so_snd.ssb_lowat = PIPE_BUF; - rso->so_state |= SS_CANTRCVMORE; + sosetstate(rso, SS_CANTRCVMORE); } if (ap->a_mode & FREAD) { fip->fi_readers++; diff --git a/sys/vfs/nfs/nfs_socket.c b/sys/vfs/nfs/nfs_socket.c index 2dbd9738f4..3d29c18de5 100644 --- a/sys/vfs/nfs/nfs_socket.c +++ b/sys/vfs/nfs/nfs_socket.c @@ -65,6 +65,7 @@ #include #include +#include #include #include @@ -271,7 +272,7 @@ nfs_connect(struct nfsmount *nmp, struct nfsreq *rep) if ((so->so_state & SS_ISCONNECTING) && so->so_error == 0 && rep && (error = nfs_sigintr(nmp, rep, rep->r_td)) != 0){ - so->so_state &= ~SS_ISCONNECTING; + soclrstate(so, SS_ISCONNECTING); crit_exit(); goto bad; } -- 2.41.0