03bded678b60365658d840b908f60bf9c845bf37
[dragonfly.git] / sys / netinet / tcp_usrreq.c
1 /*
2  * Copyright (c) 2003, 2004 Jeffrey M. Hsu.  All rights reserved.
3  * Copyright (c) 2003, 2004 The DragonFly Project.  All rights reserved.
4  *
5  * This code is derived from software contributed to The DragonFly Project
6  * by Jeffrey M. Hsu.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  * 3. Neither the name of The DragonFly Project nor the names of its
17  *    contributors may be used to endorse or promote products derived
18  *    from this software without specific, prior written permission.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
24  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
26  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
28  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
29  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
30  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31  * SUCH DAMAGE.
32  */
33
34 /*
35  * Copyright (c) 1982, 1986, 1988, 1993
36  *      The Regents of the University of California.  All rights reserved.
37  *
38  * Redistribution and use in source and binary forms, with or without
39  * modification, are permitted provided that the following conditions
40  * are met:
41  * 1. Redistributions of source code must retain the above copyright
42  *    notice, this list of conditions and the following disclaimer.
43  * 2. Redistributions in binary form must reproduce the above copyright
44  *    notice, this list of conditions and the following disclaimer in the
45  *    documentation and/or other materials provided with the distribution.
46  * 3. Neither the name of the University nor the names of its contributors
47  *    may be used to endorse or promote products derived from this software
48  *    without specific prior written permission.
49  *
50  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
51  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
52  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
53  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
54  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
55  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
56  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
57  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
58  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
59  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
60  * SUCH DAMAGE.
61  *
62  *      From: @(#)tcp_usrreq.c  8.2 (Berkeley) 1/3/94
63  * $FreeBSD: src/sys/netinet/tcp_usrreq.c,v 1.51.2.17 2002/10/11 11:46:44 ume Exp $
64  */
65
66 #include "opt_inet.h"
67 #include "opt_inet6.h"
68 #include "opt_tcpdebug.h"
69
70 #include <sys/param.h>
71 #include <sys/systm.h>
72 #include <sys/kernel.h>
73 #include <sys/malloc.h>
74 #include <sys/sysctl.h>
75 #include <sys/globaldata.h>
76 #include <sys/thread.h>
77
78 #include <sys/mbuf.h>
79 #ifdef INET6
80 #include <sys/domain.h>
81 #endif /* INET6 */
82 #include <sys/socket.h>
83 #include <sys/socketvar.h>
84 #include <sys/socketops.h>
85 #include <sys/protosw.h>
86 #include <sys/jail.h>
87
88 #include <sys/msgport2.h>
89 #include <sys/socketvar2.h>
90
91 #include <net/if.h>
92 #include <net/netisr.h>
93 #include <net/route.h>
94
95 #include <net/netmsg2.h>
96 #include <net/netisr2.h>
97
98 #include <netinet/in.h>
99 #include <netinet/in_systm.h>
100 #ifdef INET6
101 #include <netinet/ip6.h>
102 #endif
103 #include <netinet/in_pcb.h>
104 #ifdef INET6
105 #include <netinet6/in6_pcb.h>
106 #endif
107 #include <netinet/in_var.h>
108 #include <netinet/ip_var.h>
109 #ifdef INET6
110 #include <netinet6/ip6_var.h>
111 #include <netinet6/tcp6_var.h>
112 #endif
113 #include <netinet/tcp.h>
114 #include <netinet/tcp_fsm.h>
115 #include <netinet/tcp_seq.h>
116 #include <netinet/tcp_timer.h>
117 #include <netinet/tcp_timer2.h>
118 #include <netinet/tcp_var.h>
119 #include <netinet/tcpip.h>
120 #ifdef TCPDEBUG
121 #include <netinet/tcp_debug.h>
122 #endif
123 #include <machine/limits.h>
124
125 /*
126  * Instead of erroring out, just cap values such that the t_maxidle doesn't
127  * overflow.
128  */
129 #define MAXKEEP         (INT_MAX / 2)
130
131 /*
132  * TCP protocol interface to socket abstraction.
133  */
134 extern  char *tcpstates[];      /* XXX ??? */
135
136 static int      tcp_attach (struct socket *, struct pru_attach_info *);
137 static void     tcp_connect (netmsg_t msg);
138 #ifdef INET6
139 static void     tcp6_connect (netmsg_t msg);
140 static int      tcp6_connect_oncpu(struct tcpcb *tp, int flags,
141                                 struct mbuf **mp,
142                                 struct sockaddr_in6 *sin6,
143                                 struct in6_addr *addr6);
144 #endif /* INET6 */
145 static struct tcpcb *
146                 tcp_disconnect (struct tcpcb *);
147 static struct tcpcb *
148                 tcp_usrclosed (struct tcpcb *);
149
150 #ifdef TCPDEBUG
151 #define TCPDEBUG0       int ostate = 0
152 #define TCPDEBUG1()     ostate = tp ? tp->t_state : 0
153 #define TCPDEBUG2(req)  if (tp && (so->so_options & SO_DEBUG)) \
154                                 tcp_trace(TA_USER, ostate, tp, 0, 0, req)
155 #else
156 #define TCPDEBUG0
157 #define TCPDEBUG1()
158 #define TCPDEBUG2(req)
159 #endif
160
161 /*
162  * For some ill optimized programs, which try to use TCP_NOPUSH
163  * to improve performance, will have small amount of data sits
164  * in the sending buffer.  These small amount of data will _not_
165  * be pushed into the network until more data are written into
166  * the socket or the socket write side is shutdown.
167  */ 
168 static int      tcp_disable_nopush = 1;
169 SYSCTL_INT(_net_inet_tcp, OID_AUTO, disable_nopush, CTLFLAG_RW,
170     &tcp_disable_nopush, 0, "TCP_NOPUSH socket option will have no effect");
171
172 /*
173  * Allocate socket buffer space.
174  */
175 static int
176 tcp_usr_preattach(struct socket *so, int proto __unused,
177     struct pru_attach_info *ai)
178 {
179         int error;
180
181         if (so->so_snd.ssb_hiwat == 0 || so->so_rcv.ssb_hiwat == 0) {
182                 error = soreserve(so, tcp_sendspace, tcp_recvspace,
183                                   ai->sb_rlimit);
184                 if (error)
185                         return (error);
186         }
187         atomic_set_int(&so->so_rcv.ssb_flags, SSB_AUTOSIZE);
188         atomic_set_int(&so->so_snd.ssb_flags, SSB_AUTOSIZE | SSB_PREALLOC);
189
190         return 0;
191 }
192
193 /*
194  * TCP attaches to socket via pru_attach(), reserving space,
195  * and an internet control block.  This socket may move to
196  * other CPU later when we bind/connect.
197  */
198 static void
199 tcp_usr_attach(netmsg_t msg)
200 {
201         struct socket *so = msg->base.nm_so;
202         struct pru_attach_info *ai = msg->attach.nm_ai;
203         int error;
204         struct inpcb *inp;
205         struct tcpcb *tp = NULL;
206         TCPDEBUG0;
207
208         inp = so->so_pcb;
209         KASSERT(inp == NULL, ("tcp socket attached"));
210         TCPDEBUG1();
211
212         error = tcp_attach(so, ai);
213         if (error)
214                 goto out;
215
216         if ((so->so_options & SO_LINGER) && so->so_linger == 0)
217                 so->so_linger = TCP_LINGERTIME;
218         tp = sototcpcb(so);
219 out:
220         TCPDEBUG2(PRU_ATTACH);
221         lwkt_replymsg(&msg->lmsg, error);
222 }
223
224 /*
225  * pru_detach() detaches the TCP protocol from the socket.
226  * If the protocol state is non-embryonic, then can't
227  * do this directly: have to initiate a pru_disconnect(),
228  * which may finish later; embryonic TCB's can just
229  * be discarded here.
230  */
231 static void
232 tcp_usr_detach(netmsg_t msg)
233 {
234         struct socket *so = msg->base.nm_so;
235         int error = 0;
236         struct inpcb *inp;
237         struct tcpcb *tp;
238         TCPDEBUG0;
239
240         inp = so->so_pcb;
241
242         /*
243          * If the inp is already detached or never attached, it may have
244          * been due to an async close or async attach failure.  Just return
245          * as if no error occured.
246          */
247         if (inp) {
248                 tp = intotcpcb(inp);
249                 KASSERT(tp != NULL, ("tcp_usr_detach: tp is NULL"));
250                 TCPDEBUG1();
251                 tp = tcp_disconnect(tp);
252                 TCPDEBUG2(PRU_DETACH);
253         }
254         lwkt_replymsg(&msg->lmsg, error);
255 }
256
257 /*
258  * NOTE: ignore_error is non-zero for certain disconnection races
259  * which we want to silently allow, otherwise close() may return
260  * an unexpected error.
261  *
262  * NOTE: The variables (msg) and (tp) are assumed.
263  */
264 #define COMMON_START(so, inp, ignore_error)                     \
265         TCPDEBUG0;                                              \
266                                                                 \
267         inp = so->so_pcb;                                       \
268         do {                                                    \
269                 if (inp == NULL) {                              \
270                         error = ignore_error ? 0 : EINVAL;      \
271                         tp = NULL;                              \
272                         goto out;                               \
273                 }                                               \
274                 tp = intotcpcb(inp);                            \
275                 TCPDEBUG1();                                    \
276         } while(0)
277
278 #define COMMON_END1(req, noreply)                               \
279         out: do {                                               \
280                 TCPDEBUG2(req);                                 \
281                 if (!(noreply))                                 \
282                         lwkt_replymsg(&msg->lmsg, error);       \
283                 return;                                         \
284         } while(0)
285
286 #define COMMON_END(req)         COMMON_END1((req), 0)
287
288 static void
289 tcp_sosetport(struct lwkt_msg *msg, lwkt_port_t port)
290 {
291         sosetport(((struct netmsg_base *)msg)->nm_so, port);
292 }
293
294 /*
295  * Give the socket an address.
296  */
297 static void
298 tcp_usr_bind(netmsg_t msg)
299 {
300         struct socket *so = msg->bind.base.nm_so;
301         struct sockaddr *nam = msg->bind.nm_nam;
302         struct thread *td = msg->bind.nm_td;
303         int error = 0;
304         struct inpcb *inp;
305         struct tcpcb *tp;
306         struct sockaddr_in *sinp;
307         lwkt_port_t port0 = netisr_cpuport(0);
308
309         COMMON_START(so, inp, 0);
310
311         /*
312          * Must check for multicast addresses and disallow binding
313          * to them.
314          */
315         sinp = (struct sockaddr_in *)nam;
316         if (sinp->sin_family == AF_INET &&
317             IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
318                 error = EAFNOSUPPORT;
319                 goto out;
320         }
321
322         /*
323          * Check "already bound" here (in_pcbbind() does the same check
324          * though), so we don't forward a connected socket to netisr0,
325          * which would panic in the following in_pcbunlink().
326          */
327         if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY) {
328                 error = EINVAL; /* already bound */
329                 goto out;
330         }
331
332         /*
333          * Use netisr0 to serialize in_pcbbind(), so that pru_detach and
334          * pru_bind for different sockets on the same local port could be
335          * properly ordered.  The original race is illustrated here for
336          * reference.
337          *
338          * s1 = socket();
339          * bind(s1, *.PORT);
340          * close(s1);  <----- asynchronous
341          * s2 = socket();
342          * bind(s2, *.PORT);
343          *
344          * All will expect bind(s2, *.PORT) to succeed.  However, it will
345          * fail, if following sequence happens due to random socket initial
346          * msgport and asynchronous close(2):
347          *
348          *    netisrN                  netisrM
349          *       :                        :
350          *       :                    pru_bind(s2) [*.PORT is used by s1]
351          *  pru_detach(s1)                :
352          */
353         if (&curthread->td_msgport != port0) {
354                 lwkt_msg_t lmsg = &msg->bind.base.lmsg;
355
356                 KASSERT((msg->bind.nm_flags & PRUB_RELINK) == 0,
357                     ("already asked to relink"));
358
359                 in_pcbunlink(so->so_pcb, &tcbinfo[mycpuid]);
360                 msg->bind.nm_flags |= PRUB_RELINK;
361
362                 TCP_STATE_MIGRATE_START(tp);
363
364                 /* See the related comment in tcp_connect() */
365                 lwkt_setmsg_receipt(lmsg, tcp_sosetport);
366                 lwkt_forwardmsg(port0, lmsg);
367                 /* msg invalid now */
368                 return;
369         }
370         KASSERT(so->so_port == port0, ("so_port is not netisr0"));
371
372         if (msg->bind.nm_flags & PRUB_RELINK) {
373                 msg->bind.nm_flags &= ~PRUB_RELINK;
374                 TCP_STATE_MIGRATE_END(tp);
375                 in_pcblink(so->so_pcb, &tcbinfo[mycpuid]);
376         }
377         KASSERT(inp->inp_pcbinfo == &tcbinfo[0], ("pcbinfo is not tcbinfo0"));
378
379         error = in_pcbbind(inp, nam, td);
380         if (error)
381                 goto out;
382
383         COMMON_END(PRU_BIND);
384 }
385
386 #ifdef INET6
387
388 static void
389 tcp6_usr_bind(netmsg_t msg)
390 {
391         struct socket *so = msg->bind.base.nm_so;
392         struct sockaddr *nam = msg->bind.nm_nam;
393         struct thread *td = msg->bind.nm_td;
394         int error = 0;
395         struct inpcb *inp;
396         struct tcpcb *tp;
397         struct sockaddr_in6 *sin6p;
398
399         COMMON_START(so, inp, 0);
400
401         /*
402          * Must check for multicast addresses and disallow binding
403          * to them.
404          */
405         sin6p = (struct sockaddr_in6 *)nam;
406         if (sin6p->sin6_family == AF_INET6 &&
407             IN6_IS_ADDR_MULTICAST(&sin6p->sin6_addr)) {
408                 error = EAFNOSUPPORT;
409                 goto out;
410         }
411         error = in6_pcbbind(inp, nam, td);
412         if (error)
413                 goto out;
414         COMMON_END(PRU_BIND);
415 }
416 #endif /* INET6 */
417
418 struct netmsg_inswildcard {
419         struct netmsg_base      base;
420         struct inpcb            *nm_inp;
421 };
422
423 static void
424 in_pcbinswildcardhash_handler(netmsg_t msg)
425 {
426         struct netmsg_inswildcard *nm = (struct netmsg_inswildcard *)msg;
427         int cpu = mycpuid, nextcpu;
428
429         in_pcbinswildcardhash_oncpu(nm->nm_inp, &tcbinfo[cpu]);
430
431         nextcpu = cpu + 1;
432         if (nextcpu < netisr_ncpus)
433                 lwkt_forwardmsg(netisr_cpuport(nextcpu), &nm->base.lmsg);
434         else
435                 lwkt_replymsg(&nm->base.lmsg, 0);
436 }
437
438 /*
439  * Prepare to accept connections.
440  */
441 static void
442 tcp_usr_listen(netmsg_t msg)
443 {
444         struct socket *so = msg->listen.base.nm_so;
445         struct thread *td = msg->listen.nm_td;
446         int error = 0;
447         struct inpcb *inp;
448         struct tcpcb *tp;
449         struct netmsg_inswildcard nm;
450         lwkt_port_t port0 = netisr_cpuport(0);
451
452         COMMON_START(so, inp, 0);
453
454         if (&curthread->td_msgport != port0) {
455                 lwkt_msg_t lmsg = &msg->listen.base.lmsg;
456
457                 KASSERT((msg->listen.nm_flags & PRUL_RELINK) == 0,
458                     ("already asked to relink"));
459
460                 in_pcbunlink(so->so_pcb, &tcbinfo[mycpuid]);
461                 msg->listen.nm_flags |= PRUL_RELINK;
462
463                 TCP_STATE_MIGRATE_START(tp);
464
465                 /* See the related comment in tcp_connect() */
466                 lwkt_setmsg_receipt(lmsg, tcp_sosetport);
467                 lwkt_forwardmsg(port0, lmsg);
468                 /* msg invalid now */
469                 return;
470         }
471         KASSERT(so->so_port == port0, ("so_port is not netisr0"));
472
473         if (msg->listen.nm_flags & PRUL_RELINK) {
474                 msg->listen.nm_flags &= ~PRUL_RELINK;
475                 TCP_STATE_MIGRATE_END(tp);
476                 in_pcblink(so->so_pcb, &tcbinfo[mycpuid]);
477         }
478         KASSERT(inp->inp_pcbinfo == &tcbinfo[0], ("pcbinfo is not tcbinfo0"));
479
480         if (tp->t_flags & TF_LISTEN)
481                 goto out;
482
483         if (inp->inp_lport == 0) {
484                 error = in_pcbbind(inp, NULL, td);
485                 if (error)
486                         goto out;
487         }
488
489         TCP_STATE_CHANGE(tp, TCPS_LISTEN);
490         tp->t_flags |= TF_LISTEN;
491         tp->tt_msg = NULL; /* Catch any invalid timer usage */
492
493         /*
494          * Create tcpcb per-cpu port cache
495          *
496          * NOTE:
497          * This _must_ be done before installing this inpcb into
498          * wildcard hash.
499          */
500         tcp_pcbport_create(tp);
501
502         if (netisr_ncpus > 1) {
503                 /*
504                  * Put this inpcb into wildcard hash on other cpus.
505                  */
506                 ASSERT_INP_NOTINHASH(inp);
507                 netmsg_init(&nm.base, NULL, &curthread->td_msgport,
508                             MSGF_PRIORITY, in_pcbinswildcardhash_handler);
509                 nm.nm_inp = inp;
510                 lwkt_domsg(netisr_cpuport(1), &nm.base.lmsg, 0);
511         }
512         in_pcbinswildcardhash(inp);
513         COMMON_END(PRU_LISTEN);
514 }
515
516 #ifdef INET6
517
518 static void
519 tcp6_usr_listen(netmsg_t msg)
520 {
521         struct socket *so = msg->listen.base.nm_so;
522         struct thread *td = msg->listen.nm_td;
523         int error = 0;
524         struct inpcb *inp;
525         struct tcpcb *tp;
526         struct netmsg_inswildcard nm;
527
528         COMMON_START(so, inp, 0);
529
530         if (tp->t_flags & TF_LISTEN)
531                 goto out;
532
533         if (inp->inp_lport == 0) {
534                 error = in6_pcbbind(inp, NULL, td);
535                 if (error)
536                         goto out;
537         }
538
539         TCP_STATE_CHANGE(tp, TCPS_LISTEN);
540         tp->t_flags |= TF_LISTEN;
541         tp->tt_msg = NULL; /* Catch any invalid timer usage */
542
543         /*
544          * Create tcpcb per-cpu port cache
545          *
546          * NOTE:
547          * This _must_ be done before installing this inpcb into
548          * wildcard hash.
549          */
550         tcp_pcbport_create(tp);
551
552         if (netisr_ncpus > 1) {
553                 /*
554                  * Put this inpcb into wildcard hash on other cpus.
555                  */
556                 KKASSERT(so->so_port == netisr_cpuport(0));
557                 ASSERT_NETISR0;
558                 KKASSERT(inp->inp_pcbinfo == &tcbinfo[0]);
559                 ASSERT_INP_NOTINHASH(inp);
560
561                 netmsg_init(&nm.base, NULL, &curthread->td_msgport,
562                             MSGF_PRIORITY, in_pcbinswildcardhash_handler);
563                 nm.nm_inp = inp;
564                 lwkt_domsg(netisr_cpuport(1), &nm.base.lmsg, 0);
565         }
566         in_pcbinswildcardhash(inp);
567         COMMON_END(PRU_LISTEN);
568 }
569 #endif /* INET6 */
570
571 /*
572  * Initiate connection to peer.
573  * Create a template for use in transmissions on this connection.
574  * Enter SYN_SENT state, and mark socket as connecting.
575  * Start keep-alive timer, and seed output sequence space.
576  * Send initial segment on connection.
577  */
578 static void
579 tcp_usr_connect(netmsg_t msg)
580 {
581         struct socket *so = msg->connect.base.nm_so;
582         struct sockaddr *nam = msg->connect.nm_nam;
583         struct thread *td = msg->connect.nm_td;
584         int error = 0;
585         struct inpcb *inp;
586         struct tcpcb *tp;
587         struct sockaddr_in *sinp;
588
589         ASSERT_NETISR_NCPUS(mycpuid);
590
591         COMMON_START(so, inp, 0);
592
593         /*
594          * Must disallow TCP ``connections'' to multicast addresses.
595          */
596         sinp = (struct sockaddr_in *)nam;
597         if (sinp->sin_family == AF_INET
598             && IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
599                 error = EAFNOSUPPORT;
600                 goto out;
601         }
602
603         if (!prison_remote_ip(td, (struct sockaddr*)sinp)) {
604                 error = EAFNOSUPPORT; /* IPv6 only jail */
605                 goto out;
606         }
607
608         tcp_connect(msg);
609         /* msg is invalid now */
610         return;
611 out:
612         if (msg->connect.nm_m) {
613                 m_freem(msg->connect.nm_m);
614                 msg->connect.nm_m = NULL;
615         }
616         if (msg->connect.nm_flags & PRUC_HELDTD)
617                 lwkt_rele(td);
618         if (error && (msg->connect.nm_flags & PRUC_ASYNC)) {
619                 so->so_error = error;
620                 soisdisconnected(so);
621         }
622         lwkt_replymsg(&msg->lmsg, error);
623 }
624
625 #ifdef INET6
626
627 static void
628 tcp6_usr_connect(netmsg_t msg)
629 {
630         struct socket *so = msg->connect.base.nm_so;
631         struct sockaddr *nam = msg->connect.nm_nam;
632         struct thread *td = msg->connect.nm_td;
633         int error = 0;
634         struct inpcb *inp;
635         struct tcpcb *tp;
636         struct sockaddr_in6 *sin6p;
637
638         ASSERT_NETISR_NCPUS(mycpuid);
639
640         COMMON_START(so, inp, 0);
641
642         /*
643          * Must disallow TCP ``connections'' to multicast addresses.
644          */
645         sin6p = (struct sockaddr_in6 *)nam;
646         if (sin6p->sin6_family == AF_INET6
647             && IN6_IS_ADDR_MULTICAST(&sin6p->sin6_addr)) {
648                 error = EAFNOSUPPORT;
649                 goto out;
650         }
651
652         if (!prison_remote_ip(td, nam)) {
653                 error = EAFNOSUPPORT; /* IPv4 only jail */
654                 goto out;
655         }
656
657         /* Reject v4-mapped address */
658         if (IN6_IS_ADDR_V4MAPPED(&sin6p->sin6_addr)) {
659                 error = EADDRNOTAVAIL;
660                 goto out;
661         }
662
663         inp->inp_inc.inc_isipv6 = 1;
664         tcp6_connect(msg);
665         /* msg is invalid now */
666         return;
667 out:
668         if (msg->connect.nm_m) {
669                 m_freem(msg->connect.nm_m);
670                 msg->connect.nm_m = NULL;
671         }
672         lwkt_replymsg(&msg->lmsg, error);
673 }
674
675 #endif /* INET6 */
676
677 /*
678  * Initiate disconnect from peer.
679  * If connection never passed embryonic stage, just drop;
680  * else if don't need to let data drain, then can just drop anyways,
681  * else have to begin TCP shutdown process: mark socket disconnecting,
682  * drain unread data, state switch to reflect user close, and
683  * send segment (e.g. FIN) to peer.  Socket will be really disconnected
684  * when peer sends FIN and acks ours.
685  *
686  * SHOULD IMPLEMENT LATER PRU_CONNECT VIA REALLOC TCPCB.
687  */
688 static void
689 tcp_usr_disconnect(netmsg_t msg)
690 {
691         struct socket *so = msg->disconnect.base.nm_so;
692         int error = 0;
693         struct inpcb *inp;
694         struct tcpcb *tp;
695
696         COMMON_START(so, inp, 1);
697         tp = tcp_disconnect(tp);
698         COMMON_END(PRU_DISCONNECT);
699 }
700
701 /*
702  * Accept a connection.  Essentially all the work is
703  * done at higher levels; just return the address
704  * of the peer, storing through addr.
705  */
706 static void
707 tcp_usr_accept(netmsg_t msg)
708 {
709         struct socket *so = msg->accept.base.nm_so;
710         struct sockaddr **nam = msg->accept.nm_nam;
711         int error = 0;
712         struct inpcb *inp;
713         struct tcpcb *tp = NULL;
714         TCPDEBUG0;
715
716         inp = so->so_pcb;
717         if (so->so_state & SS_ISDISCONNECTED) {
718                 error = ECONNABORTED;
719                 goto out;
720         }
721         if (inp == NULL) {
722                 error = EINVAL;
723                 goto out;
724         }
725
726         tp = intotcpcb(inp);
727         TCPDEBUG1();
728         in_setpeeraddr(so, nam);
729         COMMON_END(PRU_ACCEPT);
730 }
731
732 #ifdef INET6
733 static void
734 tcp6_usr_accept(netmsg_t msg)
735 {
736         struct socket *so = msg->accept.base.nm_so;
737         struct sockaddr **nam = msg->accept.nm_nam;
738         int error = 0;
739         struct inpcb *inp;
740         struct tcpcb *tp = NULL;
741         TCPDEBUG0;
742
743         inp = so->so_pcb;
744
745         if (so->so_state & SS_ISDISCONNECTED) {
746                 error = ECONNABORTED;
747                 goto out;
748         }
749         if (inp == NULL) {
750                 error = EINVAL;
751                 goto out;
752         }
753         tp = intotcpcb(inp);
754         TCPDEBUG1();
755         in6_setpeeraddr(so, nam);
756         COMMON_END(PRU_ACCEPT);
757 }
758 #endif /* INET6 */
759
760 /*
761  * Mark the connection as being incapable of further output.
762  */
763 static void
764 tcp_usr_shutdown(netmsg_t msg)
765 {
766         struct socket *so = msg->shutdown.base.nm_so;
767         int error = 0;
768         struct inpcb *inp;
769         struct tcpcb *tp;
770
771         COMMON_START(so, inp, 0);
772         socantsendmore(so);
773         tp = tcp_usrclosed(tp);
774         if (tp)
775                 error = tcp_output(tp);
776         COMMON_END(PRU_SHUTDOWN);
777 }
778
779 /*
780  * After a receive, possibly send window update to peer.
781  */
782 static void
783 tcp_usr_rcvd(netmsg_t msg)
784 {
785         struct socket *so = msg->rcvd.base.nm_so;
786         int error = 0, noreply = 0;
787         struct inpcb *inp;
788         struct tcpcb *tp;
789
790         COMMON_START(so, inp, 0);
791
792         if (msg->rcvd.nm_pru_flags & PRUR_ASYNC) {
793                 noreply = 1;
794                 so_async_rcvd_reply(so);
795         }
796         tcp_output(tp);
797
798         COMMON_END1(PRU_RCVD, noreply);
799 }
800
801 /*
802  * Do a send by putting data in output queue and updating urgent
803  * marker if URG set.  Possibly send more data.  Unlike the other
804  * pru_*() routines, the mbuf chains are our responsibility.  We
805  * must either enqueue them or free them.  The other pru_* routines
806  * generally are caller-frees.
807  */
808 static void
809 tcp_usr_send(netmsg_t msg)
810 {
811         struct socket *so = msg->send.base.nm_so;
812         int flags = msg->send.nm_flags;
813         struct mbuf *m = msg->send.nm_m;
814         int error = 0;
815         struct inpcb *inp;
816         struct tcpcb *tp;
817         TCPDEBUG0;
818
819         KKASSERT(msg->send.nm_control == NULL);
820         KKASSERT(msg->send.nm_addr == NULL);
821         KKASSERT((flags & PRUS_FREEADDR) == 0);
822
823         inp = so->so_pcb;
824
825         if (inp == NULL) {
826                 /*
827                  * OOPS! we lost a race, the TCP session got reset after
828                  * we checked SS_CANTSENDMORE, eg: while doing uiomove or a
829                  * network interrupt in the non-critical section of sosend().
830                  */
831                 m_freem(m);
832                 error = ECONNRESET;     /* XXX EPIPE? */
833                 tp = NULL;
834                 TCPDEBUG1();
835                 goto out;
836         }
837         tp = intotcpcb(inp);
838         TCPDEBUG1();
839
840 #ifdef foo
841         /*
842          * This is no longer necessary, since:
843          * - sosendtcp() has already checked it for us
844          * - It does not work with asynchronized send
845          */
846
847         /*
848          * Don't let too much OOB data build up
849          */
850         if (flags & PRUS_OOB) {
851                 if (ssb_space(&so->so_snd) < -512) {
852                         m_freem(m);
853                         error = ENOBUFS;
854                         goto out;
855                 }
856         }
857 #endif
858
859         /*
860          * Pump the data into the socket.
861          */
862         if (m) {
863                 ssb_appendstream(&so->so_snd, m);
864                 sowwakeup(so);
865         }
866         if (flags & PRUS_OOB) {
867                 /*
868                  * According to RFC961 (Assigned Protocols),
869                  * the urgent pointer points to the last octet
870                  * of urgent data.  We continue, however,
871                  * to consider it to indicate the first octet
872                  * of data past the urgent section.
873                  * Otherwise, snd_up should be one lower.
874                  */
875                 tp->snd_up = tp->snd_una + so->so_snd.ssb_cc;
876                 tp->t_flags |= TF_FORCE;
877                 error = tcp_output(tp);
878                 tp->t_flags &= ~TF_FORCE;
879         } else {
880                 if (flags & PRUS_EOF) {
881                         /*
882                          * Close the send side of the connection after
883                          * the data is sent.
884                          */
885                         socantsendmore(so);
886                         tp = tcp_usrclosed(tp);
887                 }
888                 if (tp != NULL && !tcp_output_pending(tp)) {
889                         if (flags & PRUS_MORETOCOME)
890                                 tp->t_flags |= TF_MORETOCOME;
891                         error = tcp_output_fair(tp);
892                         if (flags & PRUS_MORETOCOME)
893                                 tp->t_flags &= ~TF_MORETOCOME;
894                 }
895         }
896         COMMON_END1((flags & PRUS_OOB) ? PRU_SENDOOB :
897                    ((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND),
898                    (flags & PRUS_NOREPLY));
899 }
900
901 /*
902  * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort()
903  *       will sofree() it when we return.
904  */
905 static void
906 tcp_usr_abort(netmsg_t msg)
907 {
908         struct socket *so = msg->abort.base.nm_so;
909         int error = 0;
910         struct inpcb *inp;
911         struct tcpcb *tp;
912
913         COMMON_START(so, inp, 1);
914         tp = tcp_drop(tp, ECONNABORTED);
915         COMMON_END(PRU_ABORT);
916 }
917
918 /*
919  * Receive out-of-band data.
920  */
921 static void
922 tcp_usr_rcvoob(netmsg_t msg)
923 {
924         struct socket *so = msg->rcvoob.base.nm_so;
925         struct mbuf *m = msg->rcvoob.nm_m;
926         int flags = msg->rcvoob.nm_flags;
927         int error = 0;
928         struct inpcb *inp;
929         struct tcpcb *tp;
930
931         COMMON_START(so, inp, 0);
932         if ((so->so_oobmark == 0 &&
933              (so->so_state & SS_RCVATMARK) == 0) ||
934             so->so_options & SO_OOBINLINE ||
935             tp->t_oobflags & TCPOOB_HADDATA) {
936                 error = EINVAL;
937                 goto out;
938         }
939         if ((tp->t_oobflags & TCPOOB_HAVEDATA) == 0) {
940                 error = EWOULDBLOCK;
941                 goto out;
942         }
943         m->m_len = 1;
944         *mtod(m, caddr_t) = tp->t_iobc;
945         if ((flags & MSG_PEEK) == 0)
946                 tp->t_oobflags ^= (TCPOOB_HAVEDATA | TCPOOB_HADDATA);
947         COMMON_END(PRU_RCVOOB);
948 }
949
950 static void
951 tcp_usr_savefaddr(struct socket *so, const struct sockaddr *faddr)
952 {
953         in_savefaddr(so, faddr);
954 }
955
956 #ifdef INET6
957 static void
958 tcp6_usr_savefaddr(struct socket *so, const struct sockaddr *faddr)
959 {
960         in6_savefaddr(so, faddr);
961 }
962 #endif
963
964 static int
965 tcp_usr_preconnect(struct socket *so, const struct sockaddr *nam,
966     struct thread *td __unused)
967 {
968         const struct sockaddr_in *sinp;
969
970         sinp = (const struct sockaddr_in *)nam;
971         if (sinp->sin_family == AF_INET &&
972             IN_MULTICAST(ntohl(sinp->sin_addr.s_addr)))
973                 return EAFNOSUPPORT;
974
975         soisconnecting(so);
976         return 0;
977 }
978
979 /* xxx - should be const */
980 struct pr_usrreqs tcp_usrreqs = {
981         .pru_abort = tcp_usr_abort,
982         .pru_accept = tcp_usr_accept,
983         .pru_attach = tcp_usr_attach,
984         .pru_bind = tcp_usr_bind,
985         .pru_connect = tcp_usr_connect,
986         .pru_connect2 = pr_generic_notsupp,
987         .pru_control = in_control_dispatch,
988         .pru_detach = tcp_usr_detach,
989         .pru_disconnect = tcp_usr_disconnect,
990         .pru_listen = tcp_usr_listen,
991         .pru_peeraddr = in_setpeeraddr_dispatch,
992         .pru_rcvd = tcp_usr_rcvd,
993         .pru_rcvoob = tcp_usr_rcvoob,
994         .pru_send = tcp_usr_send,
995         .pru_sense = pru_sense_null,
996         .pru_shutdown = tcp_usr_shutdown,
997         .pru_sockaddr = in_setsockaddr_dispatch,
998         .pru_sosend = sosendtcp,
999         .pru_soreceive = sorecvtcp,
1000         .pru_savefaddr = tcp_usr_savefaddr,
1001         .pru_preconnect = tcp_usr_preconnect,
1002         .pru_preattach = tcp_usr_preattach
1003 };
1004
1005 #ifdef INET6
1006 struct pr_usrreqs tcp6_usrreqs = {
1007         .pru_abort = tcp_usr_abort,
1008         .pru_accept = tcp6_usr_accept,
1009         .pru_attach = tcp_usr_attach,
1010         .pru_bind = tcp6_usr_bind,
1011         .pru_connect = tcp6_usr_connect,
1012         .pru_connect2 = pr_generic_notsupp,
1013         .pru_control = in6_control_dispatch,
1014         .pru_detach = tcp_usr_detach,
1015         .pru_disconnect = tcp_usr_disconnect,
1016         .pru_listen = tcp6_usr_listen,
1017         .pru_peeraddr = in6_setpeeraddr_dispatch,
1018         .pru_rcvd = tcp_usr_rcvd,
1019         .pru_rcvoob = tcp_usr_rcvoob,
1020         .pru_send = tcp_usr_send,
1021         .pru_sense = pru_sense_null,
1022         .pru_shutdown = tcp_usr_shutdown,
1023         .pru_sockaddr = in6_setsockaddr_dispatch,
1024         .pru_sosend = sosendtcp,
1025         .pru_soreceive = sorecvtcp,
1026         .pru_savefaddr = tcp6_usr_savefaddr
1027 };
1028 #endif /* INET6 */
1029
1030 static int
1031 tcp_connect_oncpu(struct tcpcb *tp, int flags, struct mbuf *m,
1032                   const struct sockaddr_in *sin, struct sockaddr_in *if_sin,
1033                   uint16_t hash)
1034 {
1035         struct inpcb *inp = tp->t_inpcb, *oinp;
1036         struct socket *so = inp->inp_socket;
1037         struct route *ro = &inp->inp_route;
1038
1039         KASSERT(inp->inp_pcbinfo == &tcbinfo[mycpu->gd_cpuid],
1040             ("pcbinfo mismatch"));
1041
1042         oinp = in_pcblookup_hash(inp->inp_pcbinfo,
1043                                  sin->sin_addr, sin->sin_port,
1044                                  (inp->inp_laddr.s_addr != INADDR_ANY ?
1045                                   inp->inp_laddr : if_sin->sin_addr),
1046                                 inp->inp_lport, 0, NULL);
1047         if (oinp != NULL) {
1048                 m_freem(m);
1049                 return (EADDRINUSE);
1050         }
1051         if (inp->inp_laddr.s_addr == INADDR_ANY)
1052                 inp->inp_laddr = if_sin->sin_addr;
1053         KASSERT(inp->inp_faddr.s_addr == sin->sin_addr.s_addr,
1054             ("faddr mismatch for reconnect"));
1055         KASSERT(inp->inp_fport == sin->sin_port,
1056             ("fport mismatch for reconnect"));
1057         in_pcbinsconnhash(inp);
1058
1059         inp->inp_flags |= INP_HASH;
1060         inp->inp_hashval = hash;
1061
1062         /*
1063          * We are now on the inpcb's owner CPU, if the cached route was
1064          * freed because the rtentry's owner CPU is not the current CPU
1065          * (e.g. in tcp_connect()), then we try to reallocate it here with
1066          * the hope that a rtentry may be cloned from a RTF_PRCLONING
1067          * rtentry.
1068          */
1069         if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/
1070             ro->ro_rt == NULL) {
1071                 bzero(&ro->ro_dst, sizeof(struct sockaddr_in));
1072                 ro->ro_dst.sa_family = AF_INET;
1073                 ro->ro_dst.sa_len = sizeof(struct sockaddr_in);
1074                 ((struct sockaddr_in *)&ro->ro_dst)->sin_addr =
1075                         sin->sin_addr;
1076                 rtalloc(ro);
1077         }
1078
1079         /*
1080          * Now that no more errors can occur, change the protocol processing
1081          * port to the current thread (which is the correct thread).
1082          *
1083          * Create TCP timer message now; we are on the tcpcb's owner
1084          * CPU/thread.
1085          */
1086         tcp_create_timermsg(tp, &curthread->td_msgport);
1087
1088         /*
1089          * Compute window scaling to request.  Use a larger scaling then
1090          * needed for the initial receive buffer in case the receive buffer
1091          * gets expanded.
1092          */
1093         if (tp->request_r_scale < TCP_MIN_WINSHIFT)
1094                 tp->request_r_scale = TCP_MIN_WINSHIFT;
1095         while (tp->request_r_scale < TCP_MAX_WINSHIFT &&
1096                (TCP_MAXWIN << tp->request_r_scale) < so->so_rcv.ssb_hiwat
1097         ) {
1098                 tp->request_r_scale++;
1099         }
1100
1101         soisconnecting(so);
1102         tcpstat.tcps_connattempt++;
1103         TCP_STATE_CHANGE(tp, TCPS_SYN_SENT);
1104         tcp_callout_reset(tp, tp->tt_keep, tp->t_keepinit, tcp_timer_keep);
1105         tp->iss = tcp_new_isn(tp);
1106         tcp_sendseqinit(tp);
1107         if (m) {
1108                 ssb_appendstream(&so->so_snd, m);
1109                 m = NULL;
1110                 if (flags & PRUS_OOB)
1111                         tp->snd_up = tp->snd_una + so->so_snd.ssb_cc;
1112         }
1113
1114         /*
1115          * Close the send side of the connection after
1116          * the data is sent if flagged.
1117          */
1118         if ((flags & (PRUS_OOB|PRUS_EOF)) == PRUS_EOF) {
1119                 socantsendmore(so);
1120                 tp = tcp_usrclosed(tp);
1121         }
1122         return (tcp_output(tp));
1123 }
1124
1125 /*
1126  * Common subroutine to open a TCP connection to remote host specified
1127  * by struct sockaddr_in in mbuf *nam.  Call in_pcbbind to assign a local
1128  * port number if needed.  Call in_pcbladdr to do the routing and to choose
1129  * a local host address (interface).
1130  * Initialize connection parameters and enter SYN-SENT state.
1131  */
1132 static void
1133 tcp_connect(netmsg_t msg)
1134 {
1135         struct socket *so = msg->connect.base.nm_so;
1136         struct sockaddr *nam = msg->connect.nm_nam;
1137         struct thread *td = msg->connect.nm_td;
1138         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
1139         struct sockaddr_in *if_sin = NULL;
1140         struct inpcb *inp;
1141         struct tcpcb *tp;
1142         int error;
1143         uint16_t hash;
1144         lwkt_port_t port;
1145
1146         COMMON_START(so, inp, 0);
1147
1148         /*
1149          * Reconnect our pcb if we have to
1150          */
1151         if (msg->connect.nm_flags & PRUC_RECONNECT) {
1152                 msg->connect.nm_flags &= ~PRUC_RECONNECT;
1153                 TCP_STATE_MIGRATE_END(tp);
1154                 in_pcblink(so->so_pcb, &tcbinfo[mycpu->gd_cpuid]);
1155         } else {
1156                 if (inp->inp_faddr.s_addr != INADDR_ANY) {
1157                         kprintf("inpcb %p, double-connect race\n", inp);
1158                         error = EISCONN;
1159                         if (so->so_state & SS_ISCONNECTING)
1160                                 error = EALREADY;
1161                         goto out;
1162                 }
1163                 KASSERT(inp->inp_fport == 0, ("invalid fport"));
1164         }
1165
1166         /*
1167          * Select local port, if it is not yet selected.
1168          */
1169         if (inp->inp_lport == 0) {
1170                 KKASSERT(inp->inp_laddr.s_addr == INADDR_ANY);
1171
1172                 error = in_pcbladdr(inp, nam, &if_sin, td);
1173                 if (error)
1174                         goto out;
1175                 inp->inp_laddr.s_addr = if_sin->sin_addr.s_addr;
1176                 msg->connect.nm_flags |= PRUC_HASLADDR;
1177
1178                 /*
1179                  * Install faddr/fport earlier, so that when this
1180                  * inpcb is installed on to the lport hash, the
1181                  * 4-tuple contains correct value.
1182                  *
1183                  * NOTE: The faddr/fport will have to be installed
1184                  * after the in_pcbladdr(), which may change them.
1185                  */
1186                 inp->inp_faddr = sin->sin_addr;
1187                 inp->inp_fport = sin->sin_port;
1188
1189                 error = in_pcbbind_remote(inp, nam, td);
1190                 if (error)
1191                         goto out;
1192         }
1193
1194         if ((msg->connect.nm_flags & PRUC_HASLADDR) == 0) {
1195                 /*
1196                  * Rarely used path:
1197                  * This inpcb was bound before this connect.
1198                  */
1199                 error = in_pcbladdr(inp, nam, &if_sin, td);
1200                 if (error)
1201                         goto out;
1202
1203                 /*
1204                  * Save or refresh the faddr/fport, since they may
1205                  * be changed by in_pcbladdr().
1206                  */
1207                 inp->inp_faddr = sin->sin_addr;
1208                 inp->inp_fport = sin->sin_port;
1209         }
1210 #ifdef INVARIANTS
1211         else {
1212                 KASSERT(inp->inp_faddr.s_addr == sin->sin_addr.s_addr,
1213                     ("faddr mismatch for reconnect"));
1214                 KASSERT(inp->inp_fport == sin->sin_port,
1215                     ("fport mismatch for reconnect"));
1216         }
1217 #endif
1218         KKASSERT(inp->inp_socket == so);
1219
1220         hash = tcp_addrhash(sin->sin_addr.s_addr, sin->sin_port,
1221                             (inp->inp_laddr.s_addr != INADDR_ANY ?
1222                              inp->inp_laddr.s_addr : if_sin->sin_addr.s_addr),
1223                             inp->inp_lport);
1224         port = netisr_hashport(hash);
1225
1226         if (port != &curthread->td_msgport) {
1227                 lwkt_msg_t lmsg = &msg->connect.base.lmsg;
1228
1229                 /*
1230                  * in_pcbladdr() may have allocated a route entry for us
1231                  * on the current CPU, but we need a route entry on the
1232                  * inpcb's owner CPU, so free it here.
1233                  */
1234                 in_pcbresetroute(inp);
1235
1236                 /*
1237                  * We are moving the protocol processing port the socket
1238                  * is on, we have to unlink here and re-link on the
1239                  * target cpu.
1240                  */
1241                 in_pcbunlink(so->so_pcb, &tcbinfo[mycpu->gd_cpuid]);
1242                 msg->connect.nm_flags |= PRUC_RECONNECT;
1243                 msg->connect.base.nm_dispatch = tcp_connect;
1244
1245                 TCP_STATE_MIGRATE_START(tp);
1246
1247                 /*
1248                  * Use message put done receipt to change this socket's
1249                  * so_port, i.e. _after_ this message was put onto the
1250                  * target netisr's msgport but _before_ the message could
1251                  * be pulled from the target netisr's msgport, so that:
1252                  * - The upper half (socket code) will not see the new
1253                  *   msgport before this message reaches the new msgport
1254                  *   and messages for this socket will be ordered.
1255                  * - This message will see the new msgport, when its
1256                  *   handler is called in the target netisr.
1257                  *
1258                  * NOTE:
1259                  * We MUST use messege put done receipt to change this
1260                  * socket's so_port:
1261                  * If we changed the so_port in this netisr after the
1262                  * lwkt_forwardmsg (so messages for this socket will be
1263                  * ordered) and changed the so_port in the target netisr
1264                  * at the very beginning of this message's handler, we
1265                  * would suffer so_port overwritten race, given this
1266                  * message might be forwarded again.
1267                  *
1268                  * NOTE:
1269                  * This mechanism depends on that the netisr's msgport
1270                  * is spin msgport (currently it is :).
1271                  *
1272                  * If the upper half saw the new msgport before this
1273                  * message reached the target netisr's msgport, the
1274                  * messages sent from the upper half could reach the new
1275                  * msgport before this message, thus there would be
1276                  * message reordering.  The worst case could be soclose()
1277                  * saw the new msgport and the detach message could reach
1278                  * the new msgport before this message, i.e. the inpcb
1279                  * could have been destroyed when this message was still
1280                  * pending on or on its way to the new msgport.  Other
1281                  * weird cases could also happen, e.g. inpcb->inp_pcbinfo,
1282                  * since we have unlinked this inpcb from the current
1283                  * pcbinfo first.
1284                  */
1285                 lwkt_setmsg_receipt(lmsg, tcp_sosetport);
1286                 lwkt_forwardmsg(port, lmsg);
1287                 /* msg invalid now */
1288                 return;
1289         } else if (msg->connect.nm_flags & PRUC_HELDTD) {
1290                 /*
1291                  * The original thread is no longer needed; release it.
1292                  */
1293                 lwkt_rele(td);
1294                 msg->connect.nm_flags &= ~PRUC_HELDTD;
1295         }
1296         error = tcp_connect_oncpu(tp, msg->connect.nm_sndflags,
1297                                   msg->connect.nm_m, sin, if_sin, hash);
1298         msg->connect.nm_m = NULL;
1299 out:
1300         if (msg->connect.nm_m) {
1301                 m_freem(msg->connect.nm_m);
1302                 msg->connect.nm_m = NULL;
1303         }
1304         if (msg->connect.nm_flags & PRUC_HELDTD)
1305                 lwkt_rele(td);
1306         if (error && (msg->connect.nm_flags & PRUC_ASYNC)) {
1307                 so->so_error = error;
1308                 soisdisconnected(so);
1309         }
1310         lwkt_replymsg(&msg->connect.base.lmsg, error);
1311         /* msg invalid now */
1312 }
1313
1314 #ifdef INET6
1315
1316 static void
1317 tcp6_connect(netmsg_t msg)
1318 {
1319         struct tcpcb *tp;
1320         struct socket *so = msg->connect.base.nm_so;
1321         struct sockaddr *nam = msg->connect.nm_nam;
1322         struct thread *td = msg->connect.nm_td;
1323         struct inpcb *inp;
1324         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)nam;
1325         struct in6_addr *addr6;
1326         lwkt_port_t port;
1327         int error;
1328
1329         COMMON_START(so, inp, 0);
1330
1331         /*
1332          * Reconnect our pcb if we have to
1333          */
1334         if (msg->connect.nm_flags & PRUC_RECONNECT) {
1335                 msg->connect.nm_flags &= ~PRUC_RECONNECT;
1336                 TCP_STATE_MIGRATE_END(tp);
1337                 in_pcblink(so->so_pcb, &tcbinfo[mycpu->gd_cpuid]);
1338         }
1339
1340         /*
1341          * Bind if we have to
1342          */
1343         if (inp->inp_lport == 0) {
1344                 error = in6_pcbbind(inp, NULL, td);
1345                 if (error)
1346                         goto out;
1347         }
1348
1349         /*
1350          * Cannot simply call in_pcbconnect, because there might be an
1351          * earlier incarnation of this same connection still in
1352          * TIME_WAIT state, creating an ADDRINUSE error.
1353          */
1354         error = in6_pcbladdr(inp, nam, &addr6, td);
1355         if (error)
1356                 goto out;
1357
1358         port = tcp6_addrport(); /* XXX hack for now, always cpu0 */
1359
1360         if (port != &curthread->td_msgport) {
1361                 lwkt_msg_t lmsg = &msg->connect.base.lmsg;
1362
1363                 /*
1364                  * in_pcbladdr() may have allocated a route entry for us
1365                  * on the current CPU, but we need a route entry on the
1366                  * inpcb's owner CPU, so free it here.
1367                  */
1368                 in_pcbresetroute(inp);
1369
1370                 in_pcbunlink(so->so_pcb, &tcbinfo[mycpu->gd_cpuid]);
1371                 msg->connect.nm_flags |= PRUC_RECONNECT;
1372                 msg->connect.base.nm_dispatch = tcp6_connect;
1373
1374                 TCP_STATE_MIGRATE_START(tp);
1375
1376                 /* See the related comment in tcp_connect() */
1377                 lwkt_setmsg_receipt(lmsg, tcp_sosetport);
1378                 lwkt_forwardmsg(port, lmsg);
1379                 /* msg invalid now */
1380                 return;
1381         }
1382         error = tcp6_connect_oncpu(tp, msg->connect.nm_sndflags,
1383                                    &msg->connect.nm_m, sin6, addr6);
1384         /* nm_m may still be intact */
1385 out:
1386         if (msg->connect.nm_m) {
1387                 m_freem(msg->connect.nm_m);
1388                 msg->connect.nm_m = NULL;
1389         }
1390         lwkt_replymsg(&msg->connect.base.lmsg, error);
1391         /* msg invalid now */
1392 }
1393
1394 static int
1395 tcp6_connect_oncpu(struct tcpcb *tp, int flags, struct mbuf **mp,
1396                    struct sockaddr_in6 *sin6, struct in6_addr *addr6)
1397 {
1398         struct mbuf *m = *mp;
1399         struct inpcb *inp = tp->t_inpcb;
1400         struct socket *so = inp->inp_socket;
1401         struct inpcb *oinp;
1402
1403         /*
1404          * Cannot simply call in_pcbconnect, because there might be an
1405          * earlier incarnation of this same connection still in
1406          * TIME_WAIT state, creating an ADDRINUSE error.
1407          */
1408         oinp = in6_pcblookup_hash(inp->inp_pcbinfo,
1409                                   &sin6->sin6_addr, sin6->sin6_port,
1410                                   (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr) ?
1411                                       addr6 : &inp->in6p_laddr),
1412                                   inp->inp_lport,  0, NULL);
1413         if (oinp)
1414                 return (EADDRINUSE);
1415
1416         if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr))
1417                 inp->in6p_laddr = *addr6;
1418         inp->in6p_faddr = sin6->sin6_addr;
1419         inp->inp_fport = sin6->sin6_port;
1420         if ((sin6->sin6_flowinfo & IPV6_FLOWINFO_MASK) != 0)
1421                 inp->in6p_flowinfo = sin6->sin6_flowinfo;
1422         in_pcbinsconnhash(inp);
1423
1424         /*
1425          * Now that no more errors can occur, change the protocol processing
1426          * port to the current thread (which is the correct thread).
1427          *
1428          * Create TCP timer message now; we are on the tcpcb's owner
1429          * CPU/thread.
1430          */
1431         tcp_create_timermsg(tp, &curthread->td_msgport);
1432
1433         /* Compute window scaling to request.  */
1434         if (tp->request_r_scale < TCP_MIN_WINSHIFT)
1435                 tp->request_r_scale = TCP_MIN_WINSHIFT;
1436         while (tp->request_r_scale < TCP_MAX_WINSHIFT &&
1437             (TCP_MAXWIN << tp->request_r_scale) < so->so_rcv.ssb_hiwat) {
1438                 tp->request_r_scale++;
1439         }
1440
1441         soisconnecting(so);
1442         tcpstat.tcps_connattempt++;
1443         TCP_STATE_CHANGE(tp, TCPS_SYN_SENT);
1444         tcp_callout_reset(tp, tp->tt_keep, tp->t_keepinit, tcp_timer_keep);
1445         tp->iss = tcp_new_isn(tp);
1446         tcp_sendseqinit(tp);
1447         if (m) {
1448                 ssb_appendstream(&so->so_snd, m);
1449                 *mp = NULL;
1450                 if (flags & PRUS_OOB)
1451                         tp->snd_up = tp->snd_una + so->so_snd.ssb_cc;
1452         }
1453
1454         /*
1455          * Close the send side of the connection after
1456          * the data is sent if flagged.
1457          */
1458         if ((flags & (PRUS_OOB|PRUS_EOF)) == PRUS_EOF) {
1459                 socantsendmore(so);
1460                 tp = tcp_usrclosed(tp);
1461         }
1462         return (tcp_output(tp));
1463 }
1464
1465 #endif /* INET6 */
1466
1467 /*
1468  * The new sockopt interface makes it possible for us to block in the
1469  * copyin/out step (if we take a page fault).  Taking a page fault while
1470  * in a critical section is probably a Bad Thing.  (Since sockets and pcbs
1471  * both now use TSM, there probably isn't any need for this function to 
1472  * run in a critical section any more.  This needs more examination.)
1473  */
1474 void
1475 tcp_ctloutput(netmsg_t msg)
1476 {
1477         struct socket *so = msg->base.nm_so;
1478         struct sockopt *sopt = msg->ctloutput.nm_sopt;
1479         struct thread *td = NULL;
1480         int     error, opt, optval, opthz;
1481         struct  inpcb *inp;
1482         struct  tcpcb *tp;
1483
1484         if (msg->ctloutput.nm_flags & PRCO_HELDTD)
1485                 td = sopt->sopt_td;
1486
1487         error = 0;
1488         inp = so->so_pcb;
1489         if (inp == NULL) {
1490                 error = ECONNRESET;
1491                 goto done;
1492         }
1493         tp = intotcpcb(inp);
1494
1495         /* Get socket's owner cpuid hint */
1496         if (sopt->sopt_level == SOL_SOCKET &&
1497             sopt->sopt_dir == SOPT_GET &&
1498             sopt->sopt_name == SO_CPUHINT) {
1499                 if (tp->t_flags & TF_LISTEN) {
1500                         /*
1501                          * Listen sockets owner cpuid is always 0,
1502                          * which does not make sense if SO_REUSEPORT
1503                          * is not set.
1504                          *
1505                          * NOTE: inp_lgrpindex is _not_ assigned in jail.
1506                          */
1507                         if ((so->so_options & SO_REUSEPORT) &&
1508                             inp->inp_lgrpindex >= 0)
1509                                 optval = inp->inp_lgrpindex % netisr_ncpus;
1510                         else
1511                                 optval = -1; /* no hint */
1512                 } else {
1513                         optval = mycpuid;
1514                 }
1515                 soopt_from_kbuf(sopt, &optval, sizeof(optval));
1516                 goto done;
1517         }
1518
1519         if (sopt->sopt_level != IPPROTO_TCP) {
1520                 if (sopt->sopt_level == IPPROTO_IP) {
1521                         switch (sopt->sopt_name) {
1522                         case IP_MULTICAST_IF:
1523                         case IP_MULTICAST_VIF:
1524                         case IP_MULTICAST_TTL:
1525                         case IP_MULTICAST_LOOP:
1526                         case IP_ADD_MEMBERSHIP:
1527                         case IP_DROP_MEMBERSHIP:
1528                                 /*
1529                                  * Multicast does not make sense on
1530                                  * TCP sockets.
1531                                  */
1532                                 error = EOPNOTSUPP;
1533                                 goto done;
1534                         }
1535                 }
1536 #ifdef INET6
1537                 if (INP_CHECK_SOCKAF(so, AF_INET6))
1538                         ip6_ctloutput_dispatch(msg);
1539                 else
1540 #endif /* INET6 */
1541                 ip_ctloutput(msg);
1542                 /* msg invalid now */
1543                 if (td != NULL)
1544                         lwkt_rele(td);
1545                 return;
1546         }
1547
1548         switch (sopt->sopt_dir) {
1549         case SOPT_SET:
1550                 error = soopt_to_kbuf(sopt, &optval, sizeof optval,
1551                                       sizeof optval);
1552                 if (error)
1553                         break;
1554                 switch (sopt->sopt_name) {
1555                 case TCP_FASTKEEP:
1556                         if (optval > 0)
1557                                 tp->t_keepidle = tp->t_keepintvl;
1558                         else
1559                                 tp->t_keepidle = tcp_keepidle;
1560                         tcp_timer_keep_activity(tp, 0);
1561                         break;
1562 #ifdef TCP_SIGNATURE
1563                 case TCP_SIGNATURE_ENABLE:
1564                         if (tp->t_state == TCPS_CLOSED) {
1565                                 /*
1566                                  * This is the only safe state that this
1567                                  * option could be changed.  Some segments
1568                                  * could already have been sent in other
1569                                  * states.
1570                                  */
1571                                 if (optval > 0)
1572                                         tp->t_flags |= TF_SIGNATURE;
1573                                 else
1574                                         tp->t_flags &= ~TF_SIGNATURE;
1575                         } else {
1576                                 error = EOPNOTSUPP;
1577                         }
1578                         break;
1579 #endif /* TCP_SIGNATURE */
1580                 case TCP_NODELAY:
1581                 case TCP_NOOPT:
1582                         switch (sopt->sopt_name) {
1583                         case TCP_NODELAY:
1584                                 opt = TF_NODELAY;
1585                                 break;
1586                         case TCP_NOOPT:
1587                                 opt = TF_NOOPT;
1588                                 break;
1589                         default:
1590                                 opt = 0; /* dead code to fool gcc */
1591                                 break;
1592                         }
1593
1594                         if (optval)
1595                                 tp->t_flags |= opt;
1596                         else
1597                                 tp->t_flags &= ~opt;
1598                         break;
1599
1600                 case TCP_NOPUSH:
1601                         if (tcp_disable_nopush)
1602                                 break;
1603                         if (optval)
1604                                 tp->t_flags |= TF_NOPUSH;
1605                         else {
1606                                 tp->t_flags &= ~TF_NOPUSH;
1607                                 error = tcp_output(tp);
1608                         }
1609                         break;
1610
1611                 case TCP_MAXSEG:
1612                         /*
1613                          * Must be between 0 and maxseg.  If the requested
1614                          * maxseg is too small to satisfy the desired minmss,
1615                          * pump it up (silently so sysctl modifications of
1616                          * minmss do not create unexpected program failures).
1617                          * Handle degenerate cases.
1618                          */
1619                         if (optval > 0 && optval <= tp->t_maxseg) {
1620                                 if (optval + 40 < tcp_minmss) {
1621                                         optval = tcp_minmss - 40;
1622                                         if (optval < 0)
1623                                                 optval = 1;
1624                                 }
1625                                 tp->t_maxseg = optval;
1626                         } else {
1627                                 error = EINVAL;
1628                         }
1629                         break;
1630
1631                 case TCP_KEEPINIT:
1632                 case TCP_KEEPIDLE:
1633                 case TCP_KEEPINTVL:
1634                         if ((uint32_t)optval >= MAXKEEP / hz)
1635                                 opthz = MAXKEEP;
1636                         else
1637                                 opthz = optval * hz;
1638                         if (opthz == 0)
1639                                 opthz = 1;
1640                         switch (sopt->sopt_name) {
1641                         case TCP_KEEPINIT:
1642                                 tp->t_keepinit = opthz;
1643                                 break;
1644                         case TCP_KEEPIDLE:
1645                                 tp->t_keepidle = opthz;
1646                                 tcp_timer_keep_activity(tp, 0);
1647                                 break;
1648                         case TCP_KEEPINTVL:
1649                                 tp->t_keepintvl = opthz;
1650                                 if (tp->t_keepcnt == 0 ||
1651                                     tp->t_keepintvl >= MAXKEEP / tp->t_keepcnt)
1652                                         tp->t_maxidle = MAXKEEP;
1653                                 else
1654                                         tp->t_maxidle = tp->t_keepintvl *
1655                                                         tp->t_keepcnt;
1656                         }
1657                         break;
1658
1659                 case TCP_KEEPCNT:
1660                         if (optval > 0) {
1661                                 tp->t_keepcnt = optval;
1662                                 if (tp->t_keepcnt == 0 ||
1663                                     tp->t_keepintvl >= MAXKEEP / tp->t_keepcnt)
1664                                         tp->t_maxidle = MAXKEEP;
1665                                 else
1666                                         tp->t_maxidle = tp->t_keepintvl *
1667                                                         tp->t_keepcnt;
1668                         } else {
1669                                 error = EINVAL;
1670                         }
1671                         break;
1672
1673                 default:
1674                         error = ENOPROTOOPT;
1675                         break;
1676                 }
1677                 break;
1678
1679         case SOPT_GET:
1680                 switch (sopt->sopt_name) {
1681 #ifdef TCP_SIGNATURE
1682                 case TCP_SIGNATURE_ENABLE:
1683                         optval = (tp->t_flags & TF_SIGNATURE) ? 1 : 0;
1684                         break;
1685 #endif /* TCP_SIGNATURE */
1686                 case TCP_NODELAY:
1687                         optval = tp->t_flags & TF_NODELAY;
1688                         break;
1689                 case TCP_MAXSEG:
1690                         optval = tp->t_maxseg;
1691                         break;
1692                 case TCP_NOOPT:
1693                         optval = tp->t_flags & TF_NOOPT;
1694                         break;
1695                 case TCP_NOPUSH:
1696                         optval = tp->t_flags & TF_NOPUSH;
1697                         break;
1698                 case TCP_KEEPINIT:
1699                         optval = tp->t_keepinit / hz;
1700                         break;
1701                 case TCP_KEEPIDLE:
1702                         optval = tp->t_keepidle / hz;
1703                         break;
1704                 case TCP_KEEPINTVL:
1705                         optval = tp->t_keepintvl / hz;
1706                         break;
1707                 case TCP_KEEPCNT:
1708                         optval = tp->t_keepcnt;
1709                         break;
1710                 default:
1711                         error = ENOPROTOOPT;
1712                         break;
1713                 }
1714                 if (error == 0)
1715                         soopt_from_kbuf(sopt, &optval, sizeof optval);
1716                 break;
1717         }
1718 done:
1719         if (td != NULL)
1720                 lwkt_rele(td);
1721         lwkt_replymsg(&msg->lmsg, error);
1722 }
1723
1724 struct netmsg_tcp_ctloutput {
1725         struct netmsg_pr_ctloutput ctloutput;
1726         struct sockopt          sopt;
1727         int                     sopt_val;
1728 };
1729
1730 /*
1731  * Allocate netmsg_pr_ctloutput for asynchronous tcp_ctloutput.
1732  */
1733 struct netmsg_pr_ctloutput *
1734 tcp_ctloutmsg(struct sockopt *sopt)
1735 {
1736         struct netmsg_tcp_ctloutput *msg;
1737         int flags = 0, error;
1738
1739         KASSERT(sopt->sopt_dir == SOPT_SET, ("not from ctloutput"));
1740
1741         /* Only small set of options allows asynchronous setting. */
1742         if (sopt->sopt_level != IPPROTO_TCP)
1743                 return NULL;
1744         switch (sopt->sopt_name) {
1745         case TCP_NODELAY:
1746         case TCP_NOOPT:
1747         case TCP_NOPUSH:
1748         case TCP_FASTKEEP:
1749                 break;
1750         default:
1751                 return NULL;
1752         }
1753
1754         msg = kmalloc(sizeof(*msg), M_LWKTMSG, M_WAITOK | M_NULLOK);
1755         if (msg == NULL) {
1756                 /* Fallback to synchronous tcp_ctloutput */
1757                 return NULL;
1758         }
1759
1760         /* Save the sockopt */
1761         msg->sopt = *sopt;
1762
1763         /* Fixup the sopt.sopt_val ptr */
1764         error = sooptcopyin(sopt, &msg->sopt_val,
1765             sizeof(msg->sopt_val), sizeof(msg->sopt_val));
1766         if (error) {
1767                 kfree(msg, M_LWKTMSG);
1768                 return NULL;
1769         }
1770         msg->sopt.sopt_val = &msg->sopt_val;
1771
1772         /* Hold the current thread */
1773         if (msg->sopt.sopt_td != NULL) {
1774                 flags |= PRCO_HELDTD;
1775                 lwkt_hold(msg->sopt.sopt_td);
1776         }
1777
1778         msg->ctloutput.nm_flags = flags;
1779         msg->ctloutput.nm_sopt = &msg->sopt;
1780
1781         return &msg->ctloutput;
1782 }
1783
1784 /*
1785  * tcp_sendspace and tcp_recvspace are the default send and receive window
1786  * sizes, respectively.  These are obsolescent (this information should
1787  * be set by the route).
1788  *
1789  * Use a default that does not require tcp window scaling to be turned
1790  * on.  Individual programs or the administrator can increase the default.
1791  */
1792 u_long  tcp_sendspace = 57344;  /* largest multiple of PAGE_SIZE < 64k */
1793 SYSCTL_INT(_net_inet_tcp, TCPCTL_SENDSPACE, sendspace, CTLFLAG_RW,
1794     &tcp_sendspace , 0, "Maximum outgoing TCP datagram size");
1795 u_long  tcp_recvspace = 57344;  /* largest multiple of PAGE_SIZE < 64k */
1796 SYSCTL_INT(_net_inet_tcp, TCPCTL_RECVSPACE, recvspace, CTLFLAG_RW,
1797     &tcp_recvspace , 0, "Maximum incoming TCP datagram size");
1798
1799 /*
1800  * Attach TCP protocol to socket, allocating internet protocol control
1801  * block, tcp control block, buffer space, and entering CLOSED state.
1802  */
1803 static int
1804 tcp_attach(struct socket *so, struct pru_attach_info *ai)
1805 {
1806         struct inpcb *inp;
1807         int error;
1808         int cpu;
1809 #ifdef INET6
1810         boolean_t isipv6 = INP_CHECK_SOCKAF(so, AF_INET6);
1811 #endif
1812
1813         if (ai != NULL) {
1814                 error = tcp_usr_preattach(so, 0 /* don't care */, ai);
1815                 if (error)
1816                         return (error);
1817         } else {
1818                 /* Post attach; do nothing */
1819         }
1820
1821         cpu = mycpu->gd_cpuid;
1822
1823         /*
1824          * Set the default pcbinfo.  This will likely change when we
1825          * bind/connect.
1826          */
1827         error = in_pcballoc(so, &tcbinfo[cpu]);
1828         if (error)
1829                 return (error);
1830         inp = so->so_pcb;
1831 #ifdef INET6
1832         if (isipv6)
1833                 inp->in6p_hops = -1;    /* use kernel default */
1834 #endif
1835         tcp_newtcpcb(inp);
1836         /* Keep a reference for asynchronized pru_rcvd */
1837         soreference(so);
1838         return (0);
1839 }
1840
1841 /*
1842  * Initiate (or continue) disconnect.
1843  * If embryonic state, just send reset (once).
1844  * If in ``let data drain'' option and linger null, just drop.
1845  * Otherwise (hard), mark socket disconnecting and drop
1846  * current input data; switch states based on user close, and
1847  * send segment to peer (with FIN).
1848  */
1849 static struct tcpcb *
1850 tcp_disconnect(struct tcpcb *tp)
1851 {
1852         struct socket *so = tp->t_inpcb->inp_socket;
1853
1854         if (tp->t_state < TCPS_ESTABLISHED) {
1855                 tp = tcp_close(tp);
1856         } else if ((so->so_options & SO_LINGER) && so->so_linger == 0) {
1857                 tp = tcp_drop(tp, 0);
1858         } else {
1859                 lwkt_gettoken(&so->so_rcv.ssb_token);
1860                 soisdisconnecting(so);
1861                 sbflush(&so->so_rcv.sb);
1862                 tp = tcp_usrclosed(tp);
1863                 if (tp)
1864                         tcp_output(tp);
1865                 lwkt_reltoken(&so->so_rcv.ssb_token);
1866         }
1867         return (tp);
1868 }
1869
1870 /*
1871  * User issued close, and wish to trail through shutdown states:
1872  * if never received SYN, just forget it.  If got a SYN from peer,
1873  * but haven't sent FIN, then go to FIN_WAIT_1 state to send peer a FIN.
1874  * If already got a FIN from peer, then almost done; go to LAST_ACK
1875  * state.  In all other cases, have already sent FIN to peer (e.g.
1876  * after PRU_SHUTDOWN), and just have to play tedious game waiting
1877  * for peer to send FIN or not respond to keep-alives, etc.
1878  * We can let the user exit from the close as soon as the FIN is acked.
1879  */
1880 static struct tcpcb *
1881 tcp_usrclosed(struct tcpcb *tp)
1882 {
1883
1884         switch (tp->t_state) {
1885
1886         case TCPS_CLOSED:
1887         case TCPS_LISTEN:
1888                 TCP_STATE_CHANGE(tp, TCPS_CLOSED);
1889                 tp = tcp_close(tp);
1890                 break;
1891
1892         case TCPS_SYN_SENT:
1893         case TCPS_SYN_RECEIVED:
1894                 tp->t_flags |= TF_NEEDFIN;
1895                 break;
1896
1897         case TCPS_ESTABLISHED:
1898                 TCP_STATE_CHANGE(tp, TCPS_FIN_WAIT_1);
1899                 break;
1900
1901         case TCPS_CLOSE_WAIT:
1902                 TCP_STATE_CHANGE(tp, TCPS_LAST_ACK);
1903                 break;
1904         }
1905         if (tp && tp->t_state >= TCPS_FIN_WAIT_2) {
1906                 soisdisconnected(tp->t_inpcb->inp_socket);
1907                 /* To prevent the connection hanging in FIN_WAIT_2 forever. */
1908                 if (tp->t_state == TCPS_FIN_WAIT_2) {
1909                         tcp_callout_reset(tp, tp->tt_2msl, tp->t_maxidle,
1910                             tcp_timer_2msl);
1911                 }
1912         }
1913         return (tp);
1914 }