sys/netinet/in_pcb.c

   1 /*
   2  * Copyright (c) 2004 Jeffrey M. Hsu.  All rights reserved.
   3  * Copyright (c) 2004 The DragonFly Project.  All rights reserved.
   4  *
   5  * This code is derived from software contributed to The DragonFly Project
   6  * by Jeffrey M. Hsu.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the name of The DragonFly Project nor the names of its
  17  *    contributors may be used to endorse or promote products derived
  18  *    from this software without specific, prior written permission.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  23  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  24  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  25  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  26  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  27  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  28  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  29  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  30  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  */
  33
  34 /*
  35  * Copyright (c) 1982, 1986, 1991, 1993, 1995
  36  *      The Regents of the University of California.  All rights reserved.
  37  *
  38  * Redistribution and use in source and binary forms, with or without
  39  * modification, are permitted provided that the following conditions
  40  * are met:
  41  * 1. Redistributions of source code must retain the above copyright
  42  *    notice, this list of conditions and the following disclaimer.
  43  * 2. Redistributions in binary form must reproduce the above copyright
  44  *    notice, this list of conditions and the following disclaimer in the
  45  *    documentation and/or other materials provided with the distribution.
  46  * 3. Neither the name of the University nor the names of its contributors
  47  *    may be used to endorse or promote products derived from this software
  48  *    without specific prior written permission.
  49  *
  50  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  51  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  52  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  53  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  54  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  55  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  56  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  57  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  58  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  59  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  60  * SUCH DAMAGE.
  61  *
  62  *      @(#)in_pcb.c    8.4 (Berkeley) 5/24/95
  63  * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $
  64  */
  65
  66 #include "opt_ipsec.h"
  67 #include "opt_inet6.h"
  68
  69 #include <sys/param.h>
  70 #include <sys/systm.h>
  71 #include <sys/malloc.h>
  72 #include <sys/mbuf.h>
  73 #include <sys/domain.h>
  74 #include <sys/protosw.h>
  75 #include <sys/socket.h>
  76 #include <sys/socketvar.h>
  77 #include <sys/proc.h>
  78 #include <sys/priv.h>
  79 #include <sys/jail.h>
  80 #include <sys/kernel.h>
  81 #include <sys/sysctl.h>
  82
  83 #include <sys/thread2.h>
  84 #include <sys/socketvar2.h>
  85 #include <sys/msgport2.h>
  86
  87 #include <machine/limits.h>
  88
  89 #include <net/if.h>
  90 #include <net/if_types.h>
  91 #include <net/route.h>
  92 #include <net/netisr2.h>
  93 #include <net/toeplitz2.h>
  94
  95 #include <netinet/in.h>
  96 #include <netinet/in_pcb.h>
  97 #include <netinet/in_var.h>
  98 #include <netinet/ip_var.h>
  99 #ifdef INET6
 100 #include <netinet/ip6.h>
 101 #include <netinet6/ip6_var.h>
 102 #endif /* INET6 */
 103
 104 #ifdef IPSEC
 105 #include <netinet6/ipsec.h>
 106 #include <netproto/key/key.h>
 107 #include <netproto/ipsec/esp_var.h>
 108 #endif
 109
 110 #ifdef FAST_IPSEC
 111 #if defined(IPSEC) || defined(IPSEC_ESP)
 112 #error "Bad idea: don't compile with both IPSEC and FAST_IPSEC!"
 113 #endif
 114
 115 #include <netproto/ipsec/ipsec.h>
 116 #include <netproto/ipsec/key.h>
 117 #define IPSEC
 118 #endif /* FAST_IPSEC */
 119
 120 #define INP_LOCALGROUP_SIZMIN   8
 121 #define INP_LOCALGROUP_SIZMAX   256
 122
 123 static struct inpcb *in_pcblookup_local(struct inpcbporthead *porthash,
 124                 struct in_addr laddr, u_int lport_arg, int wild_okay,
 125                 struct ucred *cred);
 126
 127 struct in_addr zeroin_addr;
 128
 129 /*
 130  * These configure the range of local port addresses assigned to
 131  * "unspecified" outgoing connections/packets/whatever.
 132  */
 133 int ipport_lowfirstauto = IPPORT_RESERVED - 1;  /* 1023 */
 134 int ipport_lowlastauto = IPPORT_RESERVEDSTART;  /* 600 */
 135
 136 int ipport_firstauto = IPPORT_RESERVED;         /* 1024 */
 137 int ipport_lastauto = IPPORT_USERRESERVED;      /* 5000 */
 138
 139 int ipport_hifirstauto = IPPORT_HIFIRSTAUTO;    /* 49152 */
 140 int ipport_hilastauto = IPPORT_HILASTAUTO;      /* 65535 */
 141
 142 #define RANGECHK(var, min, max) \
 143         if ((var) < (min)) { (var) = (min); } \
 144         else if ((var) > (max)) { (var) = (max); }
 145
 146 int udpencap_enable = 1;        /* enabled by default */
 147 int udpencap_port = 4500;       /* triggers decapsulation */
 148
 149 /*
 150  * Per-netisr inpcb markers.
 151  * NOTE: they should only be used in netisrs.
 152  */
 153 static struct inpcb             *in_pcbmarkers;
 154 static struct inpcontainer      *in_pcbcontainer_markers;
 155
 156 static int
 157 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
 158 {
 159         int error;
 160
 161         error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
 162         if (!error) {
 163                 RANGECHK(ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
 164                 RANGECHK(ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
 165
 166                 RANGECHK(ipport_firstauto, IPPORT_RESERVED, USHRT_MAX);
 167                 RANGECHK(ipport_lastauto, IPPORT_RESERVED, USHRT_MAX);
 168
 169                 RANGECHK(ipport_hifirstauto, IPPORT_RESERVED, USHRT_MAX);
 170                 RANGECHK(ipport_hilastauto, IPPORT_RESERVED, USHRT_MAX);
 171         }
 172         return (error);
 173 }
 174
 175 #undef RANGECHK
 176
 177 SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports");
 178
 179 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW,
 180            &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", "");
 181 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, CTLTYPE_INT|CTLFLAG_RW,
 182            &ipport_lowlastauto, 0, &sysctl_net_ipport_check, "I", "");
 183 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, CTLTYPE_INT|CTLFLAG_RW,
 184            &ipport_firstauto, 0, &sysctl_net_ipport_check, "I", "");
 185 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, CTLTYPE_INT|CTLFLAG_RW,
 186            &ipport_lastauto, 0, &sysctl_net_ipport_check, "I", "");
 187 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, CTLTYPE_INT|CTLFLAG_RW,
 188            &ipport_hifirstauto, 0, &sysctl_net_ipport_check, "I", "");
 189 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW,
 190            &ipport_hilastauto, 0, &sysctl_net_ipport_check, "I", "");
 191
 192 static int ip_porthash_trycount = 15;
 193 SYSCTL_INT(_net_inet_ip, OID_AUTO, porthash_trycount, CTLFLAG_RW,
 194     &ip_porthash_trycount, 0,
 195     "Number of tries to find local port matching hash of 4-tuple");
 196
 197 /*
 198  * in_pcb.c: manage the Protocol Control Blocks.
 199  *
 200  * NOTE: It is assumed that most of these functions will be called from
 201  * a critical section.  XXX - There are, unfortunately, a few exceptions
 202  * to this rule that should be fixed.
 203  *
 204  * NOTE: The caller should initialize the cpu field to the cpu running the
 205  * protocol stack associated with this inpcbinfo.
 206  */
 207
 208 void
 209 in_pcbinfo_init(struct inpcbinfo *pcbinfo, int cpu, boolean_t shared)
 210 {
 211         KASSERT(cpu >= 0 && cpu < ncpus, ("invalid cpu%d", cpu));
 212         pcbinfo->cpu = cpu;
 213
 214         LIST_INIT(&pcbinfo->pcblisthead);
 215         pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), M_PCB,
 216                                     M_WAITOK | M_ZERO);
 217
 218         if (shared) {
 219                 pcbinfo->infotoken = kmalloc(sizeof(struct lwkt_token),
 220                     M_PCB, M_WAITOK);
 221                 lwkt_token_init(pcbinfo->infotoken, "infotoken");
 222         } else {
 223                 pcbinfo->infotoken = NULL;
 224         }
 225 }
 226
 227 struct baddynamicports baddynamicports;
 228
 229 /*
 230  * Check if the specified port is invalid for dynamic allocation.
 231  */
 232 int
 233 in_baddynamic(u_int16_t port, u_int16_t proto)
 234 {
 235         switch (proto) {
 236         case IPPROTO_TCP:
 237                 return (DP_ISSET(baddynamicports.tcp, port));
 238         case IPPROTO_UDP:
 239 #ifdef IPSEC
 240                 /* Cannot preset this as it is a sysctl */
 241                 if (port == udpencap_port)
 242                         return (1);
 243 #endif
 244                 return (DP_ISSET(baddynamicports.udp, port));
 245         default:
 246                 return (0);
 247         }
 248 }
 249
 250 void
 251 in_pcbonlist(struct inpcb *inp)
 252 {
 253         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 254
 255         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
 256             ("not in the correct netisr"));
 257         KASSERT((inp->inp_flags & INP_ONLIST) == 0, ("already on pcblist"));
 258         inp->inp_flags |= INP_ONLIST;
 259
 260         GET_PCBINFO_TOKEN(pcbinfo);
 261         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list);
 262         pcbinfo->ipi_count++;
 263         REL_PCBINFO_TOKEN(pcbinfo);
 264 }
 265
 266 void
 267 in_pcbofflist(struct inpcb *inp)
 268 {
 269         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 270
 271         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
 272             ("not in the correct netisr"));
 273         KASSERT(inp->inp_flags & INP_ONLIST, ("not on pcblist"));
 274         inp->inp_flags &= ~INP_ONLIST;
 275
 276         GET_PCBINFO_TOKEN(pcbinfo);
 277         LIST_REMOVE(inp, inp_list);
 278         KASSERT(pcbinfo->ipi_count > 0,
 279             ("invalid inpcb count %d", pcbinfo->ipi_count));
 280         pcbinfo->ipi_count--;
 281         REL_PCBINFO_TOKEN(pcbinfo);
 282 }
 283
 284 /*
 285  * Allocate a PCB and associate it with the socket.
 286  */
 287 int
 288 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
 289 {
 290         struct inpcb *inp;
 291 #ifdef IPSEC
 292         int error;
 293 #endif
 294
 295         inp = kmalloc(pcbinfo->ipi_size, M_PCB, M_WAITOK|M_ZERO|M_NULLOK);
 296         if (inp == NULL)
 297                 return (ENOMEM);
 298         inp->inp_lgrpindex = -1;
 299         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
 300         inp->inp_pcbinfo = pcbinfo;
 301         inp->inp_socket = so;
 302 #ifdef IPSEC
 303         error = ipsec_init_policy(so, &inp->inp_sp);
 304         if (error != 0) {
 305                 kfree(inp, M_PCB);
 306                 return (error);
 307         }
 308 #endif
 309 #ifdef INET6
 310         if (INP_CHECK_SOCKAF(so, AF_INET6)) {
 311                 if (ip6_auto_flowlabel)
 312                         inp->inp_flags |= IN6P_AUTOFLOWLABEL;
 313                 inp->inp_af = AF_INET6;
 314         } else
 315 #endif
 316         inp->inp_af = AF_INET;
 317         soreference(so);
 318         so->so_pcb = inp;
 319
 320         in_pcbonlist(inp);
 321         return (0);
 322 }
 323
 324 /*
 325  * Unlink a pcb with the intention of moving it to another cpu with a
 326  * different pcbinfo.  While unlinked nothing should attempt to dereference
 327  * inp_pcbinfo, NULL it out so we assert if it does.
 328  */
 329 void
 330 in_pcbunlink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags)
 331 {
 332         KASSERT(inp->inp_pcbinfo == pcbinfo, ("pcbinfo mismatch"));
 333         KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0,
 334             ("already linked"));
 335
 336         in_pcbofflist(inp);
 337         inp->inp_pcbinfo = NULL;
 338 }
 339
 340 void
 341 in_pcbunlink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
 342 {
 343         in_pcbunlink_flags(inp, pcbinfo, INP_WILDCARD);
 344 }
 345
 346 /*
 347  * Relink a pcb into a new pcbinfo.
 348  */
 349 void
 350 in_pcblink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags)
 351 {
 352         KASSERT(inp->inp_pcbinfo == NULL, ("has pcbinfo"));
 353         KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0,
 354             ("already linked"));
 355
 356         inp->inp_pcbinfo = pcbinfo;
 357         in_pcbonlist(inp);
 358 }
 359
 360 void
 361 in_pcblink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
 362 {
 363         return in_pcblink_flags(inp, pcbinfo, INP_WILDCARD);
 364 }
 365
 366 static boolean_t
 367 in_pcbporthash_update(struct inpcbportinfo *portinfo,
 368     struct inpcb *inp, u_short lport, struct ucred *cred, int wild)
 369 {
 370         struct inpcbporthead *porthash;
 371
 372         /*
 373          * This has to be atomic.  If the porthash is shared across multiple
 374          * protocol threads, e.g. tcp and udp, then the token must be held.
 375          */
 376         porthash = in_pcbporthash_head(portinfo, lport);
 377         GET_PORTHASH_TOKEN(porthash);
 378
 379         if (in_pcblookup_local(porthash, inp->inp_laddr, lport,
 380             wild, cred) != NULL) {
 381                 REL_PORTHASH_TOKEN(porthash);
 382                 return FALSE;
 383         }
 384         inp->inp_lport = lport;
 385         in_pcbinsporthash(porthash, inp);
 386
 387         REL_PORTHASH_TOKEN(porthash);
 388         return TRUE;
 389 }
 390
 391 static int
 392 in_pcbsetlport(struct inpcb *inp, int wild, struct ucred *cred)
 393 {
 394         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 395         struct inpcbportinfo *portinfo;
 396         u_short first, last, lport, step;
 397         u_short *lastport;
 398         int count, error;
 399         int portinfo_first, portinfo_idx;
 400
 401         inp->inp_flags |= INP_ANONPORT;
 402
 403         step = pcbinfo->portinfo_mask + 1;
 404         portinfo_first = mycpuid & pcbinfo->portinfo_mask;
 405         portinfo_idx = portinfo_first;
 406 loop:
 407         portinfo = &pcbinfo->portinfo[portinfo_idx];
 408
 409         if (inp->inp_flags & INP_HIGHPORT) {
 410                 first = ipport_hifirstauto;     /* sysctl */
 411                 last  = ipport_hilastauto;
 412                 lastport = &portinfo->lasthi;
 413         } else if (inp->inp_flags & INP_LOWPORT) {
 414                 if (cred &&
 415                     (error =
 416                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 417                         inp->inp_laddr.s_addr = INADDR_ANY;
 418                         return error;
 419                 }
 420                 first = ipport_lowfirstauto;    /* 1023 */
 421                 last  = ipport_lowlastauto;     /* 600 */
 422                 lastport = &portinfo->lastlow;
 423         } else {
 424                 first = ipport_firstauto;       /* sysctl */
 425                 last  = ipport_lastauto;
 426                 lastport = &portinfo->lastport;
 427         }
 428
 429         /*
 430          * Simple check to ensure all ports are not used up causing
 431          * a deadlock here.
 432          *
 433          * We split the two cases (up and down) so that the direction
 434          * is not being tested on each round of the loop.
 435          */
 436         if (first > last) {
 437                 /*
 438                  * counting down
 439                  */
 440                 in_pcbportrange(&first, &last, portinfo->offset, step);
 441                 count = (first - last) / step;
 442
 443                 for (;;) {
 444                         if (count-- < 0) {      /* completely used? */
 445                                 error = EADDRNOTAVAIL;
 446                                 break;
 447                         }
 448                         lport = in_pcblastport_down(lastport,
 449                             first, last, step);
 450                         KKASSERT((lport & pcbinfo->portinfo_mask) ==
 451                             portinfo->offset);
 452                         lport = htons(lport);
 453
 454                         if (in_pcbporthash_update(portinfo, inp, lport,
 455                             cred, wild)) {
 456                                 error = 0;
 457                                 break;
 458                         }
 459                 }
 460         } else {
 461                 /*
 462                  * counting up
 463                  */
 464                 in_pcbportrange(&last, &first, portinfo->offset, step);
 465                 count = (last - first) / step;
 466
 467                 for (;;) {
 468                         if (count-- < 0) {      /* completely used? */
 469                                 error = EADDRNOTAVAIL;
 470                                 break;
 471                         }
 472                         lport = in_pcblastport_up(lastport, first, last, step);
 473                         KKASSERT((lport & pcbinfo->portinfo_mask) ==
 474                             portinfo->offset);
 475                         lport = htons(lport);
 476
 477                         if (in_pcbporthash_update(portinfo, inp, lport,
 478                             cred, wild)) {
 479                                 error = 0;
 480                                 break;
 481                         }
 482                 }
 483         }
 484
 485         if (error) {
 486                 /* Try next portinfo */
 487                 portinfo_idx++;
 488                 portinfo_idx &= pcbinfo->portinfo_mask;
 489                 if (portinfo_idx != portinfo_first)
 490                         goto loop;
 491                 inp->inp_laddr.s_addr = INADDR_ANY;
 492         }
 493         return error;
 494 }
 495
 496 int
 497 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
 498 {
 499         struct socket *so = inp->inp_socket;
 500         struct sockaddr_in jsin;
 501         struct ucred *cred = NULL;
 502         int wild = 0;
 503
 504         if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */
 505                 return (EADDRNOTAVAIL);
 506         if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
 507                 return (EINVAL);        /* already bound */
 508
 509         if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT)))
 510                 wild = 1;    /* neither SO_REUSEADDR nor SO_REUSEPORT is set */
 511         if (td->td_proc)
 512                 cred = td->td_proc->p_ucred;
 513
 514         if (nam != NULL) {
 515                 struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 516                 struct inpcbinfo *pcbinfo;
 517                 struct inpcbportinfo *portinfo;
 518                 struct inpcbporthead *porthash;
 519                 struct inpcb *t;
 520                 u_short lport, lport_ho;
 521                 int reuseport = (so->so_options & SO_REUSEPORT);
 522                 int error;
 523
 524                 if (nam->sa_len != sizeof *sin)
 525                         return (EINVAL);
 526 #ifdef notdef
 527                 /*
 528                  * We should check the family, but old programs
 529                  * incorrectly fail to initialize it.
 530                  */
 531                 if (sin->sin_family != AF_INET)
 532                         return (EAFNOSUPPORT);
 533 #endif
 534                 if (!prison_replace_wildcards(td, nam))
 535                         return (EINVAL);
 536
 537                 lport = sin->sin_port;
 538                 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 539                         /*
 540                          * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
 541                          * allow complete duplication of binding if
 542                          * SO_REUSEPORT is set, or if SO_REUSEADDR is set
 543                          * and a multicast address is bound on both
 544                          * new and duplicated sockets.
 545                          */
 546                         if (so->so_options & SO_REUSEADDR)
 547                                 reuseport = SO_REUSEADDR | SO_REUSEPORT;
 548                 } else if (sin->sin_addr.s_addr != INADDR_ANY) {
 549                         sin->sin_port = 0;              /* yech... */
 550                         bzero(&sin->sin_zero, sizeof sin->sin_zero);
 551                         if (ifa_ifwithaddr((struct sockaddr *)sin) == NULL)
 552                                 return (EADDRNOTAVAIL);
 553                 }
 554
 555                 inp->inp_laddr = sin->sin_addr;
 556
 557                 jsin.sin_family = AF_INET;
 558                 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 559                 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 560                         inp->inp_laddr.s_addr = INADDR_ANY;
 561                         return (EINVAL);
 562                 }
 563                 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 564
 565                 if (lport == 0) {
 566                         /* Auto-select local port */
 567                         return in_pcbsetlport(inp, wild, cred);
 568                 }
 569                 lport_ho = ntohs(lport);
 570
 571                 /* GROSS */
 572                 if (lport_ho < IPPORT_RESERVED && cred &&
 573                     (error =
 574                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 575                         inp->inp_laddr.s_addr = INADDR_ANY;
 576                         return (error);
 577                 }
 578
 579                 /*
 580                  * Locate the proper portinfo based on lport
 581                  */
 582                 pcbinfo = inp->inp_pcbinfo;
 583                 portinfo =
 584                     &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask];
 585                 KKASSERT((lport_ho & pcbinfo->portinfo_mask) ==
 586                     portinfo->offset);
 587
 588                 /*
 589                  * This has to be atomic.  If the porthash is shared across
 590                  * multiple protocol threads, e.g. tcp and udp then the token
 591                  * must be held.
 592                  */
 593                 porthash = in_pcbporthash_head(portinfo, lport);
 594                 GET_PORTHASH_TOKEN(porthash);
 595
 596                 if (so->so_cred->cr_uid != 0 &&
 597                     !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 598                         t = in_pcblookup_local(porthash, sin->sin_addr, lport,
 599                             INPLOOKUP_WILDCARD, cred);
 600                         if (t &&
 601                             (so->so_cred->cr_uid !=
 602                              t->inp_socket->so_cred->cr_uid)) {
 603                                 inp->inp_laddr.s_addr = INADDR_ANY;
 604                                 error = EADDRINUSE;
 605                                 goto done;
 606                         }
 607                 }
 608                 if (cred && !prison_replace_wildcards(td, nam)) {
 609                         inp->inp_laddr.s_addr = INADDR_ANY;
 610                         error = EADDRNOTAVAIL;
 611                         goto done;
 612                 }
 613                 t = in_pcblookup_local(porthash, sin->sin_addr, lport,
 614                     wild, cred);
 615                 if (t && !(reuseport & t->inp_socket->so_options)) {
 616                         inp->inp_laddr.s_addr = INADDR_ANY;
 617                         error = EADDRINUSE;
 618                         goto done;
 619                 }
 620                 inp->inp_lport = lport;
 621                 in_pcbinsporthash(porthash, inp);
 622                 error = 0;
 623 done:
 624                 REL_PORTHASH_TOKEN(porthash);
 625                 return (error);
 626         } else {
 627                 jsin.sin_family = AF_INET;
 628                 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 629                 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 630                         inp->inp_laddr.s_addr = INADDR_ANY;
 631                         return (EINVAL);
 632                 }
 633                 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 634
 635                 return in_pcbsetlport(inp, wild, cred);
 636         }
 637 }
 638
 639 static struct inpcb *
 640 in_pcblookup_localremote(struct inpcbporthead *porthash, struct in_addr laddr,
 641     u_short lport, struct in_addr faddr, u_short fport, struct ucred *cred)
 642 {
 643         struct inpcb *inp;
 644         struct inpcbport *phd;
 645         struct inpcb *match = NULL;
 646
 647         /*
 648          * If the porthashbase is shared across several cpus, it must
 649          * have been locked.
 650          */
 651         ASSERT_PORTHASH_TOKEN_HELD(porthash);
 652
 653         /*
 654          * Best fit PCB lookup.
 655          *
 656          * First see if this local port is in use by looking on the
 657          * port hash list.
 658          */
 659         LIST_FOREACH(phd, porthash, phd_hash) {
 660                 if (phd->phd_port == lport)
 661                         break;
 662         }
 663         if (phd != NULL) {
 664                 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
 665 #ifdef INET6
 666                         if (!INP_ISIPV4(inp))
 667                                 continue;
 668 #endif
 669                         if (inp->inp_laddr.s_addr != INADDR_ANY &&
 670                             inp->inp_laddr.s_addr != laddr.s_addr)
 671                                 continue;
 672
 673                         if (inp->inp_faddr.s_addr != INADDR_ANY &&
 674                             inp->inp_faddr.s_addr != faddr.s_addr)
 675                                 continue;
 676
 677                         if (inp->inp_fport != 0 && inp->inp_fport != fport)
 678                                 continue;
 679
 680                         if (cred == NULL ||
 681                             cred->cr_prison ==
 682                             inp->inp_socket->so_cred->cr_prison) {
 683                                 match = inp;
 684                                 break;
 685                         }
 686                 }
 687         }
 688         return (match);
 689 }
 690
 691 static boolean_t
 692 in_pcbporthash_update4(struct inpcbportinfo *portinfo,
 693     struct inpcb *inp, u_short lport, const struct sockaddr_in *sin,
 694     struct ucred *cred)
 695 {
 696         struct inpcbporthead *porthash;
 697
 698         /*
 699          * This has to be atomic.  If the porthash is shared across multiple
 700          * protocol threads, e.g. tcp and udp, then the token must be held.
 701          */
 702         porthash = in_pcbporthash_head(portinfo, lport);
 703         GET_PORTHASH_TOKEN(porthash);
 704
 705         if (in_pcblookup_localremote(porthash, inp->inp_laddr,
 706             lport, sin->sin_addr, sin->sin_port, cred) != NULL) {
 707                 REL_PORTHASH_TOKEN(porthash);
 708                 return FALSE;
 709         }
 710         inp->inp_lport = lport;
 711         in_pcbinsporthash(porthash, inp);
 712
 713         REL_PORTHASH_TOKEN(porthash);
 714         return TRUE;
 715 }
 716
 717 int
 718 in_pcbbind_remote(struct inpcb *inp, const struct sockaddr *remote,
 719     struct thread *td)
 720 {
 721         struct proc *p = td->td_proc;
 722         u_short *lastport;
 723         const struct sockaddr_in *sin = (const struct sockaddr_in *)remote;
 724         struct sockaddr_in jsin;
 725         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 726         struct inpcbportinfo *portinfo;
 727         struct ucred *cred = NULL;
 728         u_short first, last, lport, step;
 729         int count, error, selfconn;
 730         int portinfo_first, portinfo_idx;
 731         int hash_cpu, hash_count, hash_count0;
 732         uint32_t hash_base = 0, hash;
 733
 734         if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */
 735                 return (EADDRNOTAVAIL);
 736
 737         KKASSERT(inp->inp_laddr.s_addr != INADDR_ANY);
 738         if (inp->inp_lport != 0)
 739                 return (EINVAL);        /* already bound */
 740
 741         KKASSERT(p);
 742         cred = p->p_ucred;
 743
 744         jsin.sin_family = AF_INET;
 745         jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 746         if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 747                 inp->inp_laddr.s_addr = INADDR_ANY;
 748                 return (EINVAL);
 749         }
 750         inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 751
 752         hash_count0 = ip_porthash_trycount;
 753         if (hash_count0 > 0) {
 754                 hash_base = toeplitz_piecemeal_addr(sin->sin_addr.s_addr) ^
 755                     toeplitz_piecemeal_addr(inp->inp_laddr.s_addr) ^
 756                     toeplitz_piecemeal_port(sin->sin_port);
 757         } else {
 758                 hash_count0 = 0;
 759         }
 760
 761         inp->inp_flags |= INP_ANONPORT;
 762
 763         step = pcbinfo->portinfo_mask + 1;
 764         portinfo_first = mycpuid & pcbinfo->portinfo_mask;
 765         portinfo_idx = portinfo_first;
 766 loop:
 767         hash_cpu = portinfo_idx & ncpus2_mask;
 768         portinfo = &pcbinfo->portinfo[portinfo_idx];
 769         selfconn = 0;
 770
 771         if (inp->inp_flags & INP_HIGHPORT) {
 772                 first = ipport_hifirstauto;     /* sysctl */
 773                 last  = ipport_hilastauto;
 774                 lastport = &portinfo->lasthi;
 775         } else if (inp->inp_flags & INP_LOWPORT) {
 776                 if (cred &&
 777                     (error =
 778                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 779                         inp->inp_laddr.s_addr = INADDR_ANY;
 780                         return (error);
 781                 }
 782                 first = ipport_lowfirstauto;    /* 1023 */
 783                 last  = ipport_lowlastauto;     /* 600 */
 784                 lastport = &portinfo->lastlow;
 785         } else {
 786                 first = ipport_firstauto;       /* sysctl */
 787                 last  = ipport_lastauto;
 788                 lastport = &portinfo->lastport;
 789         }
 790
 791         /* This could happen on loopback interface */
 792 #define IS_SELFCONNECT(inp, lport, sin)                 \
 793         (__predict_false((sin)->sin_port == (lport) &&  \
 794          (sin)->sin_addr.s_addr == (inp)->inp_laddr.s_addr))
 795
 796         /*
 797          * Simple check to ensure all ports are not used up causing
 798          * a deadlock here.
 799          *
 800          * We split the two cases (up and down) so that the direction
 801          * is not being tested on each round of the loop.
 802          */
 803         hash_count = hash_count0;
 804         if (first > last) {
 805                 /*
 806                  * counting down
 807                  */
 808                 in_pcbportrange(&first, &last, portinfo->offset, step);
 809                 count = ((first - last) / step) + hash_count;
 810
 811                 for (;;) {
 812                         if (count-- < 0) {      /* completely used? */
 813                                 error = EADDRNOTAVAIL;
 814                                 break;
 815                         }
 816
 817                         lport = in_pcblastport_down(lastport, first, last,
 818                             step);
 819                         KKASSERT((lport & pcbinfo->portinfo_mask) ==
 820                             portinfo->offset);
 821                         lport = htons(lport);
 822                         if (IS_SELFCONNECT(inp, lport, sin)) {
 823                                 if (!selfconn) {
 824                                         ++count; /* don't count this try */
 825                                         selfconn = 1;
 826                                 }
 827                                 continue;
 828                         }
 829
 830                         if (hash_count) {
 831                                 --hash_count;
 832                                 hash = hash_base ^
 833                                     toeplitz_piecemeal_port(lport);
 834                                 if ((hash & ncpus2_mask) != hash_cpu && hash_count)
 835                                         continue;
 836                         }
 837
 838                         if (in_pcbporthash_update4(portinfo,
 839                             inp, lport, sin, cred)) {
 840                                 error = 0;
 841                                 break;
 842                         }
 843                 }
 844         } else {
 845                 /*
 846                  * counting up
 847                  */
 848                 in_pcbportrange(&last, &first, portinfo->offset, step);
 849                 count = ((last - first) / step) + hash_count;
 850
 851                 for (;;) {
 852                         if (count-- < 0) {      /* completely used? */
 853                                 error = EADDRNOTAVAIL;
 854                                 break;
 855                         }
 856
 857                         lport = in_pcblastport_up(lastport, first, last, step);
 858                         KKASSERT((lport & pcbinfo->portinfo_mask) ==
 859                             portinfo->offset);
 860                         lport = htons(lport);
 861                         if (IS_SELFCONNECT(inp, lport, sin)) {
 862                                 if (!selfconn) {
 863                                         ++count; /* don't count this try */
 864                                         selfconn = 1;
 865                                 }
 866                                 continue;
 867                         }
 868
 869                         if (hash_count) {
 870                                 --hash_count;
 871                                 hash = hash_base ^
 872                                     toeplitz_piecemeal_port(lport);
 873                                 if ((hash & ncpus2_mask) != hash_cpu && hash_count)
 874                                         continue;
 875                         }
 876
 877                         if (in_pcbporthash_update4(portinfo,
 878                             inp, lport, sin, cred)) {
 879                                 error = 0;
 880                                 break;
 881                         }
 882                 }
 883         }
 884
 885 #undef IS_SELFCONNECT
 886
 887         if (error) {
 888                 /* Try next portinfo */
 889                 portinfo_idx++;
 890                 portinfo_idx &= pcbinfo->portinfo_mask;
 891                 if (portinfo_idx != portinfo_first)
 892                         goto loop;
 893                 inp->inp_laddr.s_addr = INADDR_ANY;
 894         }
 895         return error;
 896 }
 897
 898 /*
 899  *   Transform old in_pcbconnect() into an inner subroutine for new
 900  *   in_pcbconnect(): Do some validity-checking on the remote
 901  *   address (in mbuf 'nam') and then determine local host address
 902  *   (i.e., which interface) to use to access that remote host.
 903  *
 904  *   This preserves definition of in_pcbconnect(), while supporting a
 905  *   slightly different version for T/TCP.  (This is more than
 906  *   a bit of a kludge, but cleaning up the internal interfaces would
 907  *   have forced minor changes in every protocol).
 908  */
 909 int
 910 in_pcbladdr_find(struct inpcb *inp, struct sockaddr *nam,
 911     struct sockaddr_in **plocal_sin, struct thread *td, int find)
 912 {
 913         struct in_ifaddr *ia;
 914         struct ucred *cred = NULL;
 915         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 916         struct sockaddr *jsin;
 917         int jailed = 0, alloc_route = 0;
 918
 919         if (nam->sa_len != sizeof *sin)
 920                 return (EINVAL);
 921         if (sin->sin_family != AF_INET)
 922                 return (EAFNOSUPPORT);
 923         if (sin->sin_port == 0)
 924                 return (EADDRNOTAVAIL);
 925         if (td && td->td_proc && td->td_proc->p_ucred)
 926                 cred = td->td_proc->p_ucred;
 927         if (cred && cred->cr_prison)
 928                 jailed = 1;
 929         if (!TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) {
 930                 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia;
 931                 /*
 932                  * If the destination address is INADDR_ANY,
 933                  * use the primary local address.
 934                  * If the supplied address is INADDR_BROADCAST,
 935                  * and the primary interface supports broadcast,
 936                  * choose the broadcast address for that interface.
 937                  */
 938                 if (sin->sin_addr.s_addr == INADDR_ANY)
 939                         sin->sin_addr = IA_SIN(ia)->sin_addr;
 940                 else if (sin->sin_addr.s_addr == (u_long)INADDR_BROADCAST &&
 941                     (ia->ia_ifp->if_flags & IFF_BROADCAST))
 942                         sin->sin_addr = satosin(&ia->ia_broadaddr)->sin_addr;
 943         }
 944         if (find) {
 945                 struct route *ro;
 946
 947                 ia = NULL;
 948                 /*
 949                  * If route is known or can be allocated now,
 950                  * our src addr is taken from the i/f, else punt.
 951                  * Note that we should check the address family of the cached
 952                  * destination, in case of sharing the cache with IPv6.
 953                  */
 954                 ro = &inp->inp_route;
 955                 if (ro->ro_rt &&
 956                     (!(ro->ro_rt->rt_flags & RTF_UP) ||
 957                      ro->ro_dst.sa_family != AF_INET ||
 958                      satosin(&ro->ro_dst)->sin_addr.s_addr !=
 959                                       sin->sin_addr.s_addr ||
 960                      inp->inp_socket->so_options & SO_DONTROUTE)) {
 961                         RTFREE(ro->ro_rt);
 962                         ro->ro_rt = NULL;
 963                 }
 964                 if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/
 965                     (ro->ro_rt == NULL ||
 966                     ro->ro_rt->rt_ifp == NULL)) {
 967                         /* No route yet, so try to acquire one */
 968                         bzero(&ro->ro_dst, sizeof(struct sockaddr_in));
 969                         ro->ro_dst.sa_family = AF_INET;
 970                         ro->ro_dst.sa_len = sizeof(struct sockaddr_in);
 971                         ((struct sockaddr_in *) &ro->ro_dst)->sin_addr =
 972                                 sin->sin_addr;
 973                         rtalloc(ro);
 974                         alloc_route = 1;
 975                 }
 976                 /*
 977                  * If we found a route, use the address
 978                  * corresponding to the outgoing interface
 979                  * unless it is the loopback (in case a route
 980                  * to our address on another net goes to loopback).
 981                  */
 982                 if (ro->ro_rt &&
 983                     !(ro->ro_rt->rt_ifp->if_flags & IFF_LOOPBACK)) {
 984                         if (jailed) {
 985                                 if (jailed_ip(cred->cr_prison,
 986                                     ro->ro_rt->rt_ifa->ifa_addr)) {
 987                                         ia = ifatoia(ro->ro_rt->rt_ifa);
 988                                 }
 989                         } else {
 990                                 ia = ifatoia(ro->ro_rt->rt_ifa);
 991                         }
 992                 }
 993                 if (ia == NULL) {
 994                         u_short fport = sin->sin_port;
 995
 996                         sin->sin_port = 0;
 997                         ia = ifatoia(ifa_ifwithdstaddr(sintosa(sin)));
 998                         if (ia && jailed && !jailed_ip(cred->cr_prison,
 999                             sintosa(&ia->ia_addr)))
1000                                 ia = NULL;
1001                         if (ia == NULL)
1002                                 ia = ifatoia(ifa_ifwithnet(sintosa(sin)));
1003                         if (ia && jailed && !jailed_ip(cred->cr_prison,
1004                             sintosa(&ia->ia_addr)))
1005                                 ia = NULL;
1006                         sin->sin_port = fport;
1007                         if (ia == NULL &&
1008                             !TAILQ_EMPTY(&in_ifaddrheads[mycpuid]))
1009                                 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia;
1010                         if (ia && jailed && !jailed_ip(cred->cr_prison,
1011                             sintosa(&ia->ia_addr)))
1012                                 ia = NULL;
1013
1014                         if (!jailed && ia == NULL)
1015                                 goto fail;
1016                 }
1017                 /*
1018                  * If the destination address is multicast and an outgoing
1019                  * interface has been set as a multicast option, use the
1020                  * address of that interface as our source address.
1021                  */
1022                 if (!jailed && IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
1023                     inp->inp_moptions != NULL) {
1024                         struct ip_moptions *imo;
1025                         struct ifnet *ifp;
1026
1027                         imo = inp->inp_moptions;
1028                         if ((ifp = imo->imo_multicast_ifp) != NULL) {
1029                                 struct in_ifaddr_container *iac;
1030
1031                                 ia = NULL;
1032                                 TAILQ_FOREACH(iac,
1033                                 &in_ifaddrheads[mycpuid], ia_link) {
1034                                         if (iac->ia->ia_ifp == ifp) {
1035                                                 ia = iac->ia;
1036                                                 break;
1037                                         }
1038                                 }
1039                                 if (ia == NULL)
1040                                         goto fail;
1041                         }
1042                 }
1043                 /*
1044                  * Don't do pcblookup call here; return interface in plocal_sin
1045                  * and exit to caller, that will do the lookup.
1046                  */
1047                 if (ia == NULL && jailed) {
1048                         if ((jsin = prison_get_nonlocal(
1049                                 cred->cr_prison, AF_INET, NULL)) != NULL ||
1050                             (jsin = prison_get_local(
1051                                 cred->cr_prison, AF_INET, NULL)) != NULL) {
1052                                 *plocal_sin = satosin(jsin);
1053                         } else {
1054                                 /* IPv6 only Jail */
1055                                 goto fail;
1056                         }
1057                 } else {
1058                         *plocal_sin = &ia->ia_addr;
1059                 }
1060         }
1061         return (0);
1062 fail:
1063         if (alloc_route)
1064                 in_pcbresetroute(inp);
1065         return (EADDRNOTAVAIL);
1066 }
1067
1068 int
1069 in_pcbladdr(struct inpcb *inp, struct sockaddr *nam,
1070     struct sockaddr_in **plocal_sin, struct thread *td)
1071 {
1072         return in_pcbladdr_find(inp, nam, plocal_sin, td,
1073             (inp->inp_laddr.s_addr == INADDR_ANY));
1074 }
1075
1076 /*
1077  * Outer subroutine:
1078  * Connect from a socket to a specified address.
1079  * Both address and port must be specified in argument sin.
1080  * If don't have a local address for this socket yet,
1081  * then pick one.
1082  */
1083 int
1084 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
1085 {
1086         struct sockaddr_in *if_sin;
1087         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
1088         int error;
1089
1090         /* Call inner routine to assign local interface address. */
1091         if ((error = in_pcbladdr(inp, nam, &if_sin, td)) != 0)
1092                 return (error);
1093
1094         if (in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port,
1095                               inp->inp_laddr.s_addr ?
1096                                 inp->inp_laddr : if_sin->sin_addr,
1097                               inp->inp_lport, FALSE, NULL) != NULL) {
1098                 return (EADDRINUSE);
1099         }
1100         if (inp->inp_laddr.s_addr == INADDR_ANY) {
1101                 if (inp->inp_lport == 0) {
1102                         error = in_pcbbind(inp, NULL, td);
1103                         if (error)
1104                                 return (error);
1105                 }
1106                 inp->inp_laddr = if_sin->sin_addr;
1107         }
1108         inp->inp_faddr = sin->sin_addr;
1109         inp->inp_fport = sin->sin_port;
1110         in_pcbinsconnhash(inp);
1111         return (0);
1112 }
1113
1114 void
1115 in_pcbdisconnect(struct inpcb *inp)
1116 {
1117
1118         in_pcbremconnhash(inp);
1119         inp->inp_faddr.s_addr = INADDR_ANY;
1120         inp->inp_fport = 0;
1121 }
1122
1123 void
1124 in_pcbdetach(struct inpcb *inp)
1125 {
1126         struct socket *so = inp->inp_socket;
1127         struct inpcbinfo *ipi = inp->inp_pcbinfo;
1128
1129 #ifdef IPSEC
1130         ipsec4_delete_pcbpolicy(inp);
1131 #endif /*IPSEC*/
1132         inp->inp_gencnt = ++ipi->ipi_gencnt;
1133         KKASSERT((so->so_state & SS_ASSERTINPROG) == 0);
1134         in_pcbremlists(inp);
1135         so->so_pcb = NULL;
1136         sofree(so);                     /* remove pcb ref */
1137         if (inp->inp_options)
1138                 m_free(inp->inp_options);
1139         if (inp->inp_route.ro_rt)
1140                 rtfree(inp->inp_route.ro_rt);
1141         ip_freemoptions(inp->inp_moptions);
1142         kfree(inp, M_PCB);
1143 }
1144
1145 /*
1146  * The socket may have an invalid PCB, i.e. NULL.  For example, a TCP
1147  * socket received RST.
1148  */
1149 static int
1150 in_setsockaddr(struct socket *so, struct sockaddr **nam)
1151 {
1152         struct inpcb *inp;
1153         struct sockaddr_in *sin;
1154
1155         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
1156         inp = so->so_pcb;
1157         if (!inp)
1158                 return (ECONNRESET);
1159
1160         sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1161         sin->sin_family = AF_INET;
1162         sin->sin_len = sizeof *sin;
1163         sin->sin_port = inp->inp_lport;
1164         sin->sin_addr = inp->inp_laddr;
1165
1166         *nam = (struct sockaddr *)sin;
1167         return (0);
1168 }
1169
1170 void
1171 in_setsockaddr_dispatch(netmsg_t msg)
1172 {
1173         int error;
1174
1175         error = in_setsockaddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1176         lwkt_replymsg(&msg->lmsg, error);
1177 }
1178
1179 /*
1180  * The socket may have an invalid PCB, i.e. NULL.  For example, a TCP
1181  * socket received RST.
1182  */
1183 int
1184 in_setpeeraddr(struct socket *so, struct sockaddr **nam)
1185 {
1186         struct inpcb *inp;
1187         struct sockaddr_in *sin;
1188
1189         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
1190         inp = so->so_pcb;
1191         if (!inp)
1192                 return (ECONNRESET);
1193
1194         sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1195         sin->sin_family = AF_INET;
1196         sin->sin_len = sizeof *sin;
1197         sin->sin_port = inp->inp_fport;
1198         sin->sin_addr = inp->inp_faddr;
1199
1200         *nam = (struct sockaddr *)sin;
1201         return (0);
1202 }
1203
1204 void
1205 in_setpeeraddr_dispatch(netmsg_t msg)
1206 {
1207         int error;
1208
1209         error = in_setpeeraddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1210         lwkt_replymsg(&msg->lmsg, error);
1211 }
1212
1213 void
1214 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int err,
1215     inp_notify_t notify)
1216 {
1217         struct inpcb *inp, *marker;
1218
1219         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1220             ("not in the correct netisr"));
1221         marker = &in_pcbmarkers[mycpuid];
1222
1223         /*
1224          * NOTE:
1225          * - If INP_PLACEMARKER is set we must ignore the rest of the
1226          *   structure and skip it.
1227          * - It is safe to nuke inpcbs here, since we are in their own
1228          *   netisr.
1229          */
1230         GET_PCBINFO_TOKEN(pcbinfo);
1231
1232         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1233         while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1234                 LIST_REMOVE(marker, inp_list);
1235                 LIST_INSERT_AFTER(inp, marker, inp_list);
1236
1237                 if (inp->inp_flags & INP_PLACEMARKER)
1238                         continue;
1239 #ifdef INET6
1240                 if (!INP_ISIPV4(inp))
1241                         continue;
1242 #endif
1243                 if (inp->inp_faddr.s_addr != faddr.s_addr ||
1244                     inp->inp_socket == NULL)
1245                         continue;
1246                 (*notify)(inp, err);            /* can remove inp from list! */
1247         }
1248         LIST_REMOVE(marker, inp_list);
1249
1250         REL_PCBINFO_TOKEN(pcbinfo);
1251 }
1252
1253 void
1254 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1255 {
1256         struct inpcb *inp, *marker;
1257
1258         /*
1259          * We only need to make sure that we are in netisr0, where all
1260          * multicast operation happen.  We could check inpcbinfo which
1261          * does not belong to netisr0 by holding the inpcbinfo's token.
1262          * In this case, the pcbinfo must be able to be shared, i.e.
1263          * pcbinfo->infotoken is not NULL.
1264          */
1265         ASSERT_IN_NETISR(0);
1266         KASSERT(pcbinfo->cpu == 0 || pcbinfo->infotoken != NULL,
1267             ("pcbinfo could not be shared"));
1268
1269         /*
1270          * Get a marker for the current netisr (netisr0).
1271          *
1272          * It is possible that the multicast address deletion blocks,
1273          * which could cause temporary token releasing.  So we use
1274          * inpcb marker here to get a coherent view of the inpcb list.
1275          *
1276          * While, on the other hand, moptions are only added and deleted
1277          * in netisr0, so we would not see staled moption or miss moption
1278          * even if the token was released due to the blocking multicast
1279          * address deletion.
1280          */
1281         marker = &in_pcbmarkers[mycpuid];
1282
1283         GET_PCBINFO_TOKEN(pcbinfo);
1284
1285         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1286         while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1287                 struct ip_moptions *imo;
1288
1289                 LIST_REMOVE(marker, inp_list);
1290                 LIST_INSERT_AFTER(inp, marker, inp_list);
1291
1292                 if (inp->inp_flags & INP_PLACEMARKER)
1293                         continue;
1294                 imo = inp->inp_moptions;
1295                 if (INP_ISIPV4(inp) && imo != NULL) {
1296                         int i, gap;
1297
1298                         /*
1299                          * Unselect the outgoing interface if it is being
1300                          * detached.
1301                          */
1302                         if (imo->imo_multicast_ifp == ifp)
1303                                 imo->imo_multicast_ifp = NULL;
1304
1305                         /*
1306                          * Drop multicast group membership if we joined
1307                          * through the interface being detached.
1308                          */
1309                         for (i = 0, gap = 0; i < imo->imo_num_memberships;
1310                             i++) {
1311                                 if (imo->imo_membership[i]->inm_ifp == ifp) {
1312                                         /*
1313                                          * NOTE:
1314                                          * This could block and the pcbinfo
1315                                          * token could be passively released.
1316                                          */
1317                                         in_delmulti(imo->imo_membership[i]);
1318                                         gap++;
1319                                 } else if (gap != 0)
1320                                         imo->imo_membership[i - gap] =
1321                                             imo->imo_membership[i];
1322                         }
1323                         imo->imo_num_memberships -= gap;
1324                 }
1325         }
1326         LIST_REMOVE(marker, inp_list);
1327
1328         REL_PCBINFO_TOKEN(pcbinfo);
1329 }
1330
1331 /*
1332  * Check for alternatives when higher level complains
1333  * about service problems.  For now, invalidate cached
1334  * routing information.  If the route was created dynamically
1335  * (by a redirect), time to try a default gateway again.
1336  */
1337 void
1338 in_losing(struct inpcb *inp)
1339 {
1340         struct rtentry *rt;
1341         struct rt_addrinfo rtinfo;
1342
1343         if ((rt = inp->inp_route.ro_rt)) {
1344                 bzero(&rtinfo, sizeof(struct rt_addrinfo));
1345                 rtinfo.rti_info[RTAX_DST] = rt_key(rt);
1346                 rtinfo.rti_info[RTAX_GATEWAY] = rt->rt_gateway;
1347                 rtinfo.rti_info[RTAX_NETMASK] = rt_mask(rt);
1348                 rtinfo.rti_flags = rt->rt_flags;
1349                 rt_missmsg(RTM_LOSING, &rtinfo, rt->rt_flags, 0);
1350                 if (rt->rt_flags & RTF_DYNAMIC) {
1351                         rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway,
1352                             rt_mask(rt), rt->rt_flags, NULL);
1353                 }
1354                 inp->inp_route.ro_rt = NULL;
1355                 rtfree(rt);
1356                 /*
1357                  * A new route can be allocated
1358                  * the next time output is attempted.
1359                  */
1360         }
1361 }
1362
1363 /*
1364  * After a routing change, flush old routing
1365  * and allocate a (hopefully) better one.
1366  */
1367 void
1368 in_rtchange(struct inpcb *inp, int err)
1369 {
1370         if (inp->inp_route.ro_rt) {
1371                 rtfree(inp->inp_route.ro_rt);
1372                 inp->inp_route.ro_rt = NULL;
1373                 /*
1374                  * A new route can be allocated the next time
1375                  * output is attempted.
1376                  */
1377         }
1378 }
1379
1380 /*
1381  * Lookup a PCB based on the local address and port.
1382  */
1383 static struct inpcb *
1384 in_pcblookup_local(struct inpcbporthead *porthash, struct in_addr laddr,
1385                    u_int lport_arg, int wild_okay, struct ucred *cred)
1386 {
1387         struct inpcb *inp;
1388         int matchwild = 3, wildcard;
1389         u_short lport = lport_arg;
1390         struct inpcbport *phd;
1391         struct inpcb *match = NULL;
1392
1393         /*
1394          * If the porthashbase is shared across several cpus, it must
1395          * have been locked.
1396          */
1397         ASSERT_PORTHASH_TOKEN_HELD(porthash);
1398
1399         /*
1400          * Best fit PCB lookup.
1401          *
1402          * First see if this local port is in use by looking on the
1403          * port hash list.
1404          */
1405         LIST_FOREACH(phd, porthash, phd_hash) {
1406                 if (phd->phd_port == lport)
1407                         break;
1408         }
1409         if (phd != NULL) {
1410                 /*
1411                  * Port is in use by one or more PCBs. Look for best
1412                  * fit.
1413                  */
1414                 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
1415                         wildcard = 0;
1416 #ifdef INET6
1417                         if (!INP_ISIPV4(inp))
1418                                 continue;
1419 #endif
1420                         if (inp->inp_faddr.s_addr != INADDR_ANY)
1421                                 wildcard++;
1422                         if (inp->inp_laddr.s_addr != INADDR_ANY) {
1423                                 if (laddr.s_addr == INADDR_ANY)
1424                                         wildcard++;
1425                                 else if (inp->inp_laddr.s_addr != laddr.s_addr)
1426                                         continue;
1427                         } else {
1428                                 if (laddr.s_addr != INADDR_ANY)
1429                                         wildcard++;
1430                         }
1431                         if (wildcard && !wild_okay)
1432                                 continue;
1433                         if (wildcard < matchwild &&
1434                             (cred == NULL ||
1435                              cred->cr_prison ==
1436                                         inp->inp_socket->so_cred->cr_prison)) {
1437                                 match = inp;
1438                                 matchwild = wildcard;
1439                                 if (matchwild == 0) {
1440                                         break;
1441                                 }
1442                         }
1443                 }
1444         }
1445         return (match);
1446 }
1447
1448 struct inpcb *
1449 in_pcblocalgroup_last(const struct inpcbinfo *pcbinfo,
1450     const struct inpcb *inp)
1451 {
1452         const struct inp_localgrphead *hdr;
1453         const struct inp_localgroup *grp;
1454         int i;
1455
1456         if (pcbinfo->localgrphashbase == NULL)
1457                 return NULL;
1458
1459         GET_PCBINFO_TOKEN(pcbinfo);
1460
1461         hdr = &pcbinfo->localgrphashbase[
1462             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
1463
1464         LIST_FOREACH(grp, hdr, il_list) {
1465                 if (grp->il_af == inp->inp_af &&
1466                     grp->il_lport == inp->inp_lport &&
1467                     memcmp(&grp->il_dependladdr,
1468                         &inp->inp_inc.inc_ie.ie_dependladdr,
1469                         sizeof(grp->il_dependladdr)) == 0) {
1470                         break;
1471                 }
1472         }
1473         if (grp == NULL || grp->il_inpcnt == 1) {
1474                 REL_PCBINFO_TOKEN(pcbinfo);
1475                 return NULL;
1476         }
1477
1478         KASSERT(grp->il_inpcnt >= 2,
1479             ("invalid localgroup inp count %d", grp->il_inpcnt));
1480         for (i = 0; i < grp->il_inpcnt; ++i) {
1481                 if (grp->il_inp[i] == inp) {
1482                         int last = grp->il_inpcnt - 1;
1483
1484                         if (i == last)
1485                                 last = grp->il_inpcnt - 2;
1486                         REL_PCBINFO_TOKEN(pcbinfo);
1487                         return grp->il_inp[last];
1488                 }
1489         }
1490         REL_PCBINFO_TOKEN(pcbinfo);
1491         return NULL;
1492 }
1493
1494 static struct inpcb *
1495 inp_localgroup_lookup(const struct inpcbinfo *pcbinfo,
1496     struct in_addr laddr, uint16_t lport, uint32_t pkt_hash)
1497 {
1498         struct inpcb *local_wild = NULL;
1499         const struct inp_localgrphead *hdr;
1500         const struct inp_localgroup *grp;
1501
1502         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1503
1504         hdr = &pcbinfo->localgrphashbase[
1505             INP_PCBLOCALGRPHASH(lport, pcbinfo->localgrphashmask)];
1506
1507         /*
1508          * Order of socket selection:
1509          * 1. non-wild.
1510          * 2. wild.
1511          *
1512          * NOTE: Local group does not contain jailed sockets
1513          */
1514         LIST_FOREACH(grp, hdr, il_list) {
1515 #ifdef INET6
1516                 if (grp->il_af != AF_INET)
1517                         continue;
1518 #endif
1519                 if (grp->il_lport == lport) {
1520                         int idx;
1521
1522                         /*
1523                          * Modulo-N is used here, which greatly reduces
1524                          * completion queue token contention, thus more
1525                          * cpu time is saved.
1526                          */
1527                         idx = pkt_hash % grp->il_inpcnt;
1528                         if (grp->il_laddr.s_addr == laddr.s_addr)
1529                                 return grp->il_inp[idx];
1530                         else if (grp->il_laddr.s_addr == INADDR_ANY)
1531                                 local_wild = grp->il_inp[idx];
1532                 }
1533         }
1534         if (local_wild != NULL)
1535                 return local_wild;
1536         return NULL;
1537 }
1538
1539 /*
1540  * Lookup PCB in hash list.
1541  */
1542 struct inpcb *
1543 in_pcblookup_pkthash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1544     u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1545     boolean_t wildcard, struct ifnet *ifp, const struct mbuf *m)
1546 {
1547         struct inpcbhead *head;
1548         struct inpcb *inp, *jinp=NULL;
1549         u_short fport = fport_arg, lport = lport_arg;
1550
1551         /*
1552          * First look for an exact match.
1553          */
1554         head = &pcbinfo->hashbase[INP_PCBCONNHASH(faddr.s_addr, fport,
1555             laddr.s_addr, lport, pcbinfo->hashmask)];
1556         LIST_FOREACH(inp, head, inp_hash) {
1557 #ifdef INET6
1558                 if (!INP_ISIPV4(inp))
1559                         continue;
1560 #endif
1561                 if (in_hosteq(inp->inp_faddr, faddr) &&
1562                     in_hosteq(inp->inp_laddr, laddr) &&
1563                     inp->inp_fport == fport && inp->inp_lport == lport) {
1564                         /* found */
1565                         if (inp->inp_socket == NULL ||
1566                             inp->inp_socket->so_cred->cr_prison == NULL) {
1567                                 return (inp);
1568                         } else {
1569                                 if  (jinp == NULL)
1570                                         jinp = inp;
1571                         }
1572                 }
1573         }
1574         if (jinp != NULL)
1575                 return (jinp);
1576
1577         if (wildcard) {
1578                 struct inpcb *local_wild = NULL;
1579                 struct inpcb *jinp_wild = NULL;
1580                 struct inpcontainer *ic;
1581                 struct inpcontainerhead *chead;
1582                 struct sockaddr_in jsin;
1583                 struct ucred *cred;
1584
1585                 GET_PCBINFO_TOKEN(pcbinfo);
1586
1587                 /*
1588                  * Check local group first
1589                  */
1590                 if (pcbinfo->localgrphashbase != NULL &&
1591                     m != NULL && (m->m_flags & M_HASH) &&
1592                     !(ifp && ifp->if_type == IFT_FAITH)) {
1593                         inp = inp_localgroup_lookup(pcbinfo,
1594                             laddr, lport, m->m_pkthdr.hash);
1595                         if (inp != NULL) {
1596                                 REL_PCBINFO_TOKEN(pcbinfo);
1597                                 return inp;
1598                         }
1599                 }
1600
1601                 /*
1602                  * Order of socket selection:
1603                  * 1. non-jailed, non-wild.
1604                  * 2. non-jailed, wild.
1605                  * 3. jailed, non-wild.
1606                  * 4. jailed, wild.
1607                  */
1608                 jsin.sin_family = AF_INET;
1609                 chead = &pcbinfo->wildcardhashbase[
1610                     INP_PCBWILDCARDHASH(lport, pcbinfo->wildcardhashmask)];
1611                 LIST_FOREACH(ic, chead, ic_list) {
1612                         inp = ic->ic_inp;
1613                         if (inp->inp_flags & INP_PLACEMARKER)
1614                                 continue;
1615
1616                         jsin.sin_addr.s_addr = laddr.s_addr;
1617 #ifdef INET6
1618                         if (!INP_ISIPV4(inp))
1619                                 continue;
1620 #endif
1621                         if (inp->inp_socket != NULL)
1622                                 cred = inp->inp_socket->so_cred;
1623                         else
1624                                 cred = NULL;
1625                         if (cred != NULL && jailed(cred)) {
1626                                 if (jinp != NULL)
1627                                         continue;
1628                                 else
1629                                         if (!jailed_ip(cred->cr_prison,
1630                                             (struct sockaddr *)&jsin))
1631                                                 continue;
1632                         }
1633                         if (inp->inp_lport == lport) {
1634                                 if (ifp && ifp->if_type == IFT_FAITH &&
1635                                     !(inp->inp_flags & INP_FAITH))
1636                                         continue;
1637                                 if (inp->inp_laddr.s_addr == laddr.s_addr) {
1638                                         if (cred != NULL && jailed(cred)) {
1639                                                 jinp = inp;
1640                                         } else {
1641                                                 REL_PCBINFO_TOKEN(pcbinfo);
1642                                                 return (inp);
1643                                         }
1644                                 }
1645                                 if (inp->inp_laddr.s_addr == INADDR_ANY) {
1646                                         if (cred != NULL && jailed(cred))
1647                                                 jinp_wild = inp;
1648                                         else
1649                                                 local_wild = inp;
1650                                 }
1651                         }
1652                 }
1653
1654                 REL_PCBINFO_TOKEN(pcbinfo);
1655
1656                 if (local_wild != NULL)
1657                         return (local_wild);
1658                 if (jinp != NULL)
1659                         return (jinp);
1660                 return (jinp_wild);
1661         }
1662
1663         /*
1664          * Not found.
1665          */
1666         return (NULL);
1667 }
1668
1669 struct inpcb *
1670 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1671     u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1672     boolean_t wildcard, struct ifnet *ifp)
1673 {
1674         return in_pcblookup_pkthash(pcbinfo, faddr, fport_arg,
1675             laddr, lport_arg, wildcard, ifp, NULL);
1676 }
1677
1678 /*
1679  * Insert PCB into connection hash table.
1680  */
1681 void
1682 in_pcbinsconnhash(struct inpcb *inp)
1683 {
1684         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1685         struct inpcbhead *bucket;
1686         u_int32_t hashkey_faddr, hashkey_laddr;
1687
1688 #ifdef INET6
1689         if (INP_ISIPV6(inp)) {
1690                 hashkey_faddr = inp->in6p_faddr.s6_addr32[3] /* XXX JH */;
1691                 hashkey_laddr = inp->in6p_laddr.s6_addr32[3] /* XXX JH */;
1692         } else {
1693 #endif
1694                 hashkey_faddr = inp->inp_faddr.s_addr;
1695                 hashkey_laddr = inp->inp_laddr.s_addr;
1696 #ifdef INET6
1697         }
1698 #endif
1699
1700         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1701             ("not in the correct netisr"));
1702         ASSERT_INP_NOTINHASH(inp);
1703         inp->inp_flags |= INP_CONNECTED;
1704
1705         /*
1706          * Insert into the connection hash table.
1707          */
1708         bucket = &pcbinfo->hashbase[INP_PCBCONNHASH(hashkey_faddr,
1709             inp->inp_fport, hashkey_laddr, inp->inp_lport, pcbinfo->hashmask)];
1710         LIST_INSERT_HEAD(bucket, inp, inp_hash);
1711 }
1712
1713 /*
1714  * Remove PCB from connection hash table.
1715  */
1716 void
1717 in_pcbremconnhash(struct inpcb *inp)
1718 {
1719         struct inpcbinfo *pcbinfo __debugvar = inp->inp_pcbinfo;
1720
1721         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1722             ("not in the correct netisr"));
1723         KASSERT(inp->inp_flags & INP_CONNECTED, ("inp not connected"));
1724
1725         LIST_REMOVE(inp, inp_hash);
1726         inp->inp_flags &= ~INP_CONNECTED;
1727 }
1728
1729 /*
1730  * Insert PCB into port hash table.
1731  */
1732 void
1733 in_pcbinsporthash(struct inpcbporthead *pcbporthash, struct inpcb *inp)
1734 {
1735         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1736         struct inpcbport *phd;
1737
1738         /*
1739          * If the porthashbase is shared across several cpus, it must
1740          * have been locked.
1741          */
1742         ASSERT_PORTHASH_TOKEN_HELD(pcbporthash);
1743
1744         /*
1745          * Insert into the port hash table.
1746          */
1747
1748         /* Go through port list and look for a head for this lport. */
1749         LIST_FOREACH(phd, pcbporthash, phd_hash) {
1750                 if (phd->phd_port == inp->inp_lport)
1751                         break;
1752         }
1753
1754         /* If none exists, use saved one and tack it on. */
1755         if (phd == NULL) {
1756                 KKASSERT(pcbinfo->portsave != NULL);
1757                 phd = pcbinfo->portsave;
1758                 pcbinfo->portsave = NULL;
1759                 phd->phd_port = inp->inp_lport;
1760                 LIST_INIT(&phd->phd_pcblist);
1761                 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
1762         }
1763
1764         inp->inp_porthash = pcbporthash;
1765         inp->inp_phd = phd;
1766         LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
1767
1768         /*
1769          * Malloc one inpcbport for later use.  It is safe to use
1770          * "wait" malloc here (port token would be released, if
1771          * malloc ever blocked), since all changes to the porthash
1772          * are done.
1773          */
1774         if (pcbinfo->portsave == NULL) {
1775                 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave),
1776                                             M_PCB, M_INTWAIT | M_ZERO);
1777         }
1778 }
1779
1780 void
1781 in_pcbinsporthash_lport(struct inpcb *inp)
1782 {
1783         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1784         struct inpcbportinfo *portinfo;
1785         struct inpcbporthead *porthash;
1786         u_short lport_ho;
1787
1788         /* Locate the proper portinfo based on lport */
1789         lport_ho = ntohs(inp->inp_lport);
1790         portinfo = &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask];
1791         KKASSERT((lport_ho & pcbinfo->portinfo_mask) == portinfo->offset);
1792
1793         porthash = in_pcbporthash_head(portinfo, inp->inp_lport);
1794         GET_PORTHASH_TOKEN(porthash);
1795         in_pcbinsporthash(porthash, inp);
1796         REL_PORTHASH_TOKEN(porthash);
1797 }
1798
1799 void
1800 in_pcbremporthash(struct inpcb *inp)
1801 {
1802         struct inpcbporthead *porthash;
1803         struct inpcbport *phd;
1804
1805         if (inp->inp_phd == NULL)
1806                 return;
1807         KASSERT(inp->inp_lport != 0, ("inpcb has no lport"));
1808
1809         porthash = inp->inp_porthash;
1810         GET_PORTHASH_TOKEN(porthash);
1811
1812         phd = inp->inp_phd;
1813         LIST_REMOVE(inp, inp_portlist);
1814         if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
1815                 LIST_REMOVE(phd, phd_hash);
1816                 kfree(phd, M_PCB);
1817         }
1818
1819         REL_PORTHASH_TOKEN(porthash);
1820
1821         inp->inp_phd = NULL;
1822         /* NOTE: Don't whack inp_lport, which may be used later */
1823 }
1824
1825 static struct inp_localgroup *
1826 inp_localgroup_alloc(u_char af, uint16_t port,
1827     const union in_dependaddr *addr, int size)
1828 {
1829         struct inp_localgroup *grp;
1830
1831         grp = kmalloc(__offsetof(struct inp_localgroup, il_inp[size]),
1832             M_TEMP, M_INTWAIT | M_ZERO);
1833         grp->il_af = af;
1834         grp->il_lport = port;
1835         grp->il_dependladdr = *addr;
1836         grp->il_inpsiz = size;
1837
1838         return grp;
1839 }
1840
1841 static void
1842 inp_localgroup_free(struct inp_localgroup *grp)
1843 {
1844         kfree(grp, M_TEMP);
1845 }
1846
1847 static void
1848 inp_localgroup_destroy(struct inp_localgroup *grp)
1849 {
1850         LIST_REMOVE(grp, il_list);
1851         inp_localgroup_free(grp);
1852 }
1853
1854 static void
1855 inp_localgroup_copy(struct inp_localgroup *grp,
1856     const struct inp_localgroup *old_grp)
1857 {
1858         int i;
1859
1860         KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
1861             ("invalid new local group size %d and old local group count %d",
1862              grp->il_inpsiz, old_grp->il_inpcnt));
1863         for (i = 0; i < old_grp->il_inpcnt; ++i)
1864                 grp->il_inp[i] = old_grp->il_inp[i];
1865         grp->il_inpcnt = old_grp->il_inpcnt;
1866 }
1867
1868 static void
1869 in_pcbinslocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
1870 {
1871         struct inp_localgrphead *hdr;
1872         struct inp_localgroup *grp, *grp_alloc = NULL;
1873         struct ucred *cred;
1874         int i, idx;
1875
1876         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1877
1878         if (pcbinfo->localgrphashbase == NULL)
1879                 return;
1880
1881         /*
1882          * XXX don't allow jailed socket to join local group
1883          */
1884         if (inp->inp_socket != NULL)
1885                 cred = inp->inp_socket->so_cred;
1886         else
1887                 cred = NULL;
1888         if (cred != NULL && jailed(cred))
1889                 return;
1890
1891         hdr = &pcbinfo->localgrphashbase[
1892             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
1893
1894 again:
1895         LIST_FOREACH(grp, hdr, il_list) {
1896                 if (grp->il_af == inp->inp_af &&
1897                     grp->il_lport == inp->inp_lport &&
1898                     memcmp(&grp->il_dependladdr,
1899                         &inp->inp_inc.inc_ie.ie_dependladdr,
1900                         sizeof(grp->il_dependladdr)) == 0) {
1901                         break;
1902                 }
1903         }
1904         if (grp == NULL) {
1905                 /*
1906                  * Create a new local group
1907                  */
1908                 if (grp_alloc == NULL) {
1909                         grp_alloc = inp_localgroup_alloc(inp->inp_af,
1910                             inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
1911                             INP_LOCALGROUP_SIZMIN);
1912                         /*
1913                          * Local group allocation could block and the
1914                          * local group w/ the same property might have
1915                          * been added by others when we were blocked;
1916                          * check again.
1917                          */
1918                         goto again;
1919                 } else {
1920                         /* Local group has been allocated; link it */
1921                         grp = grp_alloc;
1922                         grp_alloc = NULL;
1923                         LIST_INSERT_HEAD(hdr, grp, il_list);
1924                 }
1925         } else if (grp->il_inpcnt == grp->il_inpsiz) {
1926                 if (grp->il_inpsiz >= INP_LOCALGROUP_SIZMAX) {
1927                         static int limit_logged = 0;
1928
1929                         if (!limit_logged) {
1930                                 limit_logged = 1;
1931                                 kprintf("local group port %d, "
1932                                     "limit reached\n", ntohs(grp->il_lport));
1933                         }
1934                         if (grp_alloc != NULL) {
1935                                 /*
1936                                  * This would happen if the local group
1937                                  * w/ the same property was expanded when
1938                                  * our local group allocation blocked.
1939                                  */
1940                                 inp_localgroup_free(grp_alloc);
1941                         }
1942                         return;
1943                 }
1944
1945                 /*
1946                  * Expand this local group
1947                  */
1948                 if (grp_alloc == NULL ||
1949                     grp->il_inpcnt >= grp_alloc->il_inpsiz) {
1950                         if (grp_alloc != NULL)
1951                                 inp_localgroup_free(grp_alloc);
1952                         grp_alloc = inp_localgroup_alloc(grp->il_af,
1953                             grp->il_lport, &grp->il_dependladdr,
1954                             grp->il_inpsiz * 2);
1955                         /*
1956                          * Local group allocation could block and the
1957                          * local group w/ the same property might have
1958                          * been expanded by others when we were blocked;
1959                          * check again.
1960                          */
1961                         goto again;
1962                 }
1963
1964                 /*
1965                  * Save the old local group, link the new one, and then
1966                  * destroy the old local group
1967                  */
1968                 inp_localgroup_copy(grp_alloc, grp);
1969                 LIST_INSERT_HEAD(hdr, grp_alloc, il_list);
1970                 inp_localgroup_destroy(grp);
1971
1972                 grp = grp_alloc;
1973                 grp_alloc = NULL;
1974         } else {
1975                 /*
1976                  * Found the local group
1977                  */
1978                 if (grp_alloc != NULL) {
1979                         /*
1980                          * This would happen if the local group w/ the
1981                          * same property was added or expanded when our
1982                          * local group allocation blocked.
1983                          */
1984                         inp_localgroup_free(grp_alloc);
1985                         grp_alloc = NULL;
1986                 }
1987         }
1988
1989         KASSERT(grp->il_inpcnt < grp->il_inpsiz,
1990             ("invalid local group size %d and count %d",
1991              grp->il_inpsiz, grp->il_inpcnt));
1992
1993         /*
1994          * Keep the local group sorted by the inpcb local group index
1995          * in ascending order.
1996          *
1997          * This eases the multi-process userland application which uses
1998          * SO_REUSEPORT sockets and binds process to the owner cpu of
1999          * the SO_REUSEPORT socket:
2000          * If we didn't sort the local group by the inpcb local group
2001          * index and one of the process owning an inpcb in this local
2002          * group restarted, e.g. crashed and restarted by watchdog,
2003          * other processes owning a inpcb in this local group would have
2004          * to detect that event, refetch its socket's owner cpu, and
2005          * re-bind.
2006          */
2007         idx = grp->il_inpcnt;
2008         for (i = 0; i < idx; ++i) {
2009                 struct inpcb *oinp = grp->il_inp[i];
2010
2011                 if (oinp->inp_lgrpindex > i) {
2012                         if (inp->inp_lgrpindex < 0) {
2013                                 inp->inp_lgrpindex = i;
2014                         } else if (inp->inp_lgrpindex != i) {
2015                                 if (bootverbose) {
2016                                         kprintf("inp %p: grpidx %d, "
2017                                             "assigned to %d, cpu%d\n",
2018                                             inp, inp->inp_lgrpindex, i,
2019                                             mycpuid);
2020                                 }
2021                         }
2022                         grp->il_inp[i] = inp;
2023
2024                         /* Pull down inpcbs */
2025                         for (; i < grp->il_inpcnt; ++i) {
2026                                 struct inpcb *oinp1 = grp->il_inp[i + 1];
2027
2028                                 grp->il_inp[i + 1] = oinp;
2029                                 oinp = oinp1;
2030                         }
2031                         grp->il_inpcnt++;
2032                         return;
2033                 }
2034         }
2035
2036         if (inp->inp_lgrpindex < 0) {
2037                 inp->inp_lgrpindex = idx;
2038         } else if (inp->inp_lgrpindex != idx) {
2039                 if (bootverbose) {
2040                         kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n",
2041                             inp, inp->inp_lgrpindex, idx, mycpuid);
2042                 }
2043         }
2044         grp->il_inp[idx] = inp;
2045         grp->il_inpcnt++;
2046 }
2047
2048 void
2049 in_pcbinswildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2050 {
2051         struct inpcontainer *ic;
2052         struct inpcontainerhead *bucket;
2053
2054         GET_PCBINFO_TOKEN(pcbinfo);
2055
2056         in_pcbinslocalgrphash_oncpu(inp, pcbinfo);
2057
2058         bucket = &pcbinfo->wildcardhashbase[
2059             INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
2060
2061         ic = kmalloc(sizeof(struct inpcontainer), M_TEMP, M_INTWAIT);
2062         ic->ic_inp = inp;
2063         LIST_INSERT_HEAD(bucket, ic, ic_list);
2064
2065         REL_PCBINFO_TOKEN(pcbinfo);
2066 }
2067
2068 /*
2069  * Insert PCB into wildcard hash table.
2070  */
2071 void
2072 in_pcbinswildcardhash(struct inpcb *inp)
2073 {
2074         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2075
2076         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
2077             ("not in correct netisr"));
2078         ASSERT_INP_NOTINHASH(inp);
2079         inp->inp_flags |= INP_WILDCARD;
2080
2081         in_pcbinswildcardhash_oncpu(inp, pcbinfo);
2082 }
2083
2084 static void
2085 in_pcbremlocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2086 {
2087         struct inp_localgrphead *hdr;
2088         struct inp_localgroup *grp;
2089
2090         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
2091
2092         if (pcbinfo->localgrphashbase == NULL)
2093                 return;
2094
2095         hdr = &pcbinfo->localgrphashbase[
2096             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
2097
2098         LIST_FOREACH(grp, hdr, il_list) {
2099                 int i;
2100
2101                 for (i = 0; i < grp->il_inpcnt; ++i) {
2102                         if (grp->il_inp[i] != inp)
2103                                 continue;
2104
2105                         if (grp->il_inpcnt == 1) {
2106                                 /* Destroy this local group */
2107                                 inp_localgroup_destroy(grp);
2108                         } else {
2109                                 /* Pull up inpcbs */
2110                                 for (; i + 1 < grp->il_inpcnt; ++i)
2111                                         grp->il_inp[i] = grp->il_inp[i + 1];
2112                                 grp->il_inpcnt--;
2113                         }
2114                         return;
2115                 }
2116         }
2117 }
2118
2119 void
2120 in_pcbremwildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2121 {
2122         struct inpcontainer *ic;
2123         struct inpcontainerhead *head;
2124
2125         GET_PCBINFO_TOKEN(pcbinfo);
2126
2127         in_pcbremlocalgrphash_oncpu(inp, pcbinfo);
2128
2129         /* find bucket */
2130         head = &pcbinfo->wildcardhashbase[
2131             INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
2132
2133         LIST_FOREACH(ic, head, ic_list) {
2134                 if (ic->ic_inp == inp)
2135                         goto found;
2136         }
2137         REL_PCBINFO_TOKEN(pcbinfo);
2138         return;                 /* not found! */
2139
2140 found:
2141         LIST_REMOVE(ic, ic_list);       /* remove container from bucket chain */
2142         REL_PCBINFO_TOKEN(pcbinfo);
2143         kfree(ic, M_TEMP);              /* deallocate container */
2144 }
2145
2146 /*
2147  * Remove PCB from wildcard hash table.
2148  */
2149 void
2150 in_pcbremwildcardhash(struct inpcb *inp)
2151 {
2152         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2153
2154         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
2155             ("not in correct netisr"));
2156         KASSERT(inp->inp_flags & INP_WILDCARD, ("inp not wildcard"));
2157
2158         in_pcbremwildcardhash_oncpu(inp, pcbinfo);
2159         inp->inp_lgrpindex = -1;
2160         inp->inp_flags &= ~INP_WILDCARD;
2161 }
2162
2163 /*
2164  * Remove PCB from various lists.
2165  */
2166 void
2167 in_pcbremlists(struct inpcb *inp)
2168 {
2169         in_pcbremporthash(inp);
2170         if (inp->inp_flags & INP_WILDCARD) {
2171                 in_pcbremwildcardhash(inp);
2172         } else if (inp->inp_flags & INP_CONNECTED) {
2173                 in_pcbremconnhash(inp);
2174         }
2175
2176         if (inp->inp_flags & INP_ONLIST)
2177                 in_pcbofflist(inp);
2178 }
2179
2180 int
2181 prison_xinpcb(struct thread *td, struct inpcb *inp)
2182 {
2183         struct ucred *cr;
2184
2185         if (td->td_proc == NULL)
2186                 return (0);
2187         cr = td->td_proc->p_ucred;
2188         if (cr->cr_prison == NULL)
2189                 return (0);
2190         if (inp->inp_socket && inp->inp_socket->so_cred &&
2191             inp->inp_socket->so_cred->cr_prison &&
2192             cr->cr_prison == inp->inp_socket->so_cred->cr_prison)
2193                 return (0);
2194         return (1);
2195 }
2196
2197 int
2198 in_pcblist_global(SYSCTL_HANDLER_ARGS)
2199 {
2200         struct inpcbinfo *pcbinfo_arr = arg1;
2201         int pcbinfo_arrlen = arg2;
2202         struct inpcb *marker;
2203         int cpu, origcpu;
2204         int error, n;
2205
2206         KASSERT(pcbinfo_arrlen <= ncpus && pcbinfo_arrlen >= 1,
2207             ("invalid pcbinfo count %d", pcbinfo_arrlen));
2208
2209         /*
2210          * The process of preparing the TCB list is too time-consuming and
2211          * resource-intensive to repeat twice on every request.
2212          */
2213         n = 0;
2214         if (req->oldptr == NULL) {
2215                 for (cpu = 0; cpu < pcbinfo_arrlen; ++cpu)
2216                         n += pcbinfo_arr[cpu].ipi_count;
2217                 req->oldidx = (n + n/8 + 10) * sizeof(struct xinpcb);
2218                 return 0;
2219         }
2220
2221         if (req->newptr != NULL)
2222                 return EPERM;
2223
2224         marker = kmalloc(sizeof(struct inpcb), M_TEMP, M_WAITOK|M_ZERO);
2225         marker->inp_flags |= INP_PLACEMARKER;
2226
2227         /*
2228          * OK, now we're committed to doing something.  Re-fetch ipi_count
2229          * after obtaining the generation count.
2230          */
2231         error = 0;
2232         origcpu = mycpuid;
2233         for (cpu = 0; cpu < pcbinfo_arrlen && error == 0; ++cpu) {
2234                 struct inpcbinfo *pcbinfo = &pcbinfo_arr[cpu];
2235                 struct inpcb *inp;
2236                 struct xinpcb xi;
2237                 int i;
2238
2239                 lwkt_migratecpu(cpu);
2240
2241                 GET_PCBINFO_TOKEN(pcbinfo);
2242
2243                 n = pcbinfo->ipi_count;
2244
2245                 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
2246                 i = 0;
2247                 while ((inp = LIST_NEXT(marker, inp_list)) != NULL && i < n) {
2248                         LIST_REMOVE(marker, inp_list);
2249                         LIST_INSERT_AFTER(inp, marker, inp_list);
2250
2251                         if (inp->inp_flags & INP_PLACEMARKER)
2252                                 continue;
2253                         if (prison_xinpcb(req->td, inp))
2254                                 continue;
2255
2256                         bzero(&xi, sizeof xi);
2257                         xi.xi_len = sizeof xi;
2258                         bcopy(inp, &xi.xi_inp, sizeof *inp);
2259                         if (inp->inp_socket)
2260                                 sotoxsocket(inp->inp_socket, &xi.xi_socket);
2261                         if ((error = SYSCTL_OUT(req, &xi, sizeof xi)) != 0)
2262                                 break;
2263                         ++i;
2264                 }
2265                 LIST_REMOVE(marker, inp_list);
2266
2267                 REL_PCBINFO_TOKEN(pcbinfo);
2268
2269                 if (error == 0 && i < n) {
2270                         bzero(&xi, sizeof xi);
2271                         xi.xi_len = sizeof xi;
2272                         while (i < n) {
2273                                 error = SYSCTL_OUT(req, &xi, sizeof xi);
2274                                 if (error)
2275                                         break;
2276                                 ++i;
2277                         }
2278                 }
2279         }
2280
2281         lwkt_migratecpu(origcpu);
2282         kfree(marker, M_TEMP);
2283         return error;
2284 }
2285
2286 int
2287 in_pcblist_global_ncpus2(SYSCTL_HANDLER_ARGS)
2288 {
2289         return in_pcblist_global(oidp, arg1, ncpus2, req);
2290 }
2291
2292 void
2293 in_savefaddr(struct socket *so, const struct sockaddr *faddr)
2294 {
2295         struct sockaddr_in *sin;
2296
2297         KASSERT(faddr->sa_family == AF_INET,
2298             ("not AF_INET faddr %d", faddr->sa_family));
2299
2300         sin = kmalloc(sizeof(*sin), M_SONAME, M_WAITOK | M_ZERO);
2301         sin->sin_family = AF_INET;
2302         sin->sin_len = sizeof(*sin);
2303         sin->sin_port = ((const struct sockaddr_in *)faddr)->sin_port;
2304         sin->sin_addr = ((const struct sockaddr_in *)faddr)->sin_addr;
2305
2306         so->so_faddr = (struct sockaddr *)sin;
2307 }
2308
2309 void
2310 in_pcbportinfo_init(struct inpcbportinfo *portinfo, int hashsize,
2311     u_short offset)
2312 {
2313         memset(portinfo, 0, sizeof(*portinfo));
2314
2315         portinfo->offset = offset;
2316         portinfo->lastport = offset;
2317         portinfo->lastlow = offset;
2318         portinfo->lasthi = offset;
2319
2320         portinfo->porthashbase = hashinit(hashsize, M_PCB,
2321             &portinfo->porthashmask);
2322 }
2323
2324 void
2325 in_pcbportrange(u_short *hi0, u_short *lo0, u_short ofs, u_short step)
2326 {
2327         int hi, lo;
2328
2329         if (step == 1)
2330                 return;
2331
2332         hi = *hi0;
2333         lo = *lo0;
2334
2335         hi = rounddown2(hi, step);
2336         hi += ofs;
2337         if (hi > (int)*hi0)
2338                 hi -= step;
2339
2340         lo = roundup2(lo, step);
2341         lo -= (step - ofs);
2342         if (lo < (int)*lo0)
2343                 lo += step;
2344
2345         *hi0 = hi;
2346         *lo0 = lo;
2347 }
2348
2349 void
2350 in_pcbglobalinit(void)
2351 {
2352         int cpu;
2353
2354         in_pcbmarkers = kmalloc(ncpus * sizeof(struct inpcb), M_PCB,
2355             M_WAITOK | M_ZERO);
2356         in_pcbcontainer_markers = kmalloc(ncpus * sizeof(struct inpcontainer),
2357             M_PCB, M_WAITOK | M_ZERO);
2358
2359         for (cpu = 0; cpu < ncpus; ++cpu) {
2360                 struct inpcontainer *ic = &in_pcbcontainer_markers[cpu];
2361                 struct inpcb *marker = &in_pcbmarkers[cpu];
2362
2363                 marker->inp_flags |= INP_PLACEMARKER;
2364                 ic->ic_inp = marker;
2365         }
2366 }
2367
2368 struct inpcb *
2369 in_pcbmarker(int cpuid)
2370 {
2371         KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid));
2372         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
2373
2374         return &in_pcbmarkers[cpuid];
2375 }
2376
2377 struct inpcontainer *
2378 in_pcbcontainer_marker(int cpuid)
2379 {
2380         KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid));
2381         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
2382
2383         return &in_pcbcontainer_markers[cpuid];
2384 }
2385
2386 void
2387 in_pcbresetroute(struct inpcb *inp)
2388 {
2389         struct route *ro = &inp->inp_route;
2390
2391         if (ro->ro_rt != NULL)
2392                 RTFREE(ro->ro_rt);
2393         bzero(ro, sizeof(*ro));
2394 }
2395
2396 u_short
2397 in_pcblastport_down(volatile u_short *lastport, u_short first, u_short last,
2398     u_short step)
2399 {
2400         u_short lport;
2401
2402         for (;;) {
2403                 u_short olport;
2404
2405                 olport = *lastport;
2406                 lport = olport - step;
2407                 if (__predict_false(lport > first || lport < last))
2408                         lport = first;
2409                 if (atomic_cmpset_short(lastport, olport, lport))
2410                         break;
2411         }
2412         return lport;
2413 }
2414
2415 u_short
2416 in_pcblastport_up(volatile u_short *lastport, u_short first, u_short last,
2417     u_short step)
2418 {
2419         u_short lport;
2420
2421         for (;;) {
2422                 u_short olport;
2423
2424                 olport = *lastport;
2425                 lport = olport + step;
2426                 if (__predict_false(lport < first || lport > last))
2427                         lport = first;
2428                 if (atomic_cmpset_short(lastport, olport, lport))
2429                         break;
2430         }
2431         return lport;
2432 }