sys/netinet/in_pcb.c

   1 /*
   2  * Copyright (c) 2004 Jeffrey M. Hsu.  All rights reserved.
   3  * Copyright (c) 2004 The DragonFly Project.  All rights reserved.
   4  *
   5  * This code is derived from software contributed to The DragonFly Project
   6  * by Jeffrey M. Hsu.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the name of The DragonFly Project nor the names of its
  17  *    contributors may be used to endorse or promote products derived
  18  *    from this software without specific, prior written permission.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  23  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  24  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  25  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  26  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  27  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  28  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  29  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  30  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  */
  33
  34 /*
  35  * Copyright (c) 1982, 1986, 1991, 1993, 1995
  36  *      The Regents of the University of California.  All rights reserved.
  37  *
  38  * Redistribution and use in source and binary forms, with or without
  39  * modification, are permitted provided that the following conditions
  40  * are met:
  41  * 1. Redistributions of source code must retain the above copyright
  42  *    notice, this list of conditions and the following disclaimer.
  43  * 2. Redistributions in binary form must reproduce the above copyright
  44  *    notice, this list of conditions and the following disclaimer in the
  45  *    documentation and/or other materials provided with the distribution.
  46  * 3. Neither the name of the University nor the names of its contributors
  47  *    may be used to endorse or promote products derived from this software
  48  *    without specific prior written permission.
  49  *
  50  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  51  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  52  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  53  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  54  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  55  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  56  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  57  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  58  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  59  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  60  * SUCH DAMAGE.
  61  *
  62  *      @(#)in_pcb.c    8.4 (Berkeley) 5/24/95
  63  * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $
  64  */
  65
  66 #include "opt_ipsec.h"
  67 #include "opt_inet6.h"
  68
  69 #include <sys/param.h>
  70 #include <sys/systm.h>
  71 #include <sys/malloc.h>
  72 #include <sys/mbuf.h>
  73 #include <sys/domain.h>
  74 #include <sys/protosw.h>
  75 #include <sys/socket.h>
  76 #include <sys/socketvar.h>
  77 #include <sys/proc.h>
  78 #include <sys/priv.h>
  79 #include <sys/jail.h>
  80 #include <sys/kernel.h>
  81 #include <sys/sysctl.h>
  82
  83 #include <sys/thread2.h>
  84 #include <sys/socketvar2.h>
  85 #include <sys/msgport2.h>
  86
  87 #include <machine/limits.h>
  88
  89 #include <net/if.h>
  90 #include <net/if_types.h>
  91 #include <net/route.h>
  92 #include <net/netisr2.h>
  93 #include <net/toeplitz2.h>
  94
  95 #include <netinet/in.h>
  96 #include <netinet/in_pcb.h>
  97 #include <netinet/in_var.h>
  98 #include <netinet/ip_var.h>
  99 #ifdef INET6
 100 #include <netinet/ip6.h>
 101 #include <netinet6/ip6_var.h>
 102 #endif /* INET6 */
 103
 104 #ifdef IPSEC
 105 #include <netinet6/ipsec.h>
 106 #include <netproto/key/key.h>
 107 #include <netproto/ipsec/esp_var.h>
 108 #endif
 109
 110 #ifdef FAST_IPSEC
 111 #if defined(IPSEC) || defined(IPSEC_ESP)
 112 #error "Bad idea: don't compile with both IPSEC and FAST_IPSEC!"
 113 #endif
 114
 115 #include <netproto/ipsec/ipsec.h>
 116 #include <netproto/ipsec/key.h>
 117 #define IPSEC
 118 #endif /* FAST_IPSEC */
 119
 120 #define INP_LOCALGROUP_SIZMIN   8
 121 #define INP_LOCALGROUP_SIZMAX   256
 122
 123 static struct inpcb *in_pcblookup_local(struct inpcbporthead *porthash,
 124                 struct in_addr laddr, u_int lport_arg, int wild_okay,
 125                 struct ucred *cred);
 126
 127 struct in_addr zeroin_addr;
 128
 129 /*
 130  * These configure the range of local port addresses assigned to
 131  * "unspecified" outgoing connections/packets/whatever.
 132  */
 133 int ipport_lowfirstauto = IPPORT_RESERVED - 1;  /* 1023 */
 134 int ipport_lowlastauto = IPPORT_RESERVEDSTART;  /* 600 */
 135
 136 int ipport_firstauto = IPPORT_RESERVED;         /* 1024 */
 137 int ipport_lastauto = IPPORT_USERRESERVED;      /* 5000 */
 138
 139 int ipport_hifirstauto = IPPORT_HIFIRSTAUTO;    /* 49152 */
 140 int ipport_hilastauto = IPPORT_HILASTAUTO;      /* 65535 */
 141
 142 #define RANGECHK(var, min, max) \
 143         if ((var) < (min)) { (var) = (min); } \
 144         else if ((var) > (max)) { (var) = (max); }
 145
 146 int udpencap_enable = 1;        /* enabled by default */
 147 int udpencap_port = 4500;       /* triggers decapsulation */
 148
 149 /*
 150  * Per-netisr inpcb markers.
 151  * NOTE: they should only be used in netisrs.
 152  */
 153 static struct inpcb             *in_pcbmarkers;
 154 static struct inpcontainer      *in_pcbcontainer_markers;
 155
 156 static int
 157 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
 158 {
 159         int error;
 160
 161         error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
 162         if (!error) {
 163                 RANGECHK(ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
 164                 RANGECHK(ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
 165
 166                 RANGECHK(ipport_firstauto, IPPORT_RESERVED, USHRT_MAX);
 167                 RANGECHK(ipport_lastauto, IPPORT_RESERVED, USHRT_MAX);
 168
 169                 RANGECHK(ipport_hifirstauto, IPPORT_RESERVED, USHRT_MAX);
 170                 RANGECHK(ipport_hilastauto, IPPORT_RESERVED, USHRT_MAX);
 171         }
 172         return (error);
 173 }
 174
 175 #undef RANGECHK
 176
 177 SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports");
 178
 179 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW,
 180            &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", "");
 181 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, CTLTYPE_INT|CTLFLAG_RW,
 182            &ipport_lowlastauto, 0, &sysctl_net_ipport_check, "I", "");
 183 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, CTLTYPE_INT|CTLFLAG_RW,
 184            &ipport_firstauto, 0, &sysctl_net_ipport_check, "I", "");
 185 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, CTLTYPE_INT|CTLFLAG_RW,
 186            &ipport_lastauto, 0, &sysctl_net_ipport_check, "I", "");
 187 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, CTLTYPE_INT|CTLFLAG_RW,
 188            &ipport_hifirstauto, 0, &sysctl_net_ipport_check, "I", "");
 189 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW,
 190            &ipport_hilastauto, 0, &sysctl_net_ipport_check, "I", "");
 191
 192 static int ip_porthash_trycount = 15;
 193 SYSCTL_INT(_net_inet_ip, OID_AUTO, porthash_trycount, CTLFLAG_RW,
 194     &ip_porthash_trycount, 0,
 195     "Number of tries to find local port matching hash of 4-tuple");
 196
 197 /*
 198  * in_pcb.c: manage the Protocol Control Blocks.
 199  *
 200  * NOTE: It is assumed that most of these functions will be called from
 201  * a critical section.  XXX - There are, unfortunately, a few exceptions
 202  * to this rule that should be fixed.
 203  *
 204  * NOTE: The caller should initialize the cpu field to the cpu running the
 205  * protocol stack associated with this inpcbinfo.
 206  */
 207
 208 void
 209 in_pcbinfo_init(struct inpcbinfo *pcbinfo, int cpu, boolean_t shared)
 210 {
 211         KASSERT(cpu >= 0 && cpu < ncpus, ("invalid cpu%d", cpu));
 212         pcbinfo->cpu = cpu;
 213
 214         LIST_INIT(&pcbinfo->pcblisthead);
 215         pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), M_PCB,
 216                                     M_WAITOK | M_ZERO);
 217
 218         if (shared) {
 219                 pcbinfo->infotoken = kmalloc(sizeof(struct lwkt_token),
 220                     M_PCB, M_WAITOK);
 221                 lwkt_token_init(pcbinfo->infotoken, "infotoken");
 222         } else {
 223                 pcbinfo->infotoken = NULL;
 224         }
 225 }
 226
 227 struct baddynamicports baddynamicports;
 228
 229 /*
 230  * Check if the specified port is invalid for dynamic allocation.
 231  */
 232 int
 233 in_baddynamic(u_int16_t port, u_int16_t proto)
 234 {
 235         switch (proto) {
 236         case IPPROTO_TCP:
 237                 return (DP_ISSET(baddynamicports.tcp, port));
 238         case IPPROTO_UDP:
 239 #ifdef IPSEC
 240                 /* Cannot preset this as it is a sysctl */
 241                 if (port == udpencap_port)
 242                         return (1);
 243 #endif
 244                 return (DP_ISSET(baddynamicports.udp, port));
 245         default:
 246                 return (0);
 247         }
 248 }
 249
 250 void
 251 in_pcbonlist(struct inpcb *inp)
 252 {
 253         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 254
 255         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
 256             ("not in the correct netisr"));
 257         KASSERT((inp->inp_flags & INP_ONLIST) == 0, ("already on pcblist"));
 258         inp->inp_flags |= INP_ONLIST;
 259
 260         GET_PCBINFO_TOKEN(pcbinfo);
 261         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list);
 262         pcbinfo->ipi_count++;
 263         REL_PCBINFO_TOKEN(pcbinfo);
 264 }
 265
 266 void
 267 in_pcbofflist(struct inpcb *inp)
 268 {
 269         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 270
 271         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
 272             ("not in the correct netisr"));
 273         KASSERT(inp->inp_flags & INP_ONLIST, ("not on pcblist"));
 274         inp->inp_flags &= ~INP_ONLIST;
 275
 276         GET_PCBINFO_TOKEN(pcbinfo);
 277         LIST_REMOVE(inp, inp_list);
 278         KASSERT(pcbinfo->ipi_count > 0,
 279             ("invalid inpcb count %d", pcbinfo->ipi_count));
 280         pcbinfo->ipi_count--;
 281         REL_PCBINFO_TOKEN(pcbinfo);
 282 }
 283
 284 /*
 285  * Allocate a PCB and associate it with the socket.
 286  */
 287 int
 288 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
 289 {
 290         struct inpcb *inp;
 291 #ifdef IPSEC
 292         int error;
 293 #endif
 294
 295         inp = kmalloc(pcbinfo->ipi_size, M_PCB, M_WAITOK|M_ZERO|M_NULLOK);
 296         if (inp == NULL)
 297                 return (ENOMEM);
 298         inp->inp_lgrpindex = -1;
 299         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
 300         inp->inp_pcbinfo = pcbinfo;
 301         inp->inp_socket = so;
 302 #ifdef IPSEC
 303         error = ipsec_init_policy(so, &inp->inp_sp);
 304         if (error != 0) {
 305                 kfree(inp, M_PCB);
 306                 return (error);
 307         }
 308 #endif
 309 #ifdef INET6
 310         if (INP_CHECK_SOCKAF(so, AF_INET6)) {
 311                 if (ip6_auto_flowlabel)
 312                         inp->inp_flags |= IN6P_AUTOFLOWLABEL;
 313                 inp->inp_af = AF_INET6;
 314         } else
 315 #endif
 316         inp->inp_af = AF_INET;
 317         soreference(so);
 318         so->so_pcb = inp;
 319
 320         in_pcbonlist(inp);
 321         return (0);
 322 }
 323
 324 /*
 325  * Unlink a pcb with the intention of moving it to another cpu with a
 326  * different pcbinfo.  While unlinked nothing should attempt to dereference
 327  * inp_pcbinfo, NULL it out so we assert if it does.
 328  */
 329 void
 330 in_pcbunlink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags)
 331 {
 332         KASSERT(inp->inp_pcbinfo == pcbinfo, ("pcbinfo mismatch"));
 333         KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0,
 334             ("already linked"));
 335
 336         in_pcbofflist(inp);
 337         inp->inp_pcbinfo = NULL;
 338 }
 339
 340 void
 341 in_pcbunlink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
 342 {
 343         in_pcbunlink_flags(inp, pcbinfo, INP_WILDCARD);
 344 }
 345
 346 /*
 347  * Relink a pcb into a new pcbinfo.
 348  */
 349 void
 350 in_pcblink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags)
 351 {
 352         KASSERT(inp->inp_pcbinfo == NULL, ("has pcbinfo"));
 353         KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0,
 354             ("already linked"));
 355
 356         inp->inp_pcbinfo = pcbinfo;
 357         in_pcbonlist(inp);
 358 }
 359
 360 void
 361 in_pcblink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
 362 {
 363         return in_pcblink_flags(inp, pcbinfo, INP_WILDCARD);
 364 }
 365
 366 static boolean_t
 367 in_pcbporthash_update(struct inpcbportinfo *portinfo,
 368     struct inpcb *inp, u_short lport, struct ucred *cred, int wild)
 369 {
 370         struct inpcbporthead *porthash;
 371
 372         /*
 373          * This has to be atomic.  If the porthash is shared across multiple
 374          * protocol threads, e.g. tcp and udp, then the token must be held.
 375          */
 376         porthash = in_pcbporthash_head(portinfo, lport);
 377         GET_PORTHASH_TOKEN(porthash);
 378
 379         if (in_pcblookup_local(porthash, inp->inp_laddr, lport,
 380             wild, cred) != NULL) {
 381                 REL_PORTHASH_TOKEN(porthash);
 382                 return FALSE;
 383         }
 384         inp->inp_lport = lport;
 385         in_pcbinsporthash(porthash, inp);
 386
 387         REL_PORTHASH_TOKEN(porthash);
 388         return TRUE;
 389 }
 390
 391 static int
 392 in_pcbsetlport(struct inpcb *inp, int wild, struct ucred *cred)
 393 {
 394         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 395         struct inpcbportinfo *portinfo;
 396         u_short first, last, lport, step;
 397         u_short *lastport;
 398         int count, error;
 399         int portinfo_first, portinfo_idx;
 400
 401         inp->inp_flags |= INP_ANONPORT;
 402
 403         step = pcbinfo->portinfo_mask + 1;
 404         portinfo_first = mycpuid & pcbinfo->portinfo_mask;
 405         portinfo_idx = portinfo_first;
 406 loop:
 407         portinfo = &pcbinfo->portinfo[portinfo_idx];
 408
 409         if (inp->inp_flags & INP_HIGHPORT) {
 410                 first = ipport_hifirstauto;     /* sysctl */
 411                 last  = ipport_hilastauto;
 412                 lastport = &portinfo->lasthi;
 413         } else if (inp->inp_flags & INP_LOWPORT) {
 414                 if (cred &&
 415                     (error =
 416                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 417                         inp->inp_laddr.s_addr = INADDR_ANY;
 418                         return error;
 419                 }
 420                 first = ipport_lowfirstauto;    /* 1023 */
 421                 last  = ipport_lowlastauto;     /* 600 */
 422                 lastport = &portinfo->lastlow;
 423         } else {
 424                 first = ipport_firstauto;       /* sysctl */
 425                 last  = ipport_lastauto;
 426                 lastport = &portinfo->lastport;
 427         }
 428
 429         /*
 430          * Simple check to ensure all ports are not used up causing
 431          * a deadlock here.
 432          *
 433          * We split the two cases (up and down) so that the direction
 434          * is not being tested on each round of the loop.
 435          */
 436         if (first > last) {
 437                 /*
 438                  * counting down
 439                  */
 440                 in_pcbportrange(&first, &last, portinfo->offset, step);
 441                 count = (first - last) / step;
 442
 443                 for (;;) {
 444                         if (count-- < 0) {      /* completely used? */
 445                                 error = EADDRNOTAVAIL;
 446                                 break;
 447                         }
 448                         lport = in_pcblastport_down(lastport,
 449                             first, last, step);
 450                         KKASSERT((lport & pcbinfo->portinfo_mask) ==
 451                             portinfo->offset);
 452                         lport = htons(lport);
 453
 454                         if (in_pcbporthash_update(portinfo, inp, lport,
 455                             cred, wild)) {
 456                                 error = 0;
 457                                 break;
 458                         }
 459                 }
 460         } else {
 461                 /*
 462                  * counting up
 463                  */
 464                 in_pcbportrange(&last, &first, portinfo->offset, step);
 465                 count = (last - first) / step;
 466
 467                 for (;;) {
 468                         if (count-- < 0) {      /* completely used? */
 469                                 error = EADDRNOTAVAIL;
 470                                 break;
 471                         }
 472                         lport = in_pcblastport_up(lastport, first, last, step);
 473                         KKASSERT((lport & pcbinfo->portinfo_mask) ==
 474                             portinfo->offset);
 475                         lport = htons(lport);
 476
 477                         if (in_pcbporthash_update(portinfo, inp, lport,
 478                             cred, wild)) {
 479                                 error = 0;
 480                                 break;
 481                         }
 482                 }
 483         }
 484
 485         if (error) {
 486                 /* Try next portinfo */
 487                 portinfo_idx++;
 488                 portinfo_idx &= pcbinfo->portinfo_mask;
 489                 if (portinfo_idx != portinfo_first)
 490                         goto loop;
 491                 inp->inp_laddr.s_addr = INADDR_ANY;
 492         }
 493         return error;
 494 }
 495
 496 int
 497 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
 498 {
 499         struct socket *so = inp->inp_socket;
 500         struct sockaddr_in jsin;
 501         struct ucred *cred = NULL;
 502         int wild = 0;
 503
 504         if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */
 505                 return (EADDRNOTAVAIL);
 506         if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
 507                 return (EINVAL);        /* already bound */
 508
 509         if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT)))
 510                 wild = 1;    /* neither SO_REUSEADDR nor SO_REUSEPORT is set */
 511         if (td->td_proc)
 512                 cred = td->td_proc->p_ucred;
 513
 514         if (nam != NULL) {
 515                 struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 516                 struct inpcbinfo *pcbinfo;
 517                 struct inpcbportinfo *portinfo;
 518                 struct inpcbporthead *porthash;
 519                 struct inpcb *t;
 520                 u_short lport, lport_ho;
 521                 int reuseport = (so->so_options & SO_REUSEPORT);
 522                 int error;
 523
 524                 if (nam->sa_len != sizeof *sin)
 525                         return (EINVAL);
 526 #ifdef notdef
 527                 /*
 528                  * We should check the family, but old programs
 529                  * incorrectly fail to initialize it.
 530                  */
 531                 if (sin->sin_family != AF_INET)
 532                         return (EAFNOSUPPORT);
 533 #endif
 534                 if (!prison_replace_wildcards(td, nam))
 535                         return (EINVAL);
 536
 537                 lport = sin->sin_port;
 538                 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 539                         /*
 540                          * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
 541                          * allow complete duplication of binding if
 542                          * SO_REUSEPORT is set, or if SO_REUSEADDR is set
 543                          * and a multicast address is bound on both
 544                          * new and duplicated sockets.
 545                          */
 546                         if (so->so_options & SO_REUSEADDR)
 547                                 reuseport = SO_REUSEADDR | SO_REUSEPORT;
 548                 } else if (sin->sin_addr.s_addr != INADDR_ANY) {
 549                         sin->sin_port = 0;              /* yech... */
 550                         bzero(&sin->sin_zero, sizeof sin->sin_zero);
 551                         if (ifa_ifwithaddr((struct sockaddr *)sin) == NULL)
 552                                 return (EADDRNOTAVAIL);
 553                 }
 554
 555                 inp->inp_laddr = sin->sin_addr;
 556
 557                 jsin.sin_family = AF_INET;
 558                 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 559                 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 560                         inp->inp_laddr.s_addr = INADDR_ANY;
 561                         return (EINVAL);
 562                 }
 563                 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 564
 565                 if (lport == 0) {
 566                         /* Auto-select local port */
 567                         return in_pcbsetlport(inp, wild, cred);
 568                 }
 569                 lport_ho = ntohs(lport);
 570
 571                 /* GROSS */
 572                 if (lport_ho < IPPORT_RESERVED && cred &&
 573                     (error =
 574                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 575                         inp->inp_laddr.s_addr = INADDR_ANY;
 576                         return (error);
 577                 }
 578
 579                 /*
 580                  * Locate the proper portinfo based on lport
 581                  */
 582                 pcbinfo = inp->inp_pcbinfo;
 583                 portinfo =
 584                     &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask];
 585                 KKASSERT((lport_ho & pcbinfo->portinfo_mask) ==
 586                     portinfo->offset);
 587
 588                 /*
 589                  * This has to be atomic.  If the porthash is shared across
 590                  * multiple protocol threads, e.g. tcp and udp then the token
 591                  * must be held.
 592                  */
 593                 porthash = in_pcbporthash_head(portinfo, lport);
 594                 GET_PORTHASH_TOKEN(porthash);
 595
 596                 if (so->so_cred->cr_uid != 0 &&
 597                     !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 598                         t = in_pcblookup_local(porthash, sin->sin_addr, lport,
 599                             INPLOOKUP_WILDCARD, cred);
 600                         if (t &&
 601                             (so->so_cred->cr_uid !=
 602                              t->inp_socket->so_cred->cr_uid)) {
 603                                 inp->inp_laddr.s_addr = INADDR_ANY;
 604                                 error = EADDRINUSE;
 605                                 goto done;
 606                         }
 607                 }
 608                 if (cred && !prison_replace_wildcards(td, nam)) {
 609                         inp->inp_laddr.s_addr = INADDR_ANY;
 610                         error = EADDRNOTAVAIL;
 611                         goto done;
 612                 }
 613                 t = in_pcblookup_local(porthash, sin->sin_addr, lport,
 614                     wild, cred);
 615                 if (t && !(reuseport & t->inp_socket->so_options)) {
 616                         inp->inp_laddr.s_addr = INADDR_ANY;
 617                         error = EADDRINUSE;
 618                         goto done;
 619                 }
 620                 inp->inp_lport = lport;
 621                 in_pcbinsporthash(porthash, inp);
 622                 error = 0;
 623 done:
 624                 REL_PORTHASH_TOKEN(porthash);
 625                 return (error);
 626         } else {
 627                 jsin.sin_family = AF_INET;
 628                 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 629                 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 630                         inp->inp_laddr.s_addr = INADDR_ANY;
 631                         return (EINVAL);
 632                 }
 633                 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 634
 635                 return in_pcbsetlport(inp, wild, cred);
 636         }
 637 }
 638
 639 static struct inpcb *
 640 in_pcblookup_localremote(struct inpcbporthead *porthash, struct in_addr laddr,
 641     u_short lport, struct in_addr faddr, u_short fport, struct ucred *cred)
 642 {
 643         struct inpcb *inp;
 644         struct inpcbport *phd;
 645         struct inpcb *match = NULL;
 646
 647         /*
 648          * If the porthashbase is shared across several cpus, it must
 649          * have been locked.
 650          */
 651         ASSERT_PORTHASH_TOKEN_HELD(porthash);
 652
 653         /*
 654          * Best fit PCB lookup.
 655          *
 656          * First see if this local port is in use by looking on the
 657          * port hash list.
 658          */
 659         LIST_FOREACH(phd, porthash, phd_hash) {
 660                 if (phd->phd_port == lport)
 661                         break;
 662         }
 663         if (phd != NULL) {
 664                 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
 665 #ifdef INET6
 666                         if (!INP_ISIPV4(inp))
 667                                 continue;
 668 #endif
 669                         if (inp->inp_laddr.s_addr != INADDR_ANY &&
 670                             inp->inp_laddr.s_addr != laddr.s_addr)
 671                                 continue;
 672
 673                         if (inp->inp_faddr.s_addr != INADDR_ANY &&
 674                             inp->inp_faddr.s_addr != faddr.s_addr)
 675                                 continue;
 676
 677                         if (inp->inp_fport != 0 && inp->inp_fport != fport)
 678                                 continue;
 679
 680                         if (cred == NULL ||
 681                             cred->cr_prison ==
 682                             inp->inp_socket->so_cred->cr_prison) {
 683                                 match = inp;
 684                                 break;
 685                         }
 686                 }
 687         }
 688         return (match);
 689 }
 690
 691 static boolean_t
 692 in_pcbporthash_update4(struct inpcbportinfo *portinfo,
 693     struct inpcb *inp, u_short lport, const struct sockaddr_in *sin,
 694     struct ucred *cred)
 695 {
 696         struct inpcbporthead *porthash;
 697
 698         /*
 699          * This has to be atomic.  If the porthash is shared across multiple
 700          * protocol threads, e.g. tcp and udp, then the token must be held.
 701          */
 702         porthash = in_pcbporthash_head(portinfo, lport);
 703         GET_PORTHASH_TOKEN(porthash);
 704
 705         if (in_pcblookup_localremote(porthash, inp->inp_laddr,
 706             lport, sin->sin_addr, sin->sin_port, cred) != NULL) {
 707                 REL_PORTHASH_TOKEN(porthash);
 708                 return FALSE;
 709         }
 710         inp->inp_lport = lport;
 711         in_pcbinsporthash(porthash, inp);
 712
 713         REL_PORTHASH_TOKEN(porthash);
 714         return TRUE;
 715 }
 716
 717 int
 718 in_pcbbind_remote(struct inpcb *inp, const struct sockaddr *remote,
 719     struct thread *td)
 720 {
 721         struct proc *p = td->td_proc;
 722         u_short *lastport;
 723         const struct sockaddr_in *sin = (const struct sockaddr_in *)remote;
 724         struct sockaddr_in jsin;
 725         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 726         struct inpcbportinfo *portinfo;
 727         struct ucred *cred = NULL;
 728         u_short first, last, lport, step;
 729         int count, error, selfconn;
 730         int portinfo_first, portinfo_idx;
 731         int hash_cpu, hash_count, hash_count0;
 732         uint32_t hash_base = 0, hash;
 733
 734         if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */
 735                 return (EADDRNOTAVAIL);
 736
 737         KKASSERT(inp->inp_laddr.s_addr != INADDR_ANY);
 738         if (inp->inp_lport != 0)
 739                 return (EINVAL);        /* already bound */
 740
 741         KKASSERT(p);
 742         cred = p->p_ucred;
 743
 744         jsin.sin_family = AF_INET;
 745         jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 746         if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 747                 inp->inp_laddr.s_addr = INADDR_ANY;
 748                 return (EINVAL);
 749         }
 750         inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 751
 752         hash_count0 = ip_porthash_trycount;
 753         if (hash_count0 > 0) {
 754                 hash_base = toeplitz_piecemeal_addr(sin->sin_addr.s_addr) ^
 755                     toeplitz_piecemeal_addr(inp->inp_laddr.s_addr) ^
 756                     toeplitz_piecemeal_port(sin->sin_port);
 757         } else {
 758                 hash_count0 = 0;
 759         }
 760
 761         inp->inp_flags |= INP_ANONPORT;
 762
 763         step = pcbinfo->portinfo_mask + 1;
 764         portinfo_first = mycpuid & pcbinfo->portinfo_mask;
 765         portinfo_idx = portinfo_first;
 766 loop:
 767         hash_cpu = portinfo_idx & ncpus2_mask;
 768         portinfo = &pcbinfo->portinfo[portinfo_idx];
 769         selfconn = 0;
 770
 771         if (inp->inp_flags & INP_HIGHPORT) {
 772                 first = ipport_hifirstauto;     /* sysctl */
 773                 last  = ipport_hilastauto;
 774                 lastport = &portinfo->lasthi;
 775         } else if (inp->inp_flags & INP_LOWPORT) {
 776                 if (cred &&
 777                     (error =
 778                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 779                         inp->inp_laddr.s_addr = INADDR_ANY;
 780                         return (error);
 781                 }
 782                 first = ipport_lowfirstauto;    /* 1023 */
 783                 last  = ipport_lowlastauto;     /* 600 */
 784                 lastport = &portinfo->lastlow;
 785         } else {
 786                 first = ipport_firstauto;       /* sysctl */
 787                 last  = ipport_lastauto;
 788                 lastport = &portinfo->lastport;
 789         }
 790
 791         /* This could happen on loopback interface */
 792 #define IS_SELFCONNECT(inp, lport, sin)                 \
 793         (__predict_false((sin)->sin_port == (lport) &&  \
 794          (sin)->sin_addr.s_addr == (inp)->inp_laddr.s_addr))
 795
 796         /*
 797          * Simple check to ensure all ports are not used up causing
 798          * a deadlock here.
 799          *
 800          * We split the two cases (up and down) so that the direction
 801          * is not being tested on each round of the loop.
 802          */
 803         hash_count = hash_count0;
 804         if (first > last) {
 805                 /*
 806                  * counting down
 807                  */
 808                 in_pcbportrange(&first, &last, portinfo->offset, step);
 809                 count = ((first - last) / step) + hash_count;
 810
 811                 for (;;) {
 812                         if (count-- < 0) {      /* completely used? */
 813                                 error = EADDRNOTAVAIL;
 814                                 break;
 815                         }
 816
 817                         lport = in_pcblastport_down(lastport, first, last,
 818                             step);
 819                         KKASSERT((lport & pcbinfo->portinfo_mask) ==
 820                             portinfo->offset);
 821                         lport = htons(lport);
 822                         if (IS_SELFCONNECT(inp, lport, sin)) {
 823                                 if (!selfconn) {
 824                                         ++count; /* don't count this try */
 825                                         selfconn = 1;
 826                                 }
 827                                 continue;
 828                         }
 829
 830                         if (hash_count) {
 831                                 --hash_count;
 832                                 hash = hash_base ^
 833                                     toeplitz_piecemeal_port(lport);
 834                                 if ((hash & ncpus2_mask) != hash_cpu && hash_count)
 835                                         continue;
 836                         }
 837
 838                         if (in_pcbporthash_update4(portinfo,
 839                             inp, lport, sin, cred)) {
 840                                 error = 0;
 841                                 break;
 842                         }
 843                 }
 844         } else {
 845                 /*
 846                  * counting up
 847                  */
 848                 in_pcbportrange(&last, &first, portinfo->offset, step);
 849                 count = ((last - first) / step) + hash_count;
 850
 851                 for (;;) {
 852                         if (count-- < 0) {      /* completely used? */
 853                                 error = EADDRNOTAVAIL;
 854                                 break;
 855                         }
 856
 857                         lport = in_pcblastport_up(lastport, first, last, step);
 858                         KKASSERT((lport & pcbinfo->portinfo_mask) ==
 859                             portinfo->offset);
 860                         lport = htons(lport);
 861                         if (IS_SELFCONNECT(inp, lport, sin)) {
 862                                 if (!selfconn) {
 863                                         ++count; /* don't count this try */
 864                                         selfconn = 1;
 865                                 }
 866                                 continue;
 867                         }
 868
 869                         if (hash_count) {
 870                                 --hash_count;
 871                                 hash = hash_base ^
 872                                     toeplitz_piecemeal_port(lport);
 873                                 if ((hash & ncpus2_mask) != hash_cpu && hash_count)
 874                                         continue;
 875                         }
 876
 877                         if (in_pcbporthash_update4(portinfo,
 878                             inp, lport, sin, cred)) {
 879                                 error = 0;
 880                                 break;
 881                         }
 882                 }
 883         }
 884
 885 #undef IS_SELFCONNECT
 886
 887         if (error) {
 888                 /* Try next portinfo */
 889                 portinfo_idx++;
 890                 portinfo_idx &= pcbinfo->portinfo_mask;
 891                 if (portinfo_idx != portinfo_first)
 892                         goto loop;
 893                 inp->inp_laddr.s_addr = INADDR_ANY;
 894         }
 895         return error;
 896 }
 897
 898 /*
 899  *   Transform old in_pcbconnect() into an inner subroutine for new
 900  *   in_pcbconnect(): Do some validity-checking on the remote
 901  *   address (in mbuf 'nam') and then determine local host address
 902  *   (i.e., which interface) to use to access that remote host.
 903  *
 904  *   This preserves definition of in_pcbconnect(), while supporting a
 905  *   slightly different version for T/TCP.  (This is more than
 906  *   a bit of a kludge, but cleaning up the internal interfaces would
 907  *   have forced minor changes in every protocol).
 908  */
 909 int
 910 in_pcbladdr_find(struct inpcb *inp, struct sockaddr *nam,
 911     struct sockaddr_in **plocal_sin, struct thread *td, int find)
 912 {
 913         struct in_ifaddr *ia;
 914         struct ucred *cred = NULL;
 915         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 916         struct sockaddr *jsin;
 917         int jailed = 0, alloc_route = 0;
 918
 919         if (nam->sa_len != sizeof *sin)
 920                 return (EINVAL);
 921         if (sin->sin_family != AF_INET)
 922                 return (EAFNOSUPPORT);
 923         if (sin->sin_port == 0)
 924                 return (EADDRNOTAVAIL);
 925         if (td && td->td_proc && td->td_proc->p_ucred)
 926                 cred = td->td_proc->p_ucred;
 927         if (cred && cred->cr_prison)
 928                 jailed = 1;
 929         if (!TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) {
 930                 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia;
 931                 /*
 932                  * If the destination address is INADDR_ANY,
 933                  * use the primary local address.
 934                  * If the supplied address is INADDR_BROADCAST,
 935                  * and the primary interface supports broadcast,
 936                  * choose the broadcast address for that interface.
 937                  */
 938                 if (sin->sin_addr.s_addr == INADDR_ANY)
 939                         sin->sin_addr = IA_SIN(ia)->sin_addr;
 940                 else if (sin->sin_addr.s_addr == (u_long)INADDR_BROADCAST &&
 941                     (ia->ia_ifp->if_flags & IFF_BROADCAST))
 942                         sin->sin_addr = satosin(&ia->ia_broadaddr)->sin_addr;
 943         }
 944         if (find) {
 945                 struct route *ro;
 946
 947                 ia = NULL;
 948                 /*
 949                  * If route is known or can be allocated now,
 950                  * our src addr is taken from the i/f, else punt.
 951                  * Note that we should check the address family of the cached
 952                  * destination, in case of sharing the cache with IPv6.
 953                  */
 954                 ro = &inp->inp_route;
 955                 if (ro->ro_rt &&
 956                     (!(ro->ro_rt->rt_flags & RTF_UP) ||
 957                      ro->ro_dst.sa_family != AF_INET ||
 958                      satosin(&ro->ro_dst)->sin_addr.s_addr !=
 959                                       sin->sin_addr.s_addr ||
 960                      inp->inp_socket->so_options & SO_DONTROUTE)) {
 961                         RTFREE(ro->ro_rt);
 962                         ro->ro_rt = NULL;
 963                 }
 964                 if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/
 965                     (ro->ro_rt == NULL ||
 966                     ro->ro_rt->rt_ifp == NULL)) {
 967                         /* No route yet, so try to acquire one */
 968                         bzero(&ro->ro_dst, sizeof(struct sockaddr_in));
 969                         ro->ro_dst.sa_family = AF_INET;
 970                         ro->ro_dst.sa_len = sizeof(struct sockaddr_in);
 971                         ((struct sockaddr_in *) &ro->ro_dst)->sin_addr =
 972                                 sin->sin_addr;
 973                         rtalloc(ro);
 974                         alloc_route = 1;
 975                 }
 976                 /*
 977                  * If we found a route, use the address
 978                  * corresponding to the outgoing interface
 979                  * unless it is the loopback (in case a route
 980                  * to our address on another net goes to loopback).
 981                  */
 982                 if (ro->ro_rt &&
 983                     !(ro->ro_rt->rt_ifp->if_flags & IFF_LOOPBACK)) {
 984                         if (jailed) {
 985                                 if (jailed_ip(cred->cr_prison,
 986                                     ro->ro_rt->rt_ifa->ifa_addr)) {
 987                                         ia = ifatoia(ro->ro_rt->rt_ifa);
 988                                 }
 989                         } else {
 990                                 ia = ifatoia(ro->ro_rt->rt_ifa);
 991                         }
 992                 }
 993                 if (ia == NULL) {
 994                         u_short fport = sin->sin_port;
 995
 996                         sin->sin_port = 0;
 997                         ia = ifatoia(ifa_ifwithdstaddr(sintosa(sin)));
 998                         if (ia && jailed && !jailed_ip(cred->cr_prison,
 999                             sintosa(&ia->ia_addr)))
1000                                 ia = NULL;
1001                         if (ia == NULL)
1002                                 ia = ifatoia(ifa_ifwithnet(sintosa(sin)));
1003                         if (ia && jailed && !jailed_ip(cred->cr_prison,
1004                             sintosa(&ia->ia_addr)))
1005                                 ia = NULL;
1006                         sin->sin_port = fport;
1007                         if (ia == NULL &&
1008                             !TAILQ_EMPTY(&in_ifaddrheads[mycpuid]))
1009                                 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia;
1010                         if (ia && jailed && !jailed_ip(cred->cr_prison,
1011                             sintosa(&ia->ia_addr)))
1012                                 ia = NULL;
1013
1014                         if (!jailed && ia == NULL)
1015                                 goto fail;
1016                 }
1017                 /*
1018                  * If the destination address is multicast and an outgoing
1019                  * interface has been set as a multicast option, use the
1020                  * address of that interface as our source address.
1021                  */
1022                 if (!jailed && IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
1023                     inp->inp_moptions != NULL) {
1024                         struct ip_moptions *imo;
1025                         struct ifnet *ifp;
1026
1027                         imo = inp->inp_moptions;
1028                         if ((ifp = imo->imo_multicast_ifp) != NULL) {
1029                                 struct in_ifaddr_container *iac;
1030
1031                                 ia = NULL;
1032                                 TAILQ_FOREACH(iac,
1033                                 &in_ifaddrheads[mycpuid], ia_link) {
1034                                         if (iac->ia->ia_ifp == ifp) {
1035                                                 ia = iac->ia;
1036                                                 break;
1037                                         }
1038                                 }
1039                                 if (ia == NULL)
1040                                         goto fail;
1041                         }
1042                 }
1043                 /*
1044                  * Don't do pcblookup call here; return interface in plocal_sin
1045                  * and exit to caller, that will do the lookup.
1046                  */
1047                 if (ia == NULL && jailed) {
1048                         if ((jsin = prison_get_nonlocal(
1049                                 cred->cr_prison, AF_INET, NULL)) != NULL ||
1050                             (jsin = prison_get_local(
1051                                 cred->cr_prison, AF_INET, NULL)) != NULL) {
1052                                 *plocal_sin = satosin(jsin);
1053                         } else {
1054                                 /* IPv6 only Jail */
1055                                 goto fail;
1056                         }
1057                 } else {
1058                         *plocal_sin = &ia->ia_addr;
1059                 }
1060         }
1061         return (0);
1062 fail:
1063         if (alloc_route)
1064                 in_pcbresetroute(inp);
1065         return (EADDRNOTAVAIL);
1066 }
1067
1068 int
1069 in_pcbladdr(struct inpcb *inp, struct sockaddr *nam,
1070     struct sockaddr_in **plocal_sin, struct thread *td)
1071 {
1072         return in_pcbladdr_find(inp, nam, plocal_sin, td,
1073             (inp->inp_laddr.s_addr == INADDR_ANY));
1074 }
1075
1076 /*
1077  * Outer subroutine:
1078  * Connect from a socket to a specified address.
1079  * Both address and port must be specified in argument sin.
1080  * If don't have a local address for this socket yet,
1081  * then pick one.
1082  */
1083 int
1084 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
1085 {
1086         struct sockaddr_in *if_sin;
1087         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
1088         int error;
1089
1090         if_sin = NULL;  /* avoid gcc warnings */
1091
1092         /* Call inner routine to assign local interface address. */
1093         if ((error = in_pcbladdr(inp, nam, &if_sin, td)) != 0)
1094                 return (error);
1095
1096         if (in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port,
1097                               inp->inp_laddr.s_addr ?
1098                                 inp->inp_laddr : if_sin->sin_addr,
1099                               inp->inp_lport, FALSE, NULL) != NULL) {
1100                 return (EADDRINUSE);
1101         }
1102         if (inp->inp_laddr.s_addr == INADDR_ANY) {
1103                 if (inp->inp_lport == 0) {
1104                         error = in_pcbbind(inp, NULL, td);
1105                         if (error)
1106                                 return (error);
1107                 }
1108                 inp->inp_laddr = if_sin->sin_addr;
1109         }
1110         inp->inp_faddr = sin->sin_addr;
1111         inp->inp_fport = sin->sin_port;
1112         in_pcbinsconnhash(inp);
1113         return (0);
1114 }
1115
1116 void
1117 in_pcbdisconnect(struct inpcb *inp)
1118 {
1119
1120         in_pcbremconnhash(inp);
1121         inp->inp_faddr.s_addr = INADDR_ANY;
1122         inp->inp_fport = 0;
1123 }
1124
1125 void
1126 in_pcbdetach(struct inpcb *inp)
1127 {
1128         struct socket *so = inp->inp_socket;
1129         struct inpcbinfo *ipi = inp->inp_pcbinfo;
1130
1131 #ifdef IPSEC
1132         ipsec4_delete_pcbpolicy(inp);
1133 #endif /*IPSEC*/
1134         inp->inp_gencnt = ++ipi->ipi_gencnt;
1135         KKASSERT((so->so_state & SS_ASSERTINPROG) == 0);
1136         in_pcbremlists(inp);
1137         so->so_pcb = NULL;
1138         sofree(so);                     /* remove pcb ref */
1139         if (inp->inp_options)
1140                 m_free(inp->inp_options);
1141         if (inp->inp_route.ro_rt)
1142                 rtfree(inp->inp_route.ro_rt);
1143         ip_freemoptions(inp->inp_moptions);
1144         kfree(inp, M_PCB);
1145 }
1146
1147 /*
1148  * The socket may have an invalid PCB, i.e. NULL.  For example, a TCP
1149  * socket received RST.
1150  */
1151 static int
1152 in_setsockaddr(struct socket *so, struct sockaddr **nam)
1153 {
1154         struct inpcb *inp;
1155         struct sockaddr_in *sin;
1156
1157         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
1158         inp = so->so_pcb;
1159         if (!inp)
1160                 return (ECONNRESET);
1161
1162         sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1163         sin->sin_family = AF_INET;
1164         sin->sin_len = sizeof *sin;
1165         sin->sin_port = inp->inp_lport;
1166         sin->sin_addr = inp->inp_laddr;
1167
1168         *nam = (struct sockaddr *)sin;
1169         return (0);
1170 }
1171
1172 void
1173 in_setsockaddr_dispatch(netmsg_t msg)
1174 {
1175         int error;
1176
1177         error = in_setsockaddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1178         lwkt_replymsg(&msg->lmsg, error);
1179 }
1180
1181 /*
1182  * The socket may have an invalid PCB, i.e. NULL.  For example, a TCP
1183  * socket received RST.
1184  */
1185 int
1186 in_setpeeraddr(struct socket *so, struct sockaddr **nam)
1187 {
1188         struct inpcb *inp;
1189         struct sockaddr_in *sin;
1190
1191         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
1192         inp = so->so_pcb;
1193         if (!inp)
1194                 return (ECONNRESET);
1195
1196         sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1197         sin->sin_family = AF_INET;
1198         sin->sin_len = sizeof *sin;
1199         sin->sin_port = inp->inp_fport;
1200         sin->sin_addr = inp->inp_faddr;
1201
1202         *nam = (struct sockaddr *)sin;
1203         return (0);
1204 }
1205
1206 void
1207 in_setpeeraddr_dispatch(netmsg_t msg)
1208 {
1209         int error;
1210
1211         error = in_setpeeraddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1212         lwkt_replymsg(&msg->lmsg, error);
1213 }
1214
1215 void
1216 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int err,
1217     inp_notify_t notify)
1218 {
1219         struct inpcb *inp, *marker;
1220
1221         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1222             ("not in the correct netisr"));
1223         marker = &in_pcbmarkers[mycpuid];
1224
1225         /*
1226          * NOTE:
1227          * - If INP_PLACEMARKER is set we must ignore the rest of the
1228          *   structure and skip it.
1229          * - It is safe to nuke inpcbs here, since we are in their own
1230          *   netisr.
1231          */
1232         GET_PCBINFO_TOKEN(pcbinfo);
1233
1234         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1235         while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1236                 LIST_REMOVE(marker, inp_list);
1237                 LIST_INSERT_AFTER(inp, marker, inp_list);
1238
1239                 if (inp->inp_flags & INP_PLACEMARKER)
1240                         continue;
1241 #ifdef INET6
1242                 if (!INP_ISIPV4(inp))
1243                         continue;
1244 #endif
1245                 if (inp->inp_faddr.s_addr != faddr.s_addr ||
1246                     inp->inp_socket == NULL)
1247                         continue;
1248                 (*notify)(inp, err);            /* can remove inp from list! */
1249         }
1250         LIST_REMOVE(marker, inp_list);
1251
1252         REL_PCBINFO_TOKEN(pcbinfo);
1253 }
1254
1255 void
1256 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1257 {
1258         struct inpcb *inp, *marker;
1259
1260         /*
1261          * We only need to make sure that we are in netisr0, where all
1262          * multicast operation happen.  We could check inpcbinfo which
1263          * does not belong to netisr0 by holding the inpcbinfo's token.
1264          * In this case, the pcbinfo must be able to be shared, i.e.
1265          * pcbinfo->infotoken is not NULL.
1266          */
1267         ASSERT_IN_NETISR(0);
1268         KASSERT(pcbinfo->cpu == 0 || pcbinfo->infotoken != NULL,
1269             ("pcbinfo could not be shared"));
1270
1271         /*
1272          * Get a marker for the current netisr (netisr0).
1273          *
1274          * It is possible that the multicast address deletion blocks,
1275          * which could cause temporary token releasing.  So we use
1276          * inpcb marker here to get a coherent view of the inpcb list.
1277          *
1278          * While, on the other hand, moptions are only added and deleted
1279          * in netisr0, so we would not see staled moption or miss moption
1280          * even if the token was released due to the blocking multicast
1281          * address deletion.
1282          */
1283         marker = &in_pcbmarkers[mycpuid];
1284
1285         GET_PCBINFO_TOKEN(pcbinfo);
1286
1287         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1288         while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1289                 struct ip_moptions *imo;
1290
1291                 LIST_REMOVE(marker, inp_list);
1292                 LIST_INSERT_AFTER(inp, marker, inp_list);
1293
1294                 if (inp->inp_flags & INP_PLACEMARKER)
1295                         continue;
1296                 imo = inp->inp_moptions;
1297                 if (INP_ISIPV4(inp) && imo != NULL) {
1298                         int i, gap;
1299
1300                         /*
1301                          * Unselect the outgoing interface if it is being
1302                          * detached.
1303                          */
1304                         if (imo->imo_multicast_ifp == ifp)
1305                                 imo->imo_multicast_ifp = NULL;
1306
1307                         /*
1308                          * Drop multicast group membership if we joined
1309                          * through the interface being detached.
1310                          */
1311                         for (i = 0, gap = 0; i < imo->imo_num_memberships;
1312                             i++) {
1313                                 if (imo->imo_membership[i]->inm_ifp == ifp) {
1314                                         /*
1315                                          * NOTE:
1316                                          * This could block and the pcbinfo
1317                                          * token could be passively released.
1318                                          */
1319                                         in_delmulti(imo->imo_membership[i]);
1320                                         gap++;
1321                                 } else if (gap != 0)
1322                                         imo->imo_membership[i - gap] =
1323                                             imo->imo_membership[i];
1324                         }
1325                         imo->imo_num_memberships -= gap;
1326                 }
1327         }
1328         LIST_REMOVE(marker, inp_list);
1329
1330         REL_PCBINFO_TOKEN(pcbinfo);
1331 }
1332
1333 /*
1334  * Check for alternatives when higher level complains
1335  * about service problems.  For now, invalidate cached
1336  * routing information.  If the route was created dynamically
1337  * (by a redirect), time to try a default gateway again.
1338  */
1339 void
1340 in_losing(struct inpcb *inp)
1341 {
1342         struct rtentry *rt;
1343         struct rt_addrinfo rtinfo;
1344
1345         if ((rt = inp->inp_route.ro_rt)) {
1346                 bzero(&rtinfo, sizeof(struct rt_addrinfo));
1347                 rtinfo.rti_info[RTAX_DST] = rt_key(rt);
1348                 rtinfo.rti_info[RTAX_GATEWAY] = rt->rt_gateway;
1349                 rtinfo.rti_info[RTAX_NETMASK] = rt_mask(rt);
1350                 rtinfo.rti_flags = rt->rt_flags;
1351                 rt_missmsg(RTM_LOSING, &rtinfo, rt->rt_flags, 0);
1352                 if (rt->rt_flags & RTF_DYNAMIC) {
1353                         rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway,
1354                             rt_mask(rt), rt->rt_flags, NULL);
1355                 }
1356                 inp->inp_route.ro_rt = NULL;
1357                 rtfree(rt);
1358                 /*
1359                  * A new route can be allocated
1360                  * the next time output is attempted.
1361                  */
1362         }
1363 }
1364
1365 /*
1366  * After a routing change, flush old routing
1367  * and allocate a (hopefully) better one.
1368  */
1369 void
1370 in_rtchange(struct inpcb *inp, int err)
1371 {
1372         if (inp->inp_route.ro_rt) {
1373                 rtfree(inp->inp_route.ro_rt);
1374                 inp->inp_route.ro_rt = NULL;
1375                 /*
1376                  * A new route can be allocated the next time
1377                  * output is attempted.
1378                  */
1379         }
1380 }
1381
1382 /*
1383  * Lookup a PCB based on the local address and port.
1384  */
1385 static struct inpcb *
1386 in_pcblookup_local(struct inpcbporthead *porthash, struct in_addr laddr,
1387                    u_int lport_arg, int wild_okay, struct ucred *cred)
1388 {
1389         struct inpcb *inp;
1390         int matchwild = 3, wildcard;
1391         u_short lport = lport_arg;
1392         struct inpcbport *phd;
1393         struct inpcb *match = NULL;
1394
1395         /*
1396          * If the porthashbase is shared across several cpus, it must
1397          * have been locked.
1398          */
1399         ASSERT_PORTHASH_TOKEN_HELD(porthash);
1400
1401         /*
1402          * Best fit PCB lookup.
1403          *
1404          * First see if this local port is in use by looking on the
1405          * port hash list.
1406          */
1407         LIST_FOREACH(phd, porthash, phd_hash) {
1408                 if (phd->phd_port == lport)
1409                         break;
1410         }
1411         if (phd != NULL) {
1412                 /*
1413                  * Port is in use by one or more PCBs. Look for best
1414                  * fit.
1415                  */
1416                 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
1417                         wildcard = 0;
1418 #ifdef INET6
1419                         if (!INP_ISIPV4(inp))
1420                                 continue;
1421 #endif
1422                         if (inp->inp_faddr.s_addr != INADDR_ANY)
1423                                 wildcard++;
1424                         if (inp->inp_laddr.s_addr != INADDR_ANY) {
1425                                 if (laddr.s_addr == INADDR_ANY)
1426                                         wildcard++;
1427                                 else if (inp->inp_laddr.s_addr != laddr.s_addr)
1428                                         continue;
1429                         } else {
1430                                 if (laddr.s_addr != INADDR_ANY)
1431                                         wildcard++;
1432                         }
1433                         if (wildcard && !wild_okay)
1434                                 continue;
1435                         if (wildcard < matchwild &&
1436                             (cred == NULL ||
1437                              cred->cr_prison ==
1438                                         inp->inp_socket->so_cred->cr_prison)) {
1439                                 match = inp;
1440                                 matchwild = wildcard;
1441                                 if (matchwild == 0) {
1442                                         break;
1443                                 }
1444                         }
1445                 }
1446         }
1447         return (match);
1448 }
1449
1450 struct inpcb *
1451 in_pcblocalgroup_last(const struct inpcbinfo *pcbinfo,
1452     const struct inpcb *inp)
1453 {
1454         const struct inp_localgrphead *hdr;
1455         const struct inp_localgroup *grp;
1456         int i;
1457
1458         if (pcbinfo->localgrphashbase == NULL)
1459                 return NULL;
1460
1461         GET_PCBINFO_TOKEN(pcbinfo);
1462
1463         hdr = &pcbinfo->localgrphashbase[
1464             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
1465
1466         LIST_FOREACH(grp, hdr, il_list) {
1467                 if (grp->il_af == inp->inp_af &&
1468                     grp->il_lport == inp->inp_lport &&
1469                     memcmp(&grp->il_dependladdr,
1470                         &inp->inp_inc.inc_ie.ie_dependladdr,
1471                         sizeof(grp->il_dependladdr)) == 0) {
1472                         break;
1473                 }
1474         }
1475         if (grp == NULL || grp->il_inpcnt == 1) {
1476                 REL_PCBINFO_TOKEN(pcbinfo);
1477                 return NULL;
1478         }
1479
1480         KASSERT(grp->il_inpcnt >= 2,
1481             ("invalid localgroup inp count %d", grp->il_inpcnt));
1482         for (i = 0; i < grp->il_inpcnt; ++i) {
1483                 if (grp->il_inp[i] == inp) {
1484                         int last = grp->il_inpcnt - 1;
1485
1486                         if (i == last)
1487                                 last = grp->il_inpcnt - 2;
1488                         REL_PCBINFO_TOKEN(pcbinfo);
1489                         return grp->il_inp[last];
1490                 }
1491         }
1492         REL_PCBINFO_TOKEN(pcbinfo);
1493         return NULL;
1494 }
1495
1496 static struct inpcb *
1497 inp_localgroup_lookup(const struct inpcbinfo *pcbinfo,
1498     struct in_addr laddr, uint16_t lport, uint32_t pkt_hash)
1499 {
1500         struct inpcb *local_wild = NULL;
1501         const struct inp_localgrphead *hdr;
1502         const struct inp_localgroup *grp;
1503
1504         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1505
1506         hdr = &pcbinfo->localgrphashbase[
1507             INP_PCBLOCALGRPHASH(lport, pcbinfo->localgrphashmask)];
1508
1509         /*
1510          * Order of socket selection:
1511          * 1. non-wild.
1512          * 2. wild.
1513          *
1514          * NOTE: Local group does not contain jailed sockets
1515          */
1516         LIST_FOREACH(grp, hdr, il_list) {
1517 #ifdef INET6
1518                 if (grp->il_af != AF_INET)
1519                         continue;
1520 #endif
1521                 if (grp->il_lport == lport) {
1522                         int idx;
1523
1524                         /*
1525                          * Modulo-N is used here, which greatly reduces
1526                          * completion queue token contention, thus more
1527                          * cpu time is saved.
1528                          */
1529                         idx = pkt_hash % grp->il_inpcnt;
1530                         if (grp->il_laddr.s_addr == laddr.s_addr)
1531                                 return grp->il_inp[idx];
1532                         else if (grp->il_laddr.s_addr == INADDR_ANY)
1533                                 local_wild = grp->il_inp[idx];
1534                 }
1535         }
1536         if (local_wild != NULL)
1537                 return local_wild;
1538         return NULL;
1539 }
1540
1541 /*
1542  * Lookup PCB in hash list.
1543  */
1544 struct inpcb *
1545 in_pcblookup_pkthash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1546     u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1547     boolean_t wildcard, struct ifnet *ifp, const struct mbuf *m)
1548 {
1549         struct inpcbhead *head;
1550         struct inpcb *inp, *jinp=NULL;
1551         u_short fport = fport_arg, lport = lport_arg;
1552
1553         /*
1554          * First look for an exact match.
1555          */
1556         head = &pcbinfo->hashbase[INP_PCBCONNHASH(faddr.s_addr, fport,
1557             laddr.s_addr, lport, pcbinfo->hashmask)];
1558         LIST_FOREACH(inp, head, inp_hash) {
1559 #ifdef INET6
1560                 if (!INP_ISIPV4(inp))
1561                         continue;
1562 #endif
1563                 if (in_hosteq(inp->inp_faddr, faddr) &&
1564                     in_hosteq(inp->inp_laddr, laddr) &&
1565                     inp->inp_fport == fport && inp->inp_lport == lport) {
1566                         /* found */
1567                         if (inp->inp_socket == NULL ||
1568                             inp->inp_socket->so_cred->cr_prison == NULL) {
1569                                 return (inp);
1570                         } else {
1571                                 if  (jinp == NULL)
1572                                         jinp = inp;
1573                         }
1574                 }
1575         }
1576         if (jinp != NULL)
1577                 return (jinp);
1578
1579         if (wildcard) {
1580                 struct inpcb *local_wild = NULL;
1581                 struct inpcb *jinp_wild = NULL;
1582                 struct inpcontainer *ic;
1583                 struct inpcontainerhead *chead;
1584                 struct sockaddr_in jsin;
1585                 struct ucred *cred;
1586
1587                 GET_PCBINFO_TOKEN(pcbinfo);
1588
1589                 /*
1590                  * Check local group first
1591                  */
1592                 if (pcbinfo->localgrphashbase != NULL &&
1593                     m != NULL && (m->m_flags & M_HASH) &&
1594                     !(ifp && ifp->if_type == IFT_FAITH)) {
1595                         inp = inp_localgroup_lookup(pcbinfo,
1596                             laddr, lport, m->m_pkthdr.hash);
1597                         if (inp != NULL) {
1598                                 REL_PCBINFO_TOKEN(pcbinfo);
1599                                 return inp;
1600                         }
1601                 }
1602
1603                 /*
1604                  * Order of socket selection:
1605                  * 1. non-jailed, non-wild.
1606                  * 2. non-jailed, wild.
1607                  * 3. jailed, non-wild.
1608                  * 4. jailed, wild.
1609                  */
1610                 jsin.sin_family = AF_INET;
1611                 chead = &pcbinfo->wildcardhashbase[
1612                     INP_PCBWILDCARDHASH(lport, pcbinfo->wildcardhashmask)];
1613                 LIST_FOREACH(ic, chead, ic_list) {
1614                         inp = ic->ic_inp;
1615                         if (inp->inp_flags & INP_PLACEMARKER)
1616                                 continue;
1617
1618                         jsin.sin_addr.s_addr = laddr.s_addr;
1619 #ifdef INET6
1620                         if (!INP_ISIPV4(inp))
1621                                 continue;
1622 #endif
1623                         if (inp->inp_socket != NULL)
1624                                 cred = inp->inp_socket->so_cred;
1625                         else
1626                                 cred = NULL;
1627                         if (cred != NULL && jailed(cred)) {
1628                                 if (jinp != NULL)
1629                                         continue;
1630                                 else
1631                                         if (!jailed_ip(cred->cr_prison,
1632                                             (struct sockaddr *)&jsin))
1633                                                 continue;
1634                         }
1635                         if (inp->inp_lport == lport) {
1636                                 if (ifp && ifp->if_type == IFT_FAITH &&
1637                                     !(inp->inp_flags & INP_FAITH))
1638                                         continue;
1639                                 if (inp->inp_laddr.s_addr == laddr.s_addr) {
1640                                         if (cred != NULL && jailed(cred)) {
1641                                                 jinp = inp;
1642                                         } else {
1643                                                 REL_PCBINFO_TOKEN(pcbinfo);
1644                                                 return (inp);
1645                                         }
1646                                 }
1647                                 if (inp->inp_laddr.s_addr == INADDR_ANY) {
1648                                         if (cred != NULL && jailed(cred))
1649                                                 jinp_wild = inp;
1650                                         else
1651                                                 local_wild = inp;
1652                                 }
1653                         }
1654                 }
1655
1656                 REL_PCBINFO_TOKEN(pcbinfo);
1657
1658                 if (local_wild != NULL)
1659                         return (local_wild);
1660                 if (jinp != NULL)
1661                         return (jinp);
1662                 return (jinp_wild);
1663         }
1664
1665         /*
1666          * Not found.
1667          */
1668         return (NULL);
1669 }
1670
1671 struct inpcb *
1672 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1673     u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1674     boolean_t wildcard, struct ifnet *ifp)
1675 {
1676         return in_pcblookup_pkthash(pcbinfo, faddr, fport_arg,
1677             laddr, lport_arg, wildcard, ifp, NULL);
1678 }
1679
1680 /*
1681  * Insert PCB into connection hash table.
1682  */
1683 void
1684 in_pcbinsconnhash(struct inpcb *inp)
1685 {
1686         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1687         struct inpcbhead *bucket;
1688         u_int32_t hashkey_faddr, hashkey_laddr;
1689
1690 #ifdef INET6
1691         if (INP_ISIPV6(inp)) {
1692                 hashkey_faddr = inp->in6p_faddr.s6_addr32[3] /* XXX JH */;
1693                 hashkey_laddr = inp->in6p_laddr.s6_addr32[3] /* XXX JH */;
1694         } else {
1695 #endif
1696                 hashkey_faddr = inp->inp_faddr.s_addr;
1697                 hashkey_laddr = inp->inp_laddr.s_addr;
1698 #ifdef INET6
1699         }
1700 #endif
1701
1702         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1703             ("not in the correct netisr"));
1704         ASSERT_INP_NOTINHASH(inp);
1705         inp->inp_flags |= INP_CONNECTED;
1706
1707         /*
1708          * Insert into the connection hash table.
1709          */
1710         bucket = &pcbinfo->hashbase[INP_PCBCONNHASH(hashkey_faddr,
1711             inp->inp_fport, hashkey_laddr, inp->inp_lport, pcbinfo->hashmask)];
1712         LIST_INSERT_HEAD(bucket, inp, inp_hash);
1713 }
1714
1715 /*
1716  * Remove PCB from connection hash table.
1717  */
1718 void
1719 in_pcbremconnhash(struct inpcb *inp)
1720 {
1721         struct inpcbinfo *pcbinfo __debugvar = inp->inp_pcbinfo;
1722
1723         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1724             ("not in the correct netisr"));
1725         KASSERT(inp->inp_flags & INP_CONNECTED, ("inp not connected"));
1726
1727         LIST_REMOVE(inp, inp_hash);
1728         inp->inp_flags &= ~INP_CONNECTED;
1729 }
1730
1731 /*
1732  * Insert PCB into port hash table.
1733  */
1734 void
1735 in_pcbinsporthash(struct inpcbporthead *pcbporthash, struct inpcb *inp)
1736 {
1737         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1738         struct inpcbport *phd;
1739
1740         /*
1741          * If the porthashbase is shared across several cpus, it must
1742          * have been locked.
1743          */
1744         ASSERT_PORTHASH_TOKEN_HELD(pcbporthash);
1745
1746         /*
1747          * Insert into the port hash table.
1748          */
1749
1750         /* Go through port list and look for a head for this lport. */
1751         LIST_FOREACH(phd, pcbporthash, phd_hash) {
1752                 if (phd->phd_port == inp->inp_lport)
1753                         break;
1754         }
1755
1756         /* If none exists, use saved one and tack it on. */
1757         if (phd == NULL) {
1758                 KKASSERT(pcbinfo->portsave != NULL);
1759                 phd = pcbinfo->portsave;
1760                 pcbinfo->portsave = NULL;
1761                 phd->phd_port = inp->inp_lport;
1762                 LIST_INIT(&phd->phd_pcblist);
1763                 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
1764         }
1765
1766         inp->inp_porthash = pcbporthash;
1767         inp->inp_phd = phd;
1768         LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
1769
1770         /*
1771          * Malloc one inpcbport for later use.  It is safe to use
1772          * "wait" malloc here (port token would be released, if
1773          * malloc ever blocked), since all changes to the porthash
1774          * are done.
1775          */
1776         if (pcbinfo->portsave == NULL) {
1777                 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave),
1778                                             M_PCB, M_INTWAIT | M_ZERO);
1779         }
1780 }
1781
1782 void
1783 in_pcbinsporthash_lport(struct inpcb *inp)
1784 {
1785         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1786         struct inpcbportinfo *portinfo;
1787         struct inpcbporthead *porthash;
1788         u_short lport_ho;
1789
1790         /* Locate the proper portinfo based on lport */
1791         lport_ho = ntohs(inp->inp_lport);
1792         portinfo = &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask];
1793         KKASSERT((lport_ho & pcbinfo->portinfo_mask) == portinfo->offset);
1794
1795         porthash = in_pcbporthash_head(portinfo, inp->inp_lport);
1796         GET_PORTHASH_TOKEN(porthash);
1797         in_pcbinsporthash(porthash, inp);
1798         REL_PORTHASH_TOKEN(porthash);
1799 }
1800
1801 void
1802 in_pcbremporthash(struct inpcb *inp)
1803 {
1804         struct inpcbporthead *porthash;
1805         struct inpcbport *phd;
1806
1807         if (inp->inp_phd == NULL)
1808                 return;
1809         KASSERT(inp->inp_lport != 0, ("inpcb has no lport"));
1810
1811         porthash = inp->inp_porthash;
1812         KASSERT(porthash != NULL, ("no porthash"));
1813
1814         GET_PORTHASH_TOKEN(porthash);
1815
1816         phd = inp->inp_phd;
1817         LIST_REMOVE(inp, inp_portlist);
1818         if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
1819                 LIST_REMOVE(phd, phd_hash);
1820                 kfree(phd, M_PCB);
1821         }
1822
1823         REL_PORTHASH_TOKEN(porthash);
1824
1825         inp->inp_phd = NULL;
1826         /* NOTE: Don't whack inp_lport, which may be used later */
1827 }
1828
1829 static struct inp_localgroup *
1830 inp_localgroup_alloc(u_char af, uint16_t port,
1831     const union in_dependaddr *addr, int size)
1832 {
1833         struct inp_localgroup *grp;
1834
1835         grp = kmalloc(__offsetof(struct inp_localgroup, il_inp[size]),
1836             M_TEMP, M_INTWAIT | M_ZERO);
1837         grp->il_af = af;
1838         grp->il_lport = port;
1839         grp->il_dependladdr = *addr;
1840         grp->il_inpsiz = size;
1841
1842         return grp;
1843 }
1844
1845 static void
1846 inp_localgroup_free(struct inp_localgroup *grp)
1847 {
1848         kfree(grp, M_TEMP);
1849 }
1850
1851 static void
1852 inp_localgroup_destroy(struct inp_localgroup *grp)
1853 {
1854         LIST_REMOVE(grp, il_list);
1855         inp_localgroup_free(grp);
1856 }
1857
1858 static void
1859 inp_localgroup_copy(struct inp_localgroup *grp,
1860     const struct inp_localgroup *old_grp)
1861 {
1862         int i;
1863
1864         KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
1865             ("invalid new local group size %d and old local group count %d",
1866              grp->il_inpsiz, old_grp->il_inpcnt));
1867         for (i = 0; i < old_grp->il_inpcnt; ++i)
1868                 grp->il_inp[i] = old_grp->il_inp[i];
1869         grp->il_inpcnt = old_grp->il_inpcnt;
1870 }
1871
1872 static void
1873 in_pcbinslocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
1874 {
1875         struct inp_localgrphead *hdr;
1876         struct inp_localgroup *grp, *grp_alloc = NULL;
1877         struct ucred *cred;
1878         int i, idx;
1879
1880         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1881
1882         if (pcbinfo->localgrphashbase == NULL)
1883                 return;
1884
1885         /*
1886          * XXX don't allow jailed socket to join local group
1887          */
1888         if (inp->inp_socket != NULL)
1889                 cred = inp->inp_socket->so_cred;
1890         else
1891                 cred = NULL;
1892         if (cred != NULL && jailed(cred))
1893                 return;
1894
1895         hdr = &pcbinfo->localgrphashbase[
1896             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
1897
1898 again:
1899         LIST_FOREACH(grp, hdr, il_list) {
1900                 if (grp->il_af == inp->inp_af &&
1901                     grp->il_lport == inp->inp_lport &&
1902                     memcmp(&grp->il_dependladdr,
1903                         &inp->inp_inc.inc_ie.ie_dependladdr,
1904                         sizeof(grp->il_dependladdr)) == 0) {
1905                         break;
1906                 }
1907         }
1908         if (grp == NULL) {
1909                 /*
1910                  * Create a new local group
1911                  */
1912                 if (grp_alloc == NULL) {
1913                         grp_alloc = inp_localgroup_alloc(inp->inp_af,
1914                             inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
1915                             INP_LOCALGROUP_SIZMIN);
1916                         /*
1917                          * Local group allocation could block and the
1918                          * local group w/ the same property might have
1919                          * been added by others when we were blocked;
1920                          * check again.
1921                          */
1922                         goto again;
1923                 } else {
1924                         /* Local group has been allocated; link it */
1925                         grp = grp_alloc;
1926                         grp_alloc = NULL;
1927                         LIST_INSERT_HEAD(hdr, grp, il_list);
1928                 }
1929         } else if (grp->il_inpcnt == grp->il_inpsiz) {
1930                 if (grp->il_inpsiz >= INP_LOCALGROUP_SIZMAX) {
1931                         static int limit_logged = 0;
1932
1933                         if (!limit_logged) {
1934                                 limit_logged = 1;
1935                                 kprintf("local group port %d, "
1936                                     "limit reached\n", ntohs(grp->il_lport));
1937                         }
1938                         if (grp_alloc != NULL) {
1939                                 /*
1940                                  * This would happen if the local group
1941                                  * w/ the same property was expanded when
1942                                  * our local group allocation blocked.
1943                                  */
1944                                 inp_localgroup_free(grp_alloc);
1945                         }
1946                         return;
1947                 }
1948
1949                 /*
1950                  * Expand this local group
1951                  */
1952                 if (grp_alloc == NULL ||
1953                     grp->il_inpcnt >= grp_alloc->il_inpsiz) {
1954                         if (grp_alloc != NULL)
1955                                 inp_localgroup_free(grp_alloc);
1956                         grp_alloc = inp_localgroup_alloc(grp->il_af,
1957                             grp->il_lport, &grp->il_dependladdr,
1958                             grp->il_inpsiz * 2);
1959                         /*
1960                          * Local group allocation could block and the
1961                          * local group w/ the same property might have
1962                          * been expanded by others when we were blocked;
1963                          * check again.
1964                          */
1965                         goto again;
1966                 }
1967
1968                 /*
1969                  * Save the old local group, link the new one, and then
1970                  * destroy the old local group
1971                  */
1972                 inp_localgroup_copy(grp_alloc, grp);
1973                 LIST_INSERT_HEAD(hdr, grp_alloc, il_list);
1974                 inp_localgroup_destroy(grp);
1975
1976                 grp = grp_alloc;
1977                 grp_alloc = NULL;
1978         } else {
1979                 /*
1980                  * Found the local group
1981                  */
1982                 if (grp_alloc != NULL) {
1983                         /*
1984                          * This would happen if the local group w/ the
1985                          * same property was added or expanded when our
1986                          * local group allocation blocked.
1987                          */
1988                         inp_localgroup_free(grp_alloc);
1989                         grp_alloc = NULL;
1990                 }
1991         }
1992
1993         KASSERT(grp->il_inpcnt < grp->il_inpsiz,
1994             ("invalid local group size %d and count %d",
1995              grp->il_inpsiz, grp->il_inpcnt));
1996
1997         /*
1998          * Keep the local group sorted by the inpcb local group index
1999          * in ascending order.
2000          *
2001          * This eases the multi-process userland application which uses
2002          * SO_REUSEPORT sockets and binds process to the owner cpu of
2003          * the SO_REUSEPORT socket:
2004          * If we didn't sort the local group by the inpcb local group
2005          * index and one of the process owning an inpcb in this local
2006          * group restarted, e.g. crashed and restarted by watchdog,
2007          * other processes owning a inpcb in this local group would have
2008          * to detect that event, refetch its socket's owner cpu, and
2009          * re-bind.
2010          */
2011         idx = grp->il_inpcnt;
2012         for (i = 0; i < idx; ++i) {
2013                 struct inpcb *oinp = grp->il_inp[i];
2014
2015                 if (oinp->inp_lgrpindex > i) {
2016                         if (inp->inp_lgrpindex < 0) {
2017                                 inp->inp_lgrpindex = i;
2018                         } else if (inp->inp_lgrpindex != i) {
2019                                 if (bootverbose) {
2020                                         kprintf("inp %p: grpidx %d, "
2021                                             "assigned to %d, cpu%d\n",
2022                                             inp, inp->inp_lgrpindex, i,
2023                                             mycpuid);
2024                                 }
2025                         }
2026                         grp->il_inp[i] = inp;
2027
2028                         /* Pull down inpcbs */
2029                         for (; i < grp->il_inpcnt; ++i) {
2030                                 struct inpcb *oinp1 = grp->il_inp[i + 1];
2031
2032                                 grp->il_inp[i + 1] = oinp;
2033                                 oinp = oinp1;
2034                         }
2035                         grp->il_inpcnt++;
2036                         return;
2037                 }
2038         }
2039
2040         if (inp->inp_lgrpindex < 0) {
2041                 inp->inp_lgrpindex = idx;
2042         } else if (inp->inp_lgrpindex != idx) {
2043                 if (bootverbose) {
2044                         kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n",
2045                             inp, inp->inp_lgrpindex, idx, mycpuid);
2046                 }
2047         }
2048         grp->il_inp[idx] = inp;
2049         grp->il_inpcnt++;
2050 }
2051
2052 void
2053 in_pcbinswildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2054 {
2055         struct inpcontainer *ic;
2056         struct inpcontainerhead *bucket;
2057
2058         GET_PCBINFO_TOKEN(pcbinfo);
2059
2060         in_pcbinslocalgrphash_oncpu(inp, pcbinfo);
2061
2062         bucket = &pcbinfo->wildcardhashbase[
2063             INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
2064
2065         ic = kmalloc(sizeof(struct inpcontainer), M_TEMP, M_INTWAIT);
2066         ic->ic_inp = inp;
2067         LIST_INSERT_HEAD(bucket, ic, ic_list);
2068
2069         REL_PCBINFO_TOKEN(pcbinfo);
2070 }
2071
2072 /*
2073  * Insert PCB into wildcard hash table.
2074  */
2075 void
2076 in_pcbinswildcardhash(struct inpcb *inp)
2077 {
2078         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2079
2080         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
2081             ("not in correct netisr"));
2082         ASSERT_INP_NOTINHASH(inp);
2083         inp->inp_flags |= INP_WILDCARD;
2084
2085         in_pcbinswildcardhash_oncpu(inp, pcbinfo);
2086 }
2087
2088 static void
2089 in_pcbremlocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2090 {
2091         struct inp_localgrphead *hdr;
2092         struct inp_localgroup *grp;
2093
2094         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
2095
2096         if (pcbinfo->localgrphashbase == NULL)
2097                 return;
2098
2099         hdr = &pcbinfo->localgrphashbase[
2100             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
2101
2102         LIST_FOREACH(grp, hdr, il_list) {
2103                 int i;
2104
2105                 for (i = 0; i < grp->il_inpcnt; ++i) {
2106                         if (grp->il_inp[i] != inp)
2107                                 continue;
2108
2109                         if (grp->il_inpcnt == 1) {
2110                                 /* Destroy this local group */
2111                                 inp_localgroup_destroy(grp);
2112                         } else {
2113                                 /* Pull up inpcbs */
2114                                 for (; i + 1 < grp->il_inpcnt; ++i)
2115                                         grp->il_inp[i] = grp->il_inp[i + 1];
2116                                 grp->il_inpcnt--;
2117                         }
2118                         return;
2119                 }
2120         }
2121 }
2122
2123 void
2124 in_pcbremwildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2125 {
2126         struct inpcontainer *ic;
2127         struct inpcontainerhead *head;
2128
2129         GET_PCBINFO_TOKEN(pcbinfo);
2130
2131         in_pcbremlocalgrphash_oncpu(inp, pcbinfo);
2132
2133         /* find bucket */
2134         head = &pcbinfo->wildcardhashbase[
2135             INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
2136
2137         LIST_FOREACH(ic, head, ic_list) {
2138                 if (ic->ic_inp == inp)
2139                         goto found;
2140         }
2141         REL_PCBINFO_TOKEN(pcbinfo);
2142         return;                 /* not found! */
2143
2144 found:
2145         LIST_REMOVE(ic, ic_list);       /* remove container from bucket chain */
2146         REL_PCBINFO_TOKEN(pcbinfo);
2147         kfree(ic, M_TEMP);              /* deallocate container */
2148 }
2149
2150 /*
2151  * Remove PCB from wildcard hash table.
2152  */
2153 void
2154 in_pcbremwildcardhash(struct inpcb *inp)
2155 {
2156         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2157
2158         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
2159             ("not in correct netisr"));
2160         KASSERT(inp->inp_flags & INP_WILDCARD, ("inp not wildcard"));
2161
2162         in_pcbremwildcardhash_oncpu(inp, pcbinfo);
2163         inp->inp_lgrpindex = -1;
2164         inp->inp_flags &= ~INP_WILDCARD;
2165 }
2166
2167 /*
2168  * Remove PCB from various lists.
2169  */
2170 void
2171 in_pcbremlists(struct inpcb *inp)
2172 {
2173         in_pcbremporthash(inp);
2174         if (inp->inp_flags & INP_WILDCARD) {
2175                 in_pcbremwildcardhash(inp);
2176         } else if (inp->inp_flags & INP_CONNECTED) {
2177                 in_pcbremconnhash(inp);
2178         }
2179
2180         if (inp->inp_flags & INP_ONLIST)
2181                 in_pcbofflist(inp);
2182 }
2183
2184 int
2185 prison_xinpcb(struct thread *td, struct inpcb *inp)
2186 {
2187         struct ucred *cr;
2188
2189         if (td->td_proc == NULL)
2190                 return (0);
2191         cr = td->td_proc->p_ucred;
2192         if (cr->cr_prison == NULL)
2193                 return (0);
2194         if (inp->inp_socket && inp->inp_socket->so_cred &&
2195             inp->inp_socket->so_cred->cr_prison &&
2196             cr->cr_prison == inp->inp_socket->so_cred->cr_prison)
2197                 return (0);
2198         return (1);
2199 }
2200
2201 int
2202 in_pcblist_global(SYSCTL_HANDLER_ARGS)
2203 {
2204         struct inpcbinfo *pcbinfo_arr = arg1;
2205         int pcbinfo_arrlen = arg2;
2206         struct inpcb *marker;
2207         int cpu, origcpu;
2208         int error, n;
2209
2210         KASSERT(pcbinfo_arrlen <= ncpus && pcbinfo_arrlen >= 1,
2211             ("invalid pcbinfo count %d", pcbinfo_arrlen));
2212
2213         /*
2214          * The process of preparing the TCB list is too time-consuming and
2215          * resource-intensive to repeat twice on every request.
2216          */
2217         n = 0;
2218         if (req->oldptr == NULL) {
2219                 for (cpu = 0; cpu < pcbinfo_arrlen; ++cpu)
2220                         n += pcbinfo_arr[cpu].ipi_count;
2221                 req->oldidx = (n + n/8 + 10) * sizeof(struct xinpcb);
2222                 return 0;
2223         }
2224
2225         if (req->newptr != NULL)
2226                 return EPERM;
2227
2228         marker = kmalloc(sizeof(struct inpcb), M_TEMP, M_WAITOK|M_ZERO);
2229         marker->inp_flags |= INP_PLACEMARKER;
2230
2231         /*
2232          * OK, now we're committed to doing something.  Re-fetch ipi_count
2233          * after obtaining the generation count.
2234          */
2235         error = 0;
2236         origcpu = mycpuid;
2237         for (cpu = 0; cpu < pcbinfo_arrlen && error == 0; ++cpu) {
2238                 struct inpcbinfo *pcbinfo = &pcbinfo_arr[cpu];
2239                 struct inpcb *inp;
2240                 struct xinpcb xi;
2241                 int i;
2242
2243                 lwkt_migratecpu(cpu);
2244
2245                 GET_PCBINFO_TOKEN(pcbinfo);
2246
2247                 n = pcbinfo->ipi_count;
2248
2249                 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
2250                 i = 0;
2251                 while ((inp = LIST_NEXT(marker, inp_list)) != NULL && i < n) {
2252                         LIST_REMOVE(marker, inp_list);
2253                         LIST_INSERT_AFTER(inp, marker, inp_list);
2254
2255                         if (inp->inp_flags & INP_PLACEMARKER)
2256                                 continue;
2257                         if (prison_xinpcb(req->td, inp))
2258                                 continue;
2259
2260                         bzero(&xi, sizeof xi);
2261                         xi.xi_len = sizeof xi;
2262                         bcopy(inp, &xi.xi_inp, sizeof *inp);
2263                         if (inp->inp_socket)
2264                                 sotoxsocket(inp->inp_socket, &xi.xi_socket);
2265                         if ((error = SYSCTL_OUT(req, &xi, sizeof xi)) != 0)
2266                                 break;
2267                         ++i;
2268                 }
2269                 LIST_REMOVE(marker, inp_list);
2270
2271                 REL_PCBINFO_TOKEN(pcbinfo);
2272
2273                 if (error == 0 && i < n) {
2274                         bzero(&xi, sizeof xi);
2275                         xi.xi_len = sizeof xi;
2276                         while (i < n) {
2277                                 error = SYSCTL_OUT(req, &xi, sizeof xi);
2278                                 if (error)
2279                                         break;
2280                                 ++i;
2281                         }
2282                 }
2283         }
2284
2285         lwkt_migratecpu(origcpu);
2286         kfree(marker, M_TEMP);
2287         return error;
2288 }
2289
2290 int
2291 in_pcblist_global_ncpus2(SYSCTL_HANDLER_ARGS)
2292 {
2293         return in_pcblist_global(oidp, arg1, ncpus2, req);
2294 }
2295
2296 void
2297 in_savefaddr(struct socket *so, const struct sockaddr *faddr)
2298 {
2299         struct sockaddr_in *sin;
2300
2301         KASSERT(faddr->sa_family == AF_INET,
2302             ("not AF_INET faddr %d", faddr->sa_family));
2303
2304         sin = kmalloc(sizeof(*sin), M_SONAME, M_WAITOK | M_ZERO);
2305         sin->sin_family = AF_INET;
2306         sin->sin_len = sizeof(*sin);
2307         sin->sin_port = ((const struct sockaddr_in *)faddr)->sin_port;
2308         sin->sin_addr = ((const struct sockaddr_in *)faddr)->sin_addr;
2309
2310         so->so_faddr = (struct sockaddr *)sin;
2311 }
2312
2313 void
2314 in_pcbportinfo_init(struct inpcbportinfo *portinfo, int hashsize,
2315     u_short offset)
2316 {
2317         memset(portinfo, 0, sizeof(*portinfo));
2318
2319         portinfo->offset = offset;
2320         portinfo->lastport = offset;
2321         portinfo->lastlow = offset;
2322         portinfo->lasthi = offset;
2323
2324         portinfo->porthashbase = hashinit(hashsize, M_PCB,
2325             &portinfo->porthashmask);
2326 }
2327
2328 void
2329 in_pcbportrange(u_short *hi0, u_short *lo0, u_short ofs, u_short step)
2330 {
2331         int hi, lo;
2332
2333         if (step == 1)
2334                 return;
2335
2336         hi = *hi0;
2337         lo = *lo0;
2338
2339         hi = rounddown2(hi, step);
2340         hi += ofs;
2341         if (hi > (int)*hi0)
2342                 hi -= step;
2343
2344         lo = roundup2(lo, step);
2345         lo -= (step - ofs);
2346         if (lo < (int)*lo0)
2347                 lo += step;
2348
2349         *hi0 = hi;
2350         *lo0 = lo;
2351 }
2352
2353 void
2354 in_pcbglobalinit(void)
2355 {
2356         int cpu;
2357
2358         in_pcbmarkers = kmalloc(ncpus * sizeof(struct inpcb), M_PCB,
2359             M_WAITOK | M_ZERO);
2360         in_pcbcontainer_markers = kmalloc(ncpus * sizeof(struct inpcontainer),
2361             M_PCB, M_WAITOK | M_ZERO);
2362
2363         for (cpu = 0; cpu < ncpus; ++cpu) {
2364                 struct inpcontainer *ic = &in_pcbcontainer_markers[cpu];
2365                 struct inpcb *marker = &in_pcbmarkers[cpu];
2366
2367                 marker->inp_flags |= INP_PLACEMARKER;
2368                 ic->ic_inp = marker;
2369         }
2370 }
2371
2372 struct inpcb *
2373 in_pcbmarker(int cpuid)
2374 {
2375         KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid));
2376         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
2377
2378         return &in_pcbmarkers[cpuid];
2379 }
2380
2381 struct inpcontainer *
2382 in_pcbcontainer_marker(int cpuid)
2383 {
2384         KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid));
2385         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
2386
2387         return &in_pcbcontainer_markers[cpuid];
2388 }
2389
2390 void
2391 in_pcbresetroute(struct inpcb *inp)
2392 {
2393         struct route *ro = &inp->inp_route;
2394
2395         if (ro->ro_rt != NULL)
2396                 RTFREE(ro->ro_rt);
2397         bzero(ro, sizeof(*ro));
2398 }
2399
2400 u_short
2401 in_pcblastport_down(volatile u_short *lastport, u_short first, u_short last,
2402     u_short step)
2403 {
2404         u_short lport;
2405
2406         for (;;) {
2407                 u_short olport;
2408
2409                 olport = *lastport;
2410                 lport = olport - step;
2411                 if (__predict_false(lport > first || lport < last))
2412                         lport = first;
2413                 if (atomic_cmpset_short(lastport, olport, lport))
2414                         break;
2415         }
2416         return lport;
2417 }
2418
2419 u_short
2420 in_pcblastport_up(volatile u_short *lastport, u_short first, u_short last,
2421     u_short step)
2422 {
2423         u_short lport;
2424
2425         for (;;) {
2426                 u_short olport;
2427
2428                 olport = *lastport;
2429                 lport = olport + step;
2430                 if (__predict_false(lport < first || lport > last))
2431                         lport = first;
2432                 if (atomic_cmpset_short(lastport, olport, lport))
2433                         break;
2434         }
2435         return lport;
2436 }