sys/netinet/in_pcb.c

   1 /*
   2  * Copyright (c) 2004 Jeffrey M. Hsu.  All rights reserved.
   3  * Copyright (c) 2004 The DragonFly Project.  All rights reserved.
   4  *
   5  * This code is derived from software contributed to The DragonFly Project
   6  * by Jeffrey M. Hsu.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the name of The DragonFly Project nor the names of its
  17  *    contributors may be used to endorse or promote products derived
  18  *    from this software without specific, prior written permission.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  23  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  24  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  25  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  26  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  27  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  28  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  29  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  30  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  */
  33
  34 /*
  35  * Copyright (c) 1982, 1986, 1991, 1993, 1995
  36  *      The Regents of the University of California.  All rights reserved.
  37  *
  38  * Redistribution and use in source and binary forms, with or without
  39  * modification, are permitted provided that the following conditions
  40  * are met:
  41  * 1. Redistributions of source code must retain the above copyright
  42  *    notice, this list of conditions and the following disclaimer.
  43  * 2. Redistributions in binary form must reproduce the above copyright
  44  *    notice, this list of conditions and the following disclaimer in the
  45  *    documentation and/or other materials provided with the distribution.
  46  * 3. Neither the name of the University nor the names of its contributors
  47  *    may be used to endorse or promote products derived from this software
  48  *    without specific prior written permission.
  49  *
  50  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  51  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  52  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  53  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  54  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  55  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  56  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  57  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  58  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  59  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  60  * SUCH DAMAGE.
  61  *
  62  *      @(#)in_pcb.c    8.4 (Berkeley) 5/24/95
  63  * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $
  64  */
  65
  66 #include "opt_ipsec.h"
  67 #include "opt_inet6.h"
  68
  69 #include <sys/param.h>
  70 #include <sys/systm.h>
  71 #include <sys/malloc.h>
  72 #include <sys/mbuf.h>
  73 #include <sys/domain.h>
  74 #include <sys/protosw.h>
  75 #include <sys/socket.h>
  76 #include <sys/socketvar.h>
  77 #include <sys/proc.h>
  78 #include <sys/priv.h>
  79 #include <sys/jail.h>
  80 #include <sys/kernel.h>
  81 #include <sys/sysctl.h>
  82
  83 #include <sys/thread2.h>
  84 #include <sys/socketvar2.h>
  85 #include <sys/msgport2.h>
  86
  87 #include <machine/limits.h>
  88
  89 #include <net/if.h>
  90 #include <net/if_types.h>
  91 #include <net/route.h>
  92 #include <net/netisr2.h>
  93 #include <net/toeplitz2.h>
  94
  95 #include <netinet/in.h>
  96 #include <netinet/in_pcb.h>
  97 #include <netinet/in_var.h>
  98 #include <netinet/ip_var.h>
  99 #ifdef INET6
 100 #include <netinet/ip6.h>
 101 #include <netinet6/ip6_var.h>
 102 #endif /* INET6 */
 103
 104 #ifdef IPSEC
 105 #include <netinet6/ipsec.h>
 106 #include <netproto/key/key.h>
 107 #include <netproto/ipsec/esp_var.h>
 108 #endif
 109
 110 #ifdef FAST_IPSEC
 111 #if defined(IPSEC) || defined(IPSEC_ESP)
 112 #error "Bad idea: don't compile with both IPSEC and FAST_IPSEC!"
 113 #endif
 114
 115 #include <netproto/ipsec/ipsec.h>
 116 #include <netproto/ipsec/key.h>
 117 #define IPSEC
 118 #endif /* FAST_IPSEC */
 119
 120 #define INP_LOCALGROUP_SIZMIN   8
 121 #define INP_LOCALGROUP_SIZMAX   256
 122
 123 static struct inpcb *in_pcblookup_local(struct inpcbporthead *porthash,
 124                 struct in_addr laddr, u_int lport_arg, int wild_okay,
 125                 struct ucred *cred);
 126
 127 struct in_addr zeroin_addr;
 128
 129 /*
 130  * These configure the range of local port addresses assigned to
 131  * "unspecified" outgoing connections/packets/whatever.
 132  */
 133 int ipport_lowfirstauto = IPPORT_RESERVED - 1;  /* 1023 */
 134 int ipport_lowlastauto = IPPORT_RESERVEDSTART;  /* 600 */
 135
 136 int ipport_firstauto = IPPORT_RESERVED;         /* 1024 */
 137 int ipport_lastauto = IPPORT_USERRESERVED;      /* 5000 */
 138
 139 int ipport_hifirstauto = IPPORT_HIFIRSTAUTO;    /* 49152 */
 140 int ipport_hilastauto = IPPORT_HILASTAUTO;      /* 65535 */
 141
 142 #define RANGECHK(var, min, max) \
 143         if ((var) < (min)) { (var) = (min); } \
 144         else if ((var) > (max)) { (var) = (max); }
 145
 146 int udpencap_enable = 1;        /* enabled by default */
 147 int udpencap_port = 4500;       /* triggers decapsulation */
 148
 149 /*
 150  * Per-netisr inpcb markers.
 151  * NOTE: they should only be used in netisrs.
 152  */
 153 static struct inpcb             *in_pcbmarkers;
 154 static struct inpcontainer      *in_pcbcontainer_markers;
 155
 156 static int
 157 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
 158 {
 159         int error;
 160
 161         error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
 162         if (!error) {
 163                 RANGECHK(ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
 164                 RANGECHK(ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
 165
 166                 RANGECHK(ipport_firstauto, IPPORT_RESERVED, USHRT_MAX);
 167                 RANGECHK(ipport_lastauto, IPPORT_RESERVED, USHRT_MAX);
 168
 169                 RANGECHK(ipport_hifirstauto, IPPORT_RESERVED, USHRT_MAX);
 170                 RANGECHK(ipport_hilastauto, IPPORT_RESERVED, USHRT_MAX);
 171         }
 172         return (error);
 173 }
 174
 175 #undef RANGECHK
 176
 177 SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports");
 178
 179 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW,
 180            &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", "");
 181 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, CTLTYPE_INT|CTLFLAG_RW,
 182            &ipport_lowlastauto, 0, &sysctl_net_ipport_check, "I", "");
 183 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, CTLTYPE_INT|CTLFLAG_RW,
 184            &ipport_firstauto, 0, &sysctl_net_ipport_check, "I", "");
 185 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, CTLTYPE_INT|CTLFLAG_RW,
 186            &ipport_lastauto, 0, &sysctl_net_ipport_check, "I", "");
 187 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, CTLTYPE_INT|CTLFLAG_RW,
 188            &ipport_hifirstauto, 0, &sysctl_net_ipport_check, "I", "");
 189 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW,
 190            &ipport_hilastauto, 0, &sysctl_net_ipport_check, "I", "");
 191
 192 static int ip_porthash_trycount = 15;
 193 SYSCTL_INT(_net_inet_ip, OID_AUTO, porthash_trycount, CTLFLAG_RW,
 194     &ip_porthash_trycount, 0,
 195     "Number of tries to find local port matching hash of 4-tuple");
 196
 197 /*
 198  * in_pcb.c: manage the Protocol Control Blocks.
 199  *
 200  * NOTE: It is assumed that most of these functions will be called from
 201  * a critical section.  XXX - There are, unfortunately, a few exceptions
 202  * to this rule that should be fixed.
 203  *
 204  * NOTE: The caller should initialize the cpu field to the cpu running the
 205  * protocol stack associated with this inpcbinfo.
 206  */
 207
 208 void
 209 in_pcbinfo_init(struct inpcbinfo *pcbinfo, int cpu, boolean_t shared)
 210 {
 211         KASSERT(cpu >= 0 && cpu < ncpus, ("invalid cpu%d", cpu));
 212         pcbinfo->cpu = cpu;
 213
 214         LIST_INIT(&pcbinfo->pcblisthead);
 215         pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), M_PCB,
 216                                     M_WAITOK | M_ZERO);
 217
 218         if (shared) {
 219                 pcbinfo->infotoken = kmalloc(sizeof(struct lwkt_token),
 220                     M_PCB, M_WAITOK);
 221                 lwkt_token_init(pcbinfo->infotoken, "infotoken");
 222         } else {
 223                 pcbinfo->infotoken = NULL;
 224         }
 225 }
 226
 227 struct baddynamicports baddynamicports;
 228
 229 /*
 230  * Check if the specified port is invalid for dynamic allocation.
 231  */
 232 int
 233 in_baddynamic(u_int16_t port, u_int16_t proto)
 234 {
 235         switch (proto) {
 236         case IPPROTO_TCP:
 237                 return (DP_ISSET(baddynamicports.tcp, port));
 238         case IPPROTO_UDP:
 239 #ifdef IPSEC
 240                 /* Cannot preset this as it is a sysctl */
 241                 if (port == udpencap_port)
 242                         return (1);
 243 #endif
 244                 return (DP_ISSET(baddynamicports.udp, port));
 245         default:
 246                 return (0);
 247         }
 248 }
 249
 250 void
 251 in_pcbonlist(struct inpcb *inp)
 252 {
 253         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 254
 255         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
 256             ("not in the correct netisr"));
 257         KASSERT((inp->inp_flags & INP_ONLIST) == 0, ("already on pcblist"));
 258         inp->inp_flags |= INP_ONLIST;
 259
 260         GET_PCBINFO_TOKEN(pcbinfo);
 261         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list);
 262         pcbinfo->ipi_count++;
 263         REL_PCBINFO_TOKEN(pcbinfo);
 264 }
 265
 266 void
 267 in_pcbofflist(struct inpcb *inp)
 268 {
 269         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 270
 271         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
 272             ("not in the correct netisr"));
 273         KASSERT(inp->inp_flags & INP_ONLIST, ("not on pcblist"));
 274         inp->inp_flags &= ~INP_ONLIST;
 275
 276         GET_PCBINFO_TOKEN(pcbinfo);
 277         LIST_REMOVE(inp, inp_list);
 278         KASSERT(pcbinfo->ipi_count > 0,
 279             ("invalid inpcb count %d", pcbinfo->ipi_count));
 280         pcbinfo->ipi_count--;
 281         REL_PCBINFO_TOKEN(pcbinfo);
 282 }
 283
 284 /*
 285  * Allocate a PCB and associate it with the socket.
 286  */
 287 int
 288 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
 289 {
 290         struct inpcb *inp;
 291 #ifdef IPSEC
 292         int error;
 293 #endif
 294
 295         inp = kmalloc(pcbinfo->ipi_size, M_PCB, M_WAITOK|M_ZERO|M_NULLOK);
 296         if (inp == NULL)
 297                 return (ENOMEM);
 298         inp->inp_lgrpindex = -1;
 299         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
 300         inp->inp_pcbinfo = pcbinfo;
 301         inp->inp_socket = so;
 302 #ifdef IPSEC
 303         error = ipsec_init_policy(so, &inp->inp_sp);
 304         if (error != 0) {
 305                 kfree(inp, M_PCB);
 306                 return (error);
 307         }
 308 #endif
 309 #ifdef INET6
 310         if (INP_CHECK_SOCKAF(so, AF_INET6)) {
 311                 if (ip6_auto_flowlabel)
 312                         inp->inp_flags |= IN6P_AUTOFLOWLABEL;
 313                 inp->inp_af = AF_INET6;
 314         } else
 315 #endif
 316         inp->inp_af = AF_INET;
 317         soreference(so);
 318         so->so_pcb = inp;
 319
 320         in_pcbonlist(inp);
 321         return (0);
 322 }
 323
 324 /*
 325  * Unlink a pcb with the intention of moving it to another cpu with a
 326  * different pcbinfo.  While unlinked nothing should attempt to dereference
 327  * inp_pcbinfo, NULL it out so we assert if it does.
 328  */
 329 void
 330 in_pcbunlink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags)
 331 {
 332         KASSERT(inp->inp_pcbinfo == pcbinfo, ("pcbinfo mismatch"));
 333         KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0,
 334             ("already linked"));
 335
 336         in_pcbofflist(inp);
 337         inp->inp_pcbinfo = NULL;
 338 }
 339
 340 void
 341 in_pcbunlink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
 342 {
 343         in_pcbunlink_flags(inp, pcbinfo, INP_WILDCARD);
 344 }
 345
 346 /*
 347  * Relink a pcb into a new pcbinfo.
 348  */
 349 void
 350 in_pcblink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags)
 351 {
 352         KASSERT(inp->inp_pcbinfo == NULL, ("has pcbinfo"));
 353         KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0,
 354             ("already linked"));
 355
 356         inp->inp_pcbinfo = pcbinfo;
 357         in_pcbonlist(inp);
 358 }
 359
 360 void
 361 in_pcblink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
 362 {
 363         return in_pcblink_flags(inp, pcbinfo, INP_WILDCARD);
 364 }
 365
 366 static boolean_t
 367 in_pcbporthash_update(struct inpcbportinfo *portinfo,
 368     struct inpcb *inp, u_short lport, struct ucred *cred, int wild)
 369 {
 370         struct inpcbporthead *porthash;
 371
 372         /*
 373          * This has to be atomic.  If the porthash is shared across multiple
 374          * protocol threads, e.g. tcp and udp, then the token must be held.
 375          */
 376         porthash = in_pcbporthash_head(portinfo, lport);
 377         GET_PORTHASH_TOKEN(porthash);
 378
 379         if (in_pcblookup_local(porthash, inp->inp_laddr, lport,
 380             wild, cred) != NULL) {
 381                 REL_PORTHASH_TOKEN(porthash);
 382                 return FALSE;
 383         }
 384         inp->inp_lport = lport;
 385         in_pcbinsporthash(porthash, inp);
 386
 387         REL_PORTHASH_TOKEN(porthash);
 388         return TRUE;
 389 }
 390
 391 static int
 392 in_pcbsetlport(struct inpcb *inp, int wild, struct ucred *cred)
 393 {
 394         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 395         struct inpcbportinfo *portinfo;
 396         u_short first, last, lport, step;
 397         u_short *lastport;
 398         int count, error;
 399         int portinfo_first, portinfo_idx;
 400
 401         inp->inp_flags |= INP_ANONPORT;
 402
 403         step = pcbinfo->portinfo_mask + 1;
 404         portinfo_first = mycpuid & pcbinfo->portinfo_mask;
 405         portinfo_idx = portinfo_first;
 406 loop:
 407         portinfo = &pcbinfo->portinfo[portinfo_idx];
 408
 409         if (inp->inp_flags & INP_HIGHPORT) {
 410                 first = ipport_hifirstauto;     /* sysctl */
 411                 last  = ipport_hilastauto;
 412                 lastport = &portinfo->lasthi;
 413         } else if (inp->inp_flags & INP_LOWPORT) {
 414                 if (cred &&
 415                     (error =
 416                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 417                         inp->inp_laddr.s_addr = INADDR_ANY;
 418                         return error;
 419                 }
 420                 first = ipport_lowfirstauto;    /* 1023 */
 421                 last  = ipport_lowlastauto;     /* 600 */
 422                 lastport = &portinfo->lastlow;
 423         } else {
 424                 first = ipport_firstauto;       /* sysctl */
 425                 last  = ipport_lastauto;
 426                 lastport = &portinfo->lastport;
 427         }
 428
 429         /*
 430          * Simple check to ensure all ports are not used up causing
 431          * a deadlock here.
 432          *
 433          * We split the two cases (up and down) so that the direction
 434          * is not being tested on each round of the loop.
 435          */
 436         if (first > last) {
 437                 /*
 438                  * counting down
 439                  */
 440                 in_pcbportrange(&first, &last, portinfo->offset, step);
 441                 count = (first - last) / step;
 442
 443                 for (;;) {
 444                         if (count-- < 0) {      /* completely used? */
 445                                 error = EADDRNOTAVAIL;
 446                                 break;
 447                         }
 448                         lport = in_pcblastport_down(lastport,
 449                             first, last, step);
 450                         KKASSERT((lport & pcbinfo->portinfo_mask) ==
 451                             portinfo->offset);
 452                         lport = htons(lport);
 453
 454                         if (in_pcbporthash_update(portinfo, inp, lport,
 455                             cred, wild)) {
 456                                 error = 0;
 457                                 break;
 458                         }
 459                 }
 460         } else {
 461                 /*
 462                  * counting up
 463                  */
 464                 in_pcbportrange(&last, &first, portinfo->offset, step);
 465                 count = (last - first) / step;
 466
 467                 for (;;) {
 468                         if (count-- < 0) {      /* completely used? */
 469                                 error = EADDRNOTAVAIL;
 470                                 break;
 471                         }
 472                         lport = in_pcblastport_up(lastport, first, last, step);
 473                         KKASSERT((lport & pcbinfo->portinfo_mask) ==
 474                             portinfo->offset);
 475                         lport = htons(lport);
 476
 477                         if (in_pcbporthash_update(portinfo, inp, lport,
 478                             cred, wild)) {
 479                                 error = 0;
 480                                 break;
 481                         }
 482                 }
 483         }
 484
 485         if (error) {
 486                 /* Try next portinfo */
 487                 portinfo_idx++;
 488                 portinfo_idx &= pcbinfo->portinfo_mask;
 489                 if (portinfo_idx != portinfo_first)
 490                         goto loop;
 491                 inp->inp_laddr.s_addr = INADDR_ANY;
 492         }
 493         return error;
 494 }
 495
 496 int
 497 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
 498 {
 499         struct socket *so = inp->inp_socket;
 500         struct sockaddr_in jsin;
 501         struct ucred *cred = NULL;
 502         int wild = 0;
 503
 504         if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */
 505                 return (EADDRNOTAVAIL);
 506         if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
 507                 return (EINVAL);        /* already bound */
 508
 509         if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT)))
 510                 wild = 1;    /* neither SO_REUSEADDR nor SO_REUSEPORT is set */
 511         if (td->td_proc)
 512                 cred = td->td_proc->p_ucred;
 513
 514         if (nam != NULL) {
 515                 struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 516                 struct inpcbinfo *pcbinfo;
 517                 struct inpcbportinfo *portinfo;
 518                 struct inpcbporthead *porthash;
 519                 struct inpcb *t;
 520                 u_short lport, lport_ho;
 521                 int reuseport = (so->so_options & SO_REUSEPORT);
 522                 int error;
 523
 524                 if (nam->sa_len != sizeof *sin)
 525                         return (EINVAL);
 526 #ifdef notdef
 527                 /*
 528                  * We should check the family, but old programs
 529                  * incorrectly fail to initialize it.
 530                  */
 531                 if (sin->sin_family != AF_INET)
 532                         return (EAFNOSUPPORT);
 533 #endif
 534                 if (!prison_replace_wildcards(td, nam))
 535                         return (EINVAL);
 536
 537                 lport = sin->sin_port;
 538                 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 539                         /*
 540                          * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
 541                          * allow complete duplication of binding if
 542                          * SO_REUSEPORT is set, or if SO_REUSEADDR is set
 543                          * and a multicast address is bound on both
 544                          * new and duplicated sockets.
 545                          */
 546                         if (so->so_options & SO_REUSEADDR)
 547                                 reuseport = SO_REUSEADDR | SO_REUSEPORT;
 548                 } else if (sin->sin_addr.s_addr != INADDR_ANY) {
 549                         sin->sin_port = 0;              /* yech... */
 550                         bzero(&sin->sin_zero, sizeof sin->sin_zero);
 551                         if (ifa_ifwithaddr((struct sockaddr *)sin) == NULL)
 552                                 return (EADDRNOTAVAIL);
 553                 }
 554
 555                 inp->inp_laddr = sin->sin_addr;
 556
 557                 jsin.sin_family = AF_INET;
 558                 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 559                 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 560                         inp->inp_laddr.s_addr = INADDR_ANY;
 561                         return (EINVAL);
 562                 }
 563                 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 564
 565                 if (lport == 0) {
 566                         /* Auto-select local port */
 567                         return in_pcbsetlport(inp, wild, cred);
 568                 }
 569                 lport_ho = ntohs(lport);
 570
 571                 /* GROSS */
 572                 if (lport_ho < IPPORT_RESERVED && cred &&
 573                     (error =
 574                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 575                         inp->inp_laddr.s_addr = INADDR_ANY;
 576                         return (error);
 577                 }
 578
 579                 /*
 580                  * Locate the proper portinfo based on lport
 581                  */
 582                 pcbinfo = inp->inp_pcbinfo;
 583                 portinfo =
 584                     &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask];
 585                 KKASSERT((lport_ho & pcbinfo->portinfo_mask) ==
 586                     portinfo->offset);
 587
 588                 /*
 589                  * This has to be atomic.  If the porthash is shared across
 590                  * multiple protocol threads, e.g. tcp and udp then the token
 591                  * must be held.
 592                  */
 593                 porthash = in_pcbporthash_head(portinfo, lport);
 594                 GET_PORTHASH_TOKEN(porthash);
 595
 596                 if (so->so_cred->cr_uid != 0 &&
 597                     !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 598                         t = in_pcblookup_local(porthash, sin->sin_addr, lport,
 599                             INPLOOKUP_WILDCARD, cred);
 600                         if (t &&
 601                             (so->so_cred->cr_uid !=
 602                              t->inp_socket->so_cred->cr_uid)) {
 603                                 inp->inp_laddr.s_addr = INADDR_ANY;
 604                                 error = EADDRINUSE;
 605                                 goto done;
 606                         }
 607                 }
 608                 if (cred && !prison_replace_wildcards(td, nam)) {
 609                         inp->inp_laddr.s_addr = INADDR_ANY;
 610                         error = EADDRNOTAVAIL;
 611                         goto done;
 612                 }
 613                 t = in_pcblookup_local(porthash, sin->sin_addr, lport,
 614                     wild, cred);
 615                 if (t && !(reuseport & t->inp_socket->so_options)) {
 616                         inp->inp_laddr.s_addr = INADDR_ANY;
 617                         error = EADDRINUSE;
 618                         goto done;
 619                 }
 620                 inp->inp_lport = lport;
 621                 in_pcbinsporthash(porthash, inp);
 622                 error = 0;
 623 done:
 624                 REL_PORTHASH_TOKEN(porthash);
 625                 return (error);
 626         } else {
 627                 jsin.sin_family = AF_INET;
 628                 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 629                 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 630                         inp->inp_laddr.s_addr = INADDR_ANY;
 631                         return (EINVAL);
 632                 }
 633                 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 634
 635                 return in_pcbsetlport(inp, wild, cred);
 636         }
 637 }
 638
 639 static struct inpcb *
 640 in_pcblookup_localremote(struct inpcbporthead *porthash, struct in_addr laddr,
 641     u_short lport, struct in_addr faddr, u_short fport, struct ucred *cred)
 642 {
 643         struct inpcb *inp;
 644         struct inpcbport *phd;
 645         struct inpcb *match = NULL;
 646
 647         /*
 648          * If the porthashbase is shared across several cpus, it must
 649          * have been locked.
 650          */
 651         ASSERT_PORTHASH_TOKEN_HELD(porthash);
 652
 653         /*
 654          * Best fit PCB lookup.
 655          *
 656          * First see if this local port is in use by looking on the
 657          * port hash list.
 658          */
 659         LIST_FOREACH(phd, porthash, phd_hash) {
 660                 if (phd->phd_port == lport)
 661                         break;
 662         }
 663         if (phd != NULL) {
 664                 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
 665 #ifdef INET6
 666                         if (!INP_ISIPV4(inp))
 667                                 continue;
 668 #endif
 669                         if (inp->inp_laddr.s_addr != INADDR_ANY &&
 670                             inp->inp_laddr.s_addr != laddr.s_addr)
 671                                 continue;
 672
 673                         if (inp->inp_faddr.s_addr != INADDR_ANY &&
 674                             inp->inp_faddr.s_addr != faddr.s_addr)
 675                                 continue;
 676
 677                         if (inp->inp_fport != 0 && inp->inp_fport != fport)
 678                                 continue;
 679
 680                         if (cred == NULL ||
 681                             cred->cr_prison ==
 682                             inp->inp_socket->so_cred->cr_prison) {
 683                                 match = inp;
 684                                 break;
 685                         }
 686                 }
 687         }
 688         return (match);
 689 }
 690
 691 static boolean_t
 692 in_pcbporthash_update4(struct inpcbportinfo *portinfo,
 693     struct inpcb *inp, u_short lport, const struct sockaddr_in *sin,
 694     struct ucred *cred)
 695 {
 696         struct inpcbporthead *porthash;
 697
 698         /*
 699          * This has to be atomic.  If the porthash is shared across multiple
 700          * protocol threads, e.g. tcp and udp, then the token must be held.
 701          */
 702         porthash = in_pcbporthash_head(portinfo, lport);
 703         GET_PORTHASH_TOKEN(porthash);
 704
 705         if (in_pcblookup_localremote(porthash, inp->inp_laddr,
 706             lport, sin->sin_addr, sin->sin_port, cred) != NULL) {
 707                 REL_PORTHASH_TOKEN(porthash);
 708                 return FALSE;
 709         }
 710         inp->inp_lport = lport;
 711         in_pcbinsporthash(porthash, inp);
 712
 713         REL_PORTHASH_TOKEN(porthash);
 714         return TRUE;
 715 }
 716
 717 int
 718 in_pcbbind_remote(struct inpcb *inp, const struct sockaddr *remote,
 719     struct thread *td)
 720 {
 721         struct proc *p = td->td_proc;
 722         u_short *lastport;
 723         const struct sockaddr_in *sin = (const struct sockaddr_in *)remote;
 724         struct sockaddr_in jsin;
 725         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 726         struct inpcbportinfo *portinfo;
 727         struct ucred *cred = NULL;
 728         u_short first, last, lport, step;
 729         int count, error, selfconn;
 730         int portinfo_first, portinfo_idx;
 731         int hash_cpu, hash_count, hash_count0;
 732         uint32_t hash_base = 0, hash;
 733
 734         if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */
 735                 return (EADDRNOTAVAIL);
 736
 737         KKASSERT(inp->inp_laddr.s_addr != INADDR_ANY);
 738         if (inp->inp_lport != 0)
 739                 return (EINVAL);        /* already bound */
 740
 741         KKASSERT(p);
 742         cred = p->p_ucred;
 743
 744         jsin.sin_family = AF_INET;
 745         jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 746         if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 747                 inp->inp_laddr.s_addr = INADDR_ANY;
 748                 return (EINVAL);
 749         }
 750         inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 751
 752         hash_count0 = ip_porthash_trycount;
 753         if (hash_count0 > 0) {
 754                 hash_base = toeplitz_piecemeal_addr(sin->sin_addr.s_addr) ^
 755                     toeplitz_piecemeal_addr(inp->inp_laddr.s_addr) ^
 756                     toeplitz_piecemeal_port(sin->sin_port);
 757         } else {
 758                 hash_count0 = 0;
 759         }
 760
 761         inp->inp_flags |= INP_ANONPORT;
 762
 763         step = pcbinfo->portinfo_mask + 1;
 764         portinfo_first = mycpuid & pcbinfo->portinfo_mask;
 765         portinfo_idx = portinfo_first;
 766 loop:
 767         hash_cpu = portinfo_idx & ncpus2_mask;
 768         portinfo = &pcbinfo->portinfo[portinfo_idx];
 769         selfconn = 0;
 770
 771         if (inp->inp_flags & INP_HIGHPORT) {
 772                 first = ipport_hifirstauto;     /* sysctl */
 773                 last  = ipport_hilastauto;
 774                 lastport = &portinfo->lasthi;
 775         } else if (inp->inp_flags & INP_LOWPORT) {
 776                 if (cred &&
 777                     (error =
 778                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 779                         inp->inp_laddr.s_addr = INADDR_ANY;
 780                         return (error);
 781                 }
 782                 first = ipport_lowfirstauto;    /* 1023 */
 783                 last  = ipport_lowlastauto;     /* 600 */
 784                 lastport = &portinfo->lastlow;
 785         } else {
 786                 first = ipport_firstauto;       /* sysctl */
 787                 last  = ipport_lastauto;
 788                 lastport = &portinfo->lastport;
 789         }
 790
 791         /* This could happen on loopback interface */
 792 #define IS_SELFCONNECT(inp, lport, sin)                 \
 793         (__predict_false((sin)->sin_port == (lport) &&  \
 794          (sin)->sin_addr.s_addr == (inp)->inp_laddr.s_addr))
 795
 796         /*
 797          * Simple check to ensure all ports are not used up causing
 798          * a deadlock here.
 799          *
 800          * We split the two cases (up and down) so that the direction
 801          * is not being tested on each round of the loop.
 802          */
 803         hash_count = hash_count0;
 804         if (first > last) {
 805                 /*
 806                  * counting down
 807                  */
 808                 in_pcbportrange(&first, &last, portinfo->offset, step);
 809                 count = ((first - last) / step) + hash_count;
 810
 811                 for (;;) {
 812                         if (count-- < 0) {      /* completely used? */
 813                                 error = EADDRNOTAVAIL;
 814                                 break;
 815                         }
 816
 817                         lport = in_pcblastport_down(lastport, first, last,
 818                             step);
 819                         KKASSERT((lport & pcbinfo->portinfo_mask) ==
 820                             portinfo->offset);
 821                         lport = htons(lport);
 822                         if (IS_SELFCONNECT(inp, lport, sin)) {
 823                                 if (!selfconn) {
 824                                         ++count; /* don't count this try */
 825                                         selfconn = 1;
 826                                 }
 827                                 continue;
 828                         }
 829
 830                         if (hash_count) {
 831                                 --hash_count;
 832                                 hash = hash_base ^
 833                                     toeplitz_piecemeal_port(lport);
 834                                 if ((hash & ncpus2_mask) != hash_cpu && hash_count)
 835                                         continue;
 836                         }
 837
 838                         if (in_pcbporthash_update4(portinfo,
 839                             inp, lport, sin, cred)) {
 840                                 error = 0;
 841                                 break;
 842                         }
 843                 }
 844         } else {
 845                 /*
 846                  * counting up
 847                  */
 848                 in_pcbportrange(&last, &first, portinfo->offset, step);
 849                 count = ((last - first) / step) + hash_count;
 850
 851                 for (;;) {
 852                         if (count-- < 0) {      /* completely used? */
 853                                 error = EADDRNOTAVAIL;
 854                                 break;
 855                         }
 856
 857                         lport = in_pcblastport_up(lastport, first, last, step);
 858                         KKASSERT((lport & pcbinfo->portinfo_mask) ==
 859                             portinfo->offset);
 860                         lport = htons(lport);
 861                         if (IS_SELFCONNECT(inp, lport, sin)) {
 862                                 if (!selfconn) {
 863                                         ++count; /* don't count this try */
 864                                         selfconn = 1;
 865                                 }
 866                                 continue;
 867                         }
 868
 869                         if (hash_count) {
 870                                 --hash_count;
 871                                 hash = hash_base ^
 872                                     toeplitz_piecemeal_port(lport);
 873                                 if ((hash & ncpus2_mask) != hash_cpu && hash_count)
 874                                         continue;
 875                         }
 876
 877                         if (in_pcbporthash_update4(portinfo,
 878                             inp, lport, sin, cred)) {
 879                                 error = 0;
 880                                 break;
 881                         }
 882                 }
 883         }
 884
 885 #undef IS_SELFCONNECT
 886
 887         if (error) {
 888                 /* Try next portinfo */
 889                 portinfo_idx++;
 890                 portinfo_idx &= pcbinfo->portinfo_mask;
 891                 if (portinfo_idx != portinfo_first)
 892                         goto loop;
 893                 inp->inp_laddr.s_addr = INADDR_ANY;
 894         }
 895         return error;
 896 }
 897
 898 /*
 899  *   Transform old in_pcbconnect() into an inner subroutine for new
 900  *   in_pcbconnect(): Do some validity-checking on the remote
 901  *   address (in mbuf 'nam') and then determine local host address
 902  *   (i.e., which interface) to use to access that remote host.
 903  *
 904  *   This preserves definition of in_pcbconnect(), while supporting a
 905  *   slightly different version for T/TCP.  (This is more than
 906  *   a bit of a kludge, but cleaning up the internal interfaces would
 907  *   have forced minor changes in every protocol).
 908  */
 909 int
 910 in_pcbladdr_find(struct inpcb *inp, struct sockaddr *nam,
 911     struct sockaddr_in **plocal_sin, struct thread *td, int find)
 912 {
 913         struct in_ifaddr *ia;
 914         struct ucred *cred = NULL;
 915         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 916         struct sockaddr *jsin;
 917         int jailed = 0, alloc_route = 0;
 918
 919         if (nam->sa_len != sizeof *sin)
 920                 return (EINVAL);
 921         if (sin->sin_family != AF_INET)
 922                 return (EAFNOSUPPORT);
 923         if (sin->sin_port == 0)
 924                 return (EADDRNOTAVAIL);
 925         if (td && td->td_proc && td->td_proc->p_ucred)
 926                 cred = td->td_proc->p_ucred;
 927         if (cred && cred->cr_prison)
 928                 jailed = 1;
 929         if (!TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) {
 930                 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia;
 931                 /*
 932                  * If the destination address is INADDR_ANY,
 933                  * use the primary local address.
 934                  * If the supplied address is INADDR_BROADCAST,
 935                  * and the primary interface supports broadcast,
 936                  * choose the broadcast address for that interface.
 937                  */
 938                 if (sin->sin_addr.s_addr == INADDR_ANY)
 939                         sin->sin_addr = IA_SIN(ia)->sin_addr;
 940                 else if (sin->sin_addr.s_addr == (u_long)INADDR_BROADCAST &&
 941                     (ia->ia_ifp->if_flags & IFF_BROADCAST))
 942                         sin->sin_addr = satosin(&ia->ia_broadaddr)->sin_addr;
 943         }
 944         if (find) {
 945                 struct route *ro;
 946
 947                 ia = NULL;
 948                 /*
 949                  * If route is known or can be allocated now,
 950                  * our src addr is taken from the i/f, else punt.
 951                  * Note that we should check the address family of the cached
 952                  * destination, in case of sharing the cache with IPv6.
 953                  */
 954                 ro = &inp->inp_route;
 955                 if (ro->ro_rt &&
 956                     (!(ro->ro_rt->rt_flags & RTF_UP) ||
 957                      ro->ro_dst.sa_family != AF_INET ||
 958                      satosin(&ro->ro_dst)->sin_addr.s_addr !=
 959                                       sin->sin_addr.s_addr ||
 960                      inp->inp_socket->so_options & SO_DONTROUTE)) {
 961                         RTFREE(ro->ro_rt);
 962                         ro->ro_rt = NULL;
 963                 }
 964                 if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/
 965                     (ro->ro_rt == NULL ||
 966                     ro->ro_rt->rt_ifp == NULL)) {
 967                         /* No route yet, so try to acquire one */
 968                         bzero(&ro->ro_dst, sizeof(struct sockaddr_in));
 969                         ro->ro_dst.sa_family = AF_INET;
 970                         ro->ro_dst.sa_len = sizeof(struct sockaddr_in);
 971                         ((struct sockaddr_in *) &ro->ro_dst)->sin_addr =
 972                                 sin->sin_addr;
 973                         rtalloc(ro);
 974                         alloc_route = 1;
 975                 }
 976                 /*
 977                  * If we found a route, use the address
 978                  * corresponding to the outgoing interface
 979                  * unless it is the loopback (in case a route
 980                  * to our address on another net goes to loopback).
 981                  */
 982                 if (ro->ro_rt &&
 983                     !(ro->ro_rt->rt_ifp->if_flags & IFF_LOOPBACK)) {
 984                         if (jailed) {
 985                                 if (jailed_ip(cred->cr_prison,
 986                                     ro->ro_rt->rt_ifa->ifa_addr)) {
 987                                         ia = ifatoia(ro->ro_rt->rt_ifa);
 988                                 }
 989                         } else {
 990                                 ia = ifatoia(ro->ro_rt->rt_ifa);
 991                         }
 992                 }
 993                 if (ia == NULL) {
 994                         u_short fport = sin->sin_port;
 995
 996                         sin->sin_port = 0;
 997                         ia = ifatoia(ifa_ifwithdstaddr(sintosa(sin)));
 998                         if (ia && jailed && !jailed_ip(cred->cr_prison,
 999                             sintosa(&ia->ia_addr)))
1000                                 ia = NULL;
1001                         if (ia == NULL)
1002                                 ia = ifatoia(ifa_ifwithnet(sintosa(sin)));
1003                         if (ia && jailed && !jailed_ip(cred->cr_prison,
1004                             sintosa(&ia->ia_addr)))
1005                                 ia = NULL;
1006                         sin->sin_port = fport;
1007                         if (ia == NULL &&
1008                             !TAILQ_EMPTY(&in_ifaddrheads[mycpuid]))
1009                                 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia;
1010                         if (ia && jailed && !jailed_ip(cred->cr_prison,
1011                             sintosa(&ia->ia_addr)))
1012                                 ia = NULL;
1013
1014                         if (!jailed && ia == NULL)
1015                                 goto fail;
1016                 }
1017                 /*
1018                  * If the destination address is multicast and an outgoing
1019                  * interface has been set as a multicast option, use the
1020                  * address of that interface as our source address.
1021                  */
1022                 if (!jailed && IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
1023                     inp->inp_moptions != NULL) {
1024                         struct ip_moptions *imo;
1025                         struct ifnet *ifp;
1026
1027                         imo = inp->inp_moptions;
1028                         if (imo->imo_multicast_ifp != NULL) {
1029                                 struct in_ifaddr_container *iac;
1030
1031                                 ifp = imo->imo_multicast_ifp;
1032                                 ia = NULL;
1033                                 TAILQ_FOREACH(iac,
1034                                 &in_ifaddrheads[mycpuid], ia_link) {
1035                                         if (iac->ia->ia_ifp == ifp) {
1036                                                 ia = iac->ia;
1037                                                 break;
1038                                         }
1039                                 }
1040                                 if (ia == NULL)
1041                                         goto fail;
1042                         }
1043                 }
1044                 /*
1045                  * Don't do pcblookup call here; return interface in plocal_sin
1046                  * and exit to caller, that will do the lookup.
1047                  */
1048                 if (ia == NULL && jailed) {
1049                         if ((jsin = prison_get_nonlocal(
1050                                 cred->cr_prison, AF_INET, NULL)) != NULL ||
1051                             (jsin = prison_get_local(
1052                                 cred->cr_prison, AF_INET, NULL)) != NULL) {
1053                                 *plocal_sin = satosin(jsin);
1054                         } else {
1055                                 /* IPv6 only Jail */
1056                                 goto fail;
1057                         }
1058                 } else {
1059                         *plocal_sin = &ia->ia_addr;
1060                 }
1061         }
1062         return (0);
1063 fail:
1064         if (alloc_route)
1065                 in_pcbresetroute(inp);
1066         return (EADDRNOTAVAIL);
1067 }
1068
1069 int
1070 in_pcbladdr(struct inpcb *inp, struct sockaddr *nam,
1071     struct sockaddr_in **plocal_sin, struct thread *td)
1072 {
1073         return in_pcbladdr_find(inp, nam, plocal_sin, td,
1074             (inp->inp_laddr.s_addr == INADDR_ANY));
1075 }
1076
1077 /*
1078  * Outer subroutine:
1079  * Connect from a socket to a specified address.
1080  * Both address and port must be specified in argument sin.
1081  * If don't have a local address for this socket yet,
1082  * then pick one.
1083  */
1084 int
1085 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
1086 {
1087         struct sockaddr_in *if_sin;
1088         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
1089         int error;
1090
1091         /* Call inner routine to assign local interface address. */
1092         if ((error = in_pcbladdr(inp, nam, &if_sin, td)) != 0)
1093                 return (error);
1094
1095         if (in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port,
1096                               inp->inp_laddr.s_addr ?
1097                                 inp->inp_laddr : if_sin->sin_addr,
1098                               inp->inp_lport, FALSE, NULL) != NULL) {
1099                 return (EADDRINUSE);
1100         }
1101         if (inp->inp_laddr.s_addr == INADDR_ANY) {
1102                 if (inp->inp_lport == 0) {
1103                         error = in_pcbbind(inp, NULL, td);
1104                         if (error)
1105                                 return (error);
1106                 }
1107                 inp->inp_laddr = if_sin->sin_addr;
1108         }
1109         inp->inp_faddr = sin->sin_addr;
1110         inp->inp_fport = sin->sin_port;
1111         in_pcbinsconnhash(inp);
1112         return (0);
1113 }
1114
1115 void
1116 in_pcbdisconnect(struct inpcb *inp)
1117 {
1118
1119         in_pcbremconnhash(inp);
1120         inp->inp_faddr.s_addr = INADDR_ANY;
1121         inp->inp_fport = 0;
1122 }
1123
1124 void
1125 in_pcbdetach(struct inpcb *inp)
1126 {
1127         struct socket *so = inp->inp_socket;
1128         struct inpcbinfo *ipi = inp->inp_pcbinfo;
1129
1130 #ifdef IPSEC
1131         ipsec4_delete_pcbpolicy(inp);
1132 #endif /*IPSEC*/
1133         inp->inp_gencnt = ++ipi->ipi_gencnt;
1134         KKASSERT((so->so_state & SS_ASSERTINPROG) == 0);
1135         in_pcbremlists(inp);
1136         so->so_pcb = NULL;
1137         sofree(so);                     /* remove pcb ref */
1138         if (inp->inp_options)
1139                 m_free(inp->inp_options);
1140         if (inp->inp_route.ro_rt)
1141                 rtfree(inp->inp_route.ro_rt);
1142         ip_freemoptions(inp->inp_moptions);
1143         kfree(inp, M_PCB);
1144 }
1145
1146 /*
1147  * The socket may have an invalid PCB, i.e. NULL.  For example, a TCP
1148  * socket received RST.
1149  */
1150 static int
1151 in_setsockaddr(struct socket *so, struct sockaddr **nam)
1152 {
1153         struct inpcb *inp;
1154         struct sockaddr_in *sin;
1155
1156         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
1157         inp = so->so_pcb;
1158         if (!inp)
1159                 return (ECONNRESET);
1160
1161         sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1162         sin->sin_family = AF_INET;
1163         sin->sin_len = sizeof *sin;
1164         sin->sin_port = inp->inp_lport;
1165         sin->sin_addr = inp->inp_laddr;
1166
1167         *nam = (struct sockaddr *)sin;
1168         return (0);
1169 }
1170
1171 void
1172 in_setsockaddr_dispatch(netmsg_t msg)
1173 {
1174         int error;
1175
1176         error = in_setsockaddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1177         lwkt_replymsg(&msg->lmsg, error);
1178 }
1179
1180 /*
1181  * The socket may have an invalid PCB, i.e. NULL.  For example, a TCP
1182  * socket received RST.
1183  */
1184 int
1185 in_setpeeraddr(struct socket *so, struct sockaddr **nam)
1186 {
1187         struct inpcb *inp;
1188         struct sockaddr_in *sin;
1189
1190         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
1191         inp = so->so_pcb;
1192         if (!inp)
1193                 return (ECONNRESET);
1194
1195         sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1196         sin->sin_family = AF_INET;
1197         sin->sin_len = sizeof *sin;
1198         sin->sin_port = inp->inp_fport;
1199         sin->sin_addr = inp->inp_faddr;
1200
1201         *nam = (struct sockaddr *)sin;
1202         return (0);
1203 }
1204
1205 void
1206 in_setpeeraddr_dispatch(netmsg_t msg)
1207 {
1208         int error;
1209
1210         error = in_setpeeraddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1211         lwkt_replymsg(&msg->lmsg, error);
1212 }
1213
1214 void
1215 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int err,
1216     inp_notify_t notify)
1217 {
1218         struct inpcb *inp, *marker;
1219
1220         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1221             ("not in the correct netisr"));
1222         marker = &in_pcbmarkers[mycpuid];
1223
1224         /*
1225          * NOTE:
1226          * - If INP_PLACEMARKER is set we must ignore the rest of the
1227          *   structure and skip it.
1228          * - It is safe to nuke inpcbs here, since we are in their own
1229          *   netisr.
1230          */
1231         GET_PCBINFO_TOKEN(pcbinfo);
1232
1233         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1234         while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1235                 LIST_REMOVE(marker, inp_list);
1236                 LIST_INSERT_AFTER(inp, marker, inp_list);
1237
1238                 if (inp->inp_flags & INP_PLACEMARKER)
1239                         continue;
1240 #ifdef INET6
1241                 if (!INP_ISIPV4(inp))
1242                         continue;
1243 #endif
1244                 if (inp->inp_faddr.s_addr != faddr.s_addr ||
1245                     inp->inp_socket == NULL)
1246                         continue;
1247                 (*notify)(inp, err);            /* can remove inp from list! */
1248         }
1249         LIST_REMOVE(marker, inp_list);
1250
1251         REL_PCBINFO_TOKEN(pcbinfo);
1252 }
1253
1254 void
1255 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1256 {
1257         struct inpcb *inp, *marker;
1258
1259         /*
1260          * We only need to make sure that we are in netisr0, where all
1261          * multicast operation happen.  We could check inpcbinfo which
1262          * does not belong to netisr0 by holding the inpcbinfo's token.
1263          * In this case, the pcbinfo must be able to be shared, i.e.
1264          * pcbinfo->infotoken is not NULL.
1265          */
1266         ASSERT_IN_NETISR(0);
1267         KASSERT(pcbinfo->cpu == 0 || pcbinfo->infotoken != NULL,
1268             ("pcbinfo could not be shared"));
1269
1270         /*
1271          * Get a marker for the current netisr (netisr0).
1272          *
1273          * It is possible that the multicast address deletion blocks,
1274          * which could cause temporary token releasing.  So we use
1275          * inpcb marker here to get a coherent view of the inpcb list.
1276          *
1277          * While, on the other hand, moptions are only added and deleted
1278          * in netisr0, so we would not see staled moption or miss moption
1279          * even if the token was released due to the blocking multicast
1280          * address deletion.
1281          */
1282         marker = &in_pcbmarkers[mycpuid];
1283
1284         GET_PCBINFO_TOKEN(pcbinfo);
1285
1286         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1287         while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1288                 struct ip_moptions *imo;
1289
1290                 LIST_REMOVE(marker, inp_list);
1291                 LIST_INSERT_AFTER(inp, marker, inp_list);
1292
1293                 if (inp->inp_flags & INP_PLACEMARKER)
1294                         continue;
1295                 imo = inp->inp_moptions;
1296                 if (INP_ISIPV4(inp) && imo != NULL) {
1297                         int i, gap;
1298
1299                         /*
1300                          * Unselect the outgoing interface if it is being
1301                          * detached.
1302                          */
1303                         if (imo->imo_multicast_ifp == ifp)
1304                                 imo->imo_multicast_ifp = NULL;
1305
1306                         /*
1307                          * Drop multicast group membership if we joined
1308                          * through the interface being detached.
1309                          */
1310                         for (i = 0, gap = 0; i < imo->imo_num_memberships;
1311                             i++) {
1312                                 if (imo->imo_membership[i]->inm_ifp == ifp) {
1313                                         /*
1314                                          * NOTE:
1315                                          * This could block and the pcbinfo
1316                                          * token could be passively released.
1317                                          */
1318                                         in_delmulti(imo->imo_membership[i]);
1319                                         gap++;
1320                                 } else if (gap != 0)
1321                                         imo->imo_membership[i - gap] =
1322                                             imo->imo_membership[i];
1323                         }
1324                         imo->imo_num_memberships -= gap;
1325                 }
1326         }
1327         LIST_REMOVE(marker, inp_list);
1328
1329         REL_PCBINFO_TOKEN(pcbinfo);
1330 }
1331
1332 /*
1333  * Check for alternatives when higher level complains
1334  * about service problems.  For now, invalidate cached
1335  * routing information.  If the route was created dynamically
1336  * (by a redirect), time to try a default gateway again.
1337  */
1338 void
1339 in_losing(struct inpcb *inp)
1340 {
1341         struct rtentry *rt;
1342         struct rt_addrinfo rtinfo;
1343
1344         if ((rt = inp->inp_route.ro_rt)) {
1345                 bzero(&rtinfo, sizeof(struct rt_addrinfo));
1346                 rtinfo.rti_info[RTAX_DST] = rt_key(rt);
1347                 rtinfo.rti_info[RTAX_GATEWAY] = rt->rt_gateway;
1348                 rtinfo.rti_info[RTAX_NETMASK] = rt_mask(rt);
1349                 rtinfo.rti_flags = rt->rt_flags;
1350                 rt_missmsg(RTM_LOSING, &rtinfo, rt->rt_flags, 0);
1351                 if (rt->rt_flags & RTF_DYNAMIC) {
1352                         rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway,
1353                             rt_mask(rt), rt->rt_flags, NULL);
1354                 }
1355                 inp->inp_route.ro_rt = NULL;
1356                 rtfree(rt);
1357                 /*
1358                  * A new route can be allocated
1359                  * the next time output is attempted.
1360                  */
1361         }
1362 }
1363
1364 /*
1365  * After a routing change, flush old routing
1366  * and allocate a (hopefully) better one.
1367  */
1368 void
1369 in_rtchange(struct inpcb *inp, int err)
1370 {
1371         if (inp->inp_route.ro_rt) {
1372                 rtfree(inp->inp_route.ro_rt);
1373                 inp->inp_route.ro_rt = NULL;
1374                 /*
1375                  * A new route can be allocated the next time
1376                  * output is attempted.
1377                  */
1378         }
1379 }
1380
1381 /*
1382  * Lookup a PCB based on the local address and port.
1383  */
1384 static struct inpcb *
1385 in_pcblookup_local(struct inpcbporthead *porthash, struct in_addr laddr,
1386                    u_int lport_arg, int wild_okay, struct ucred *cred)
1387 {
1388         struct inpcb *inp;
1389         int matchwild = 3, wildcard;
1390         u_short lport = lport_arg;
1391         struct inpcbport *phd;
1392         struct inpcb *match = NULL;
1393
1394         /*
1395          * If the porthashbase is shared across several cpus, it must
1396          * have been locked.
1397          */
1398         ASSERT_PORTHASH_TOKEN_HELD(porthash);
1399
1400         /*
1401          * Best fit PCB lookup.
1402          *
1403          * First see if this local port is in use by looking on the
1404          * port hash list.
1405          */
1406         LIST_FOREACH(phd, porthash, phd_hash) {
1407                 if (phd->phd_port == lport)
1408                         break;
1409         }
1410         if (phd != NULL) {
1411                 /*
1412                  * Port is in use by one or more PCBs. Look for best
1413                  * fit.
1414                  */
1415                 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
1416                         wildcard = 0;
1417 #ifdef INET6
1418                         if (!INP_ISIPV4(inp))
1419                                 continue;
1420 #endif
1421                         if (inp->inp_faddr.s_addr != INADDR_ANY)
1422                                 wildcard++;
1423                         if (inp->inp_laddr.s_addr != INADDR_ANY) {
1424                                 if (laddr.s_addr == INADDR_ANY)
1425                                         wildcard++;
1426                                 else if (inp->inp_laddr.s_addr != laddr.s_addr)
1427                                         continue;
1428                         } else {
1429                                 if (laddr.s_addr != INADDR_ANY)
1430                                         wildcard++;
1431                         }
1432                         if (wildcard && !wild_okay)
1433                                 continue;
1434                         if (wildcard < matchwild &&
1435                             (cred == NULL ||
1436                              cred->cr_prison ==
1437                                         inp->inp_socket->so_cred->cr_prison)) {
1438                                 match = inp;
1439                                 matchwild = wildcard;
1440                                 if (matchwild == 0) {
1441                                         break;
1442                                 }
1443                         }
1444                 }
1445         }
1446         return (match);
1447 }
1448
1449 struct inpcb *
1450 in_pcblocalgroup_last(const struct inpcbinfo *pcbinfo,
1451     const struct inpcb *inp)
1452 {
1453         const struct inp_localgrphead *hdr;
1454         const struct inp_localgroup *grp;
1455         int i;
1456
1457         if (pcbinfo->localgrphashbase == NULL)
1458                 return NULL;
1459
1460         GET_PCBINFO_TOKEN(pcbinfo);
1461
1462         hdr = &pcbinfo->localgrphashbase[
1463             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
1464
1465         LIST_FOREACH(grp, hdr, il_list) {
1466                 if (grp->il_af == inp->inp_af &&
1467                     grp->il_lport == inp->inp_lport &&
1468                     memcmp(&grp->il_dependladdr,
1469                         &inp->inp_inc.inc_ie.ie_dependladdr,
1470                         sizeof(grp->il_dependladdr)) == 0) {
1471                         break;
1472                 }
1473         }
1474         if (grp == NULL || grp->il_inpcnt == 1) {
1475                 REL_PCBINFO_TOKEN(pcbinfo);
1476                 return NULL;
1477         }
1478
1479         KASSERT(grp->il_inpcnt >= 2,
1480             ("invalid localgroup inp count %d", grp->il_inpcnt));
1481         for (i = 0; i < grp->il_inpcnt; ++i) {
1482                 if (grp->il_inp[i] == inp) {
1483                         int last = grp->il_inpcnt - 1;
1484
1485                         if (i == last)
1486                                 last = grp->il_inpcnt - 2;
1487                         REL_PCBINFO_TOKEN(pcbinfo);
1488                         return grp->il_inp[last];
1489                 }
1490         }
1491         REL_PCBINFO_TOKEN(pcbinfo);
1492         return NULL;
1493 }
1494
1495 static struct inpcb *
1496 inp_localgroup_lookup(const struct inpcbinfo *pcbinfo,
1497     struct in_addr laddr, uint16_t lport, uint32_t pkt_hash)
1498 {
1499         struct inpcb *local_wild = NULL;
1500         const struct inp_localgrphead *hdr;
1501         const struct inp_localgroup *grp;
1502
1503         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1504
1505         hdr = &pcbinfo->localgrphashbase[
1506             INP_PCBLOCALGRPHASH(lport, pcbinfo->localgrphashmask)];
1507
1508         /*
1509          * Order of socket selection:
1510          * 1. non-wild.
1511          * 2. wild.
1512          *
1513          * NOTE: Local group does not contain jailed sockets
1514          */
1515         LIST_FOREACH(grp, hdr, il_list) {
1516 #ifdef INET6
1517                 if (grp->il_af != AF_INET)
1518                         continue;
1519 #endif
1520                 if (grp->il_lport == lport) {
1521                         int idx;
1522
1523                         /*
1524                          * Modulo-N is used here, which greatly reduces
1525                          * completion queue token contention, thus more
1526                          * cpu time is saved.
1527                          */
1528                         idx = pkt_hash % grp->il_inpcnt;
1529                         if (grp->il_laddr.s_addr == laddr.s_addr)
1530                                 return grp->il_inp[idx];
1531                         else if (grp->il_laddr.s_addr == INADDR_ANY)
1532                                 local_wild = grp->il_inp[idx];
1533                 }
1534         }
1535         if (local_wild != NULL)
1536                 return local_wild;
1537         return NULL;
1538 }
1539
1540 /*
1541  * Lookup PCB in hash list.
1542  */
1543 struct inpcb *
1544 in_pcblookup_pkthash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1545     u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1546     boolean_t wildcard, struct ifnet *ifp, const struct mbuf *m)
1547 {
1548         struct inpcbhead *head;
1549         struct inpcb *inp, *jinp=NULL;
1550         u_short fport = fport_arg, lport = lport_arg;
1551
1552         /*
1553          * First look for an exact match.
1554          */
1555         head = &pcbinfo->hashbase[INP_PCBCONNHASH(faddr.s_addr, fport,
1556             laddr.s_addr, lport, pcbinfo->hashmask)];
1557         LIST_FOREACH(inp, head, inp_hash) {
1558 #ifdef INET6
1559                 if (!INP_ISIPV4(inp))
1560                         continue;
1561 #endif
1562                 if (in_hosteq(inp->inp_faddr, faddr) &&
1563                     in_hosteq(inp->inp_laddr, laddr) &&
1564                     inp->inp_fport == fport && inp->inp_lport == lport) {
1565                         /* found */
1566                         if (inp->inp_socket == NULL ||
1567                             inp->inp_socket->so_cred->cr_prison == NULL) {
1568                                 return (inp);
1569                         } else {
1570                                 if  (jinp == NULL)
1571                                         jinp = inp;
1572                         }
1573                 }
1574         }
1575         if (jinp != NULL)
1576                 return (jinp);
1577
1578         if (wildcard) {
1579                 struct inpcb *local_wild = NULL;
1580                 struct inpcb *jinp_wild = NULL;
1581                 struct inpcontainer *ic;
1582                 struct inpcontainerhead *chead;
1583                 struct sockaddr_in jsin;
1584                 struct ucred *cred;
1585
1586                 GET_PCBINFO_TOKEN(pcbinfo);
1587
1588                 /*
1589                  * Check local group first
1590                  */
1591                 if (pcbinfo->localgrphashbase != NULL &&
1592                     m != NULL && (m->m_flags & M_HASH) &&
1593                     !(ifp && ifp->if_type == IFT_FAITH)) {
1594                         inp = inp_localgroup_lookup(pcbinfo,
1595                             laddr, lport, m->m_pkthdr.hash);
1596                         if (inp != NULL) {
1597                                 REL_PCBINFO_TOKEN(pcbinfo);
1598                                 return inp;
1599                         }
1600                 }
1601
1602                 /*
1603                  * Order of socket selection:
1604                  * 1. non-jailed, non-wild.
1605                  * 2. non-jailed, wild.
1606                  * 3. jailed, non-wild.
1607                  * 4. jailed, wild.
1608                  */
1609                 jsin.sin_family = AF_INET;
1610                 chead = &pcbinfo->wildcardhashbase[
1611                     INP_PCBWILDCARDHASH(lport, pcbinfo->wildcardhashmask)];
1612                 LIST_FOREACH(ic, chead, ic_list) {
1613                         inp = ic->ic_inp;
1614                         if (inp->inp_flags & INP_PLACEMARKER)
1615                                 continue;
1616
1617                         jsin.sin_addr.s_addr = laddr.s_addr;
1618 #ifdef INET6
1619                         if (!INP_ISIPV4(inp))
1620                                 continue;
1621 #endif
1622                         if (inp->inp_socket != NULL)
1623                                 cred = inp->inp_socket->so_cred;
1624                         else
1625                                 cred = NULL;
1626                         if (cred != NULL && jailed(cred)) {
1627                                 if (jinp != NULL)
1628                                         continue;
1629                                 else
1630                                         if (!jailed_ip(cred->cr_prison,
1631                                             (struct sockaddr *)&jsin))
1632                                                 continue;
1633                         }
1634                         if (inp->inp_lport == lport) {
1635                                 if (ifp && ifp->if_type == IFT_FAITH &&
1636                                     !(inp->inp_flags & INP_FAITH))
1637                                         continue;
1638                                 if (inp->inp_laddr.s_addr == laddr.s_addr) {
1639                                         if (cred != NULL && jailed(cred)) {
1640                                                 jinp = inp;
1641                                         } else {
1642                                                 REL_PCBINFO_TOKEN(pcbinfo);
1643                                                 return (inp);
1644                                         }
1645                                 }
1646                                 if (inp->inp_laddr.s_addr == INADDR_ANY) {
1647                                         if (cred != NULL && jailed(cred))
1648                                                 jinp_wild = inp;
1649                                         else
1650                                                 local_wild = inp;
1651                                 }
1652                         }
1653                 }
1654
1655                 REL_PCBINFO_TOKEN(pcbinfo);
1656
1657                 if (local_wild != NULL)
1658                         return (local_wild);
1659                 if (jinp != NULL)
1660                         return (jinp);
1661                 return (jinp_wild);
1662         }
1663
1664         /*
1665          * Not found.
1666          */
1667         return (NULL);
1668 }
1669
1670 struct inpcb *
1671 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1672     u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1673     boolean_t wildcard, struct ifnet *ifp)
1674 {
1675         return in_pcblookup_pkthash(pcbinfo, faddr, fport_arg,
1676             laddr, lport_arg, wildcard, ifp, NULL);
1677 }
1678
1679 /*
1680  * Insert PCB into connection hash table.
1681  */
1682 void
1683 in_pcbinsconnhash(struct inpcb *inp)
1684 {
1685         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1686         struct inpcbhead *bucket;
1687         u_int32_t hashkey_faddr, hashkey_laddr;
1688
1689 #ifdef INET6
1690         if (INP_ISIPV6(inp)) {
1691                 hashkey_faddr = inp->in6p_faddr.s6_addr32[3] /* XXX JH */;
1692                 hashkey_laddr = inp->in6p_laddr.s6_addr32[3] /* XXX JH */;
1693         } else {
1694 #endif
1695                 hashkey_faddr = inp->inp_faddr.s_addr;
1696                 hashkey_laddr = inp->inp_laddr.s_addr;
1697 #ifdef INET6
1698         }
1699 #endif
1700
1701         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1702             ("not in the correct netisr"));
1703         ASSERT_INP_NOTINHASH(inp);
1704         inp->inp_flags |= INP_CONNECTED;
1705
1706         /*
1707          * Insert into the connection hash table.
1708          */
1709         bucket = &pcbinfo->hashbase[INP_PCBCONNHASH(hashkey_faddr,
1710             inp->inp_fport, hashkey_laddr, inp->inp_lport, pcbinfo->hashmask)];
1711         LIST_INSERT_HEAD(bucket, inp, inp_hash);
1712 }
1713
1714 /*
1715  * Remove PCB from connection hash table.
1716  */
1717 void
1718 in_pcbremconnhash(struct inpcb *inp)
1719 {
1720         struct inpcbinfo *pcbinfo __debugvar = inp->inp_pcbinfo;
1721
1722         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1723             ("not in the correct netisr"));
1724         KASSERT(inp->inp_flags & INP_CONNECTED, ("inp not connected"));
1725
1726         LIST_REMOVE(inp, inp_hash);
1727         inp->inp_flags &= ~INP_CONNECTED;
1728 }
1729
1730 /*
1731  * Insert PCB into port hash table.
1732  */
1733 void
1734 in_pcbinsporthash(struct inpcbporthead *pcbporthash, struct inpcb *inp)
1735 {
1736         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1737         struct inpcbport *phd;
1738
1739         /*
1740          * If the porthashbase is shared across several cpus, it must
1741          * have been locked.
1742          */
1743         ASSERT_PORTHASH_TOKEN_HELD(pcbporthash);
1744
1745         /*
1746          * Insert into the port hash table.
1747          */
1748
1749         /* Go through port list and look for a head for this lport. */
1750         LIST_FOREACH(phd, pcbporthash, phd_hash) {
1751                 if (phd->phd_port == inp->inp_lport)
1752                         break;
1753         }
1754
1755         /* If none exists, use saved one and tack it on. */
1756         if (phd == NULL) {
1757                 KKASSERT(pcbinfo->portsave != NULL);
1758                 phd = pcbinfo->portsave;
1759                 pcbinfo->portsave = NULL;
1760                 phd->phd_port = inp->inp_lport;
1761                 LIST_INIT(&phd->phd_pcblist);
1762                 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
1763         }
1764
1765         inp->inp_porthash = pcbporthash;
1766         inp->inp_phd = phd;
1767         LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
1768
1769         /*
1770          * Malloc one inpcbport for later use.  It is safe to use
1771          * "wait" malloc here (port token would be released, if
1772          * malloc ever blocked), since all changes to the porthash
1773          * are done.
1774          */
1775         if (pcbinfo->portsave == NULL) {
1776                 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave),
1777                                             M_PCB, M_INTWAIT | M_ZERO);
1778         }
1779 }
1780
1781 void
1782 in_pcbinsporthash_lport(struct inpcb *inp)
1783 {
1784         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1785         struct inpcbportinfo *portinfo;
1786         struct inpcbporthead *porthash;
1787         u_short lport_ho;
1788
1789         /* Locate the proper portinfo based on lport */
1790         lport_ho = ntohs(inp->inp_lport);
1791         portinfo = &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask];
1792         KKASSERT((lport_ho & pcbinfo->portinfo_mask) == portinfo->offset);
1793
1794         porthash = in_pcbporthash_head(portinfo, inp->inp_lport);
1795         GET_PORTHASH_TOKEN(porthash);
1796         in_pcbinsporthash(porthash, inp);
1797         REL_PORTHASH_TOKEN(porthash);
1798 }
1799
1800 void
1801 in_pcbremporthash(struct inpcb *inp)
1802 {
1803         struct inpcbporthead *porthash;
1804         struct inpcbport *phd;
1805
1806         if (inp->inp_phd == NULL)
1807                 return;
1808         KASSERT(inp->inp_lport != 0, ("inpcb has no lport"));
1809
1810         porthash = inp->inp_porthash;
1811         GET_PORTHASH_TOKEN(porthash);
1812
1813         phd = inp->inp_phd;
1814         LIST_REMOVE(inp, inp_portlist);
1815         if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
1816                 LIST_REMOVE(phd, phd_hash);
1817                 kfree(phd, M_PCB);
1818         }
1819
1820         REL_PORTHASH_TOKEN(porthash);
1821
1822         inp->inp_phd = NULL;
1823         /* NOTE: Don't whack inp_lport, which may be used later */
1824 }
1825
1826 static struct inp_localgroup *
1827 inp_localgroup_alloc(u_char af, uint16_t port,
1828     const union in_dependaddr *addr, int size)
1829 {
1830         struct inp_localgroup *grp;
1831
1832         grp = kmalloc(__offsetof(struct inp_localgroup, il_inp[size]),
1833             M_TEMP, M_INTWAIT | M_ZERO);
1834         grp->il_af = af;
1835         grp->il_lport = port;
1836         grp->il_dependladdr = *addr;
1837         grp->il_inpsiz = size;
1838
1839         return grp;
1840 }
1841
1842 static void
1843 inp_localgroup_free(struct inp_localgroup *grp)
1844 {
1845         kfree(grp, M_TEMP);
1846 }
1847
1848 static void
1849 inp_localgroup_destroy(struct inp_localgroup *grp)
1850 {
1851         LIST_REMOVE(grp, il_list);
1852         inp_localgroup_free(grp);
1853 }
1854
1855 static void
1856 inp_localgroup_copy(struct inp_localgroup *grp,
1857     const struct inp_localgroup *old_grp)
1858 {
1859         int i;
1860
1861         KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
1862             ("invalid new local group size %d and old local group count %d",
1863              grp->il_inpsiz, old_grp->il_inpcnt));
1864         for (i = 0; i < old_grp->il_inpcnt; ++i)
1865                 grp->il_inp[i] = old_grp->il_inp[i];
1866         grp->il_inpcnt = old_grp->il_inpcnt;
1867 }
1868
1869 static void
1870 in_pcbinslocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
1871 {
1872         struct inp_localgrphead *hdr;
1873         struct inp_localgroup *grp, *grp_alloc = NULL;
1874         struct ucred *cred;
1875         int i, idx;
1876
1877         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1878
1879         if (pcbinfo->localgrphashbase == NULL)
1880                 return;
1881
1882         /*
1883          * XXX don't allow jailed socket to join local group
1884          */
1885         if (inp->inp_socket != NULL)
1886                 cred = inp->inp_socket->so_cred;
1887         else
1888                 cred = NULL;
1889         if (cred != NULL && jailed(cred))
1890                 return;
1891
1892         hdr = &pcbinfo->localgrphashbase[
1893             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
1894
1895 again:
1896         LIST_FOREACH(grp, hdr, il_list) {
1897                 if (grp->il_af == inp->inp_af &&
1898                     grp->il_lport == inp->inp_lport &&
1899                     memcmp(&grp->il_dependladdr,
1900                         &inp->inp_inc.inc_ie.ie_dependladdr,
1901                         sizeof(grp->il_dependladdr)) == 0) {
1902                         break;
1903                 }
1904         }
1905         if (grp == NULL) {
1906                 /*
1907                  * Create a new local group
1908                  */
1909                 if (grp_alloc == NULL) {
1910                         grp_alloc = inp_localgroup_alloc(inp->inp_af,
1911                             inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
1912                             INP_LOCALGROUP_SIZMIN);
1913                         /*
1914                          * Local group allocation could block and the
1915                          * local group w/ the same property might have
1916                          * been added by others when we were blocked;
1917                          * check again.
1918                          */
1919                         goto again;
1920                 } else {
1921                         /* Local group has been allocated; link it */
1922                         grp = grp_alloc;
1923                         grp_alloc = NULL;
1924                         LIST_INSERT_HEAD(hdr, grp, il_list);
1925                 }
1926         } else if (grp->il_inpcnt == grp->il_inpsiz) {
1927                 if (grp->il_inpsiz >= INP_LOCALGROUP_SIZMAX) {
1928                         static int limit_logged = 0;
1929
1930                         if (!limit_logged) {
1931                                 limit_logged = 1;
1932                                 kprintf("local group port %d, "
1933                                     "limit reached\n", ntohs(grp->il_lport));
1934                         }
1935                         if (grp_alloc != NULL) {
1936                                 /*
1937                                  * This would happen if the local group
1938                                  * w/ the same property was expanded when
1939                                  * our local group allocation blocked.
1940                                  */
1941                                 inp_localgroup_free(grp_alloc);
1942                         }
1943                         return;
1944                 }
1945
1946                 /*
1947                  * Expand this local group
1948                  */
1949                 if (grp_alloc == NULL ||
1950                     grp->il_inpcnt >= grp_alloc->il_inpsiz) {
1951                         if (grp_alloc != NULL)
1952                                 inp_localgroup_free(grp_alloc);
1953                         grp_alloc = inp_localgroup_alloc(grp->il_af,
1954                             grp->il_lport, &grp->il_dependladdr,
1955                             grp->il_inpsiz * 2);
1956                         /*
1957                          * Local group allocation could block and the
1958                          * local group w/ the same property might have
1959                          * been expanded by others when we were blocked;
1960                          * check again.
1961                          */
1962                         goto again;
1963                 }
1964
1965                 /*
1966                  * Save the old local group, link the new one, and then
1967                  * destroy the old local group
1968                  */
1969                 inp_localgroup_copy(grp_alloc, grp);
1970                 LIST_INSERT_HEAD(hdr, grp_alloc, il_list);
1971                 inp_localgroup_destroy(grp);
1972
1973                 grp = grp_alloc;
1974                 grp_alloc = NULL;
1975         } else {
1976                 /*
1977                  * Found the local group
1978                  */
1979                 if (grp_alloc != NULL) {
1980                         /*
1981                          * This would happen if the local group w/ the
1982                          * same property was added or expanded when our
1983                          * local group allocation blocked.
1984                          */
1985                         inp_localgroup_free(grp_alloc);
1986                         grp_alloc = NULL;
1987                 }
1988         }
1989
1990         KASSERT(grp->il_inpcnt < grp->il_inpsiz,
1991             ("invalid local group size %d and count %d",
1992              grp->il_inpsiz, grp->il_inpcnt));
1993
1994         /*
1995          * Keep the local group sorted by the inpcb local group index
1996          * in ascending order.
1997          *
1998          * This eases the multi-process userland application which uses
1999          * SO_REUSEPORT sockets and binds process to the owner cpu of
2000          * the SO_REUSEPORT socket:
2001          * If we didn't sort the local group by the inpcb local group
2002          * index and one of the process owning an inpcb in this local
2003          * group restarted, e.g. crashed and restarted by watchdog,
2004          * other processes owning a inpcb in this local group would have
2005          * to detect that event, refetch its socket's owner cpu, and
2006          * re-bind.
2007          */
2008         idx = grp->il_inpcnt;
2009         for (i = 0; i < idx; ++i) {
2010                 struct inpcb *oinp = grp->il_inp[i];
2011
2012                 if (oinp->inp_lgrpindex > i) {
2013                         if (inp->inp_lgrpindex < 0) {
2014                                 inp->inp_lgrpindex = i;
2015                         } else if (inp->inp_lgrpindex != i) {
2016                                 if (bootverbose) {
2017                                         kprintf("inp %p: grpidx %d, "
2018                                             "assigned to %d, cpu%d\n",
2019                                             inp, inp->inp_lgrpindex, i,
2020                                             mycpuid);
2021                                 }
2022                         }
2023                         grp->il_inp[i] = inp;
2024
2025                         /* Pull down inpcbs */
2026                         for (; i < grp->il_inpcnt; ++i) {
2027                                 struct inpcb *oinp1 = grp->il_inp[i + 1];
2028
2029                                 grp->il_inp[i + 1] = oinp;
2030                                 oinp = oinp1;
2031                         }
2032                         grp->il_inpcnt++;
2033                         return;
2034                 }
2035         }
2036
2037         if (inp->inp_lgrpindex < 0) {
2038                 inp->inp_lgrpindex = idx;
2039         } else if (inp->inp_lgrpindex != idx) {
2040                 if (bootverbose) {
2041                         kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n",
2042                             inp, inp->inp_lgrpindex, idx, mycpuid);
2043                 }
2044         }
2045         grp->il_inp[idx] = inp;
2046         grp->il_inpcnt++;
2047 }
2048
2049 void
2050 in_pcbinswildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2051 {
2052         struct inpcontainer *ic;
2053         struct inpcontainerhead *bucket;
2054
2055         GET_PCBINFO_TOKEN(pcbinfo);
2056
2057         in_pcbinslocalgrphash_oncpu(inp, pcbinfo);
2058
2059         bucket = &pcbinfo->wildcardhashbase[
2060             INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
2061
2062         ic = kmalloc(sizeof(struct inpcontainer), M_TEMP, M_INTWAIT);
2063         ic->ic_inp = inp;
2064         LIST_INSERT_HEAD(bucket, ic, ic_list);
2065
2066         REL_PCBINFO_TOKEN(pcbinfo);
2067 }
2068
2069 /*
2070  * Insert PCB into wildcard hash table.
2071  */
2072 void
2073 in_pcbinswildcardhash(struct inpcb *inp)
2074 {
2075         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2076
2077         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
2078             ("not in correct netisr"));
2079         ASSERT_INP_NOTINHASH(inp);
2080         inp->inp_flags |= INP_WILDCARD;
2081
2082         in_pcbinswildcardhash_oncpu(inp, pcbinfo);
2083 }
2084
2085 static void
2086 in_pcbremlocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2087 {
2088         struct inp_localgrphead *hdr;
2089         struct inp_localgroup *grp;
2090
2091         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
2092
2093         if (pcbinfo->localgrphashbase == NULL)
2094                 return;
2095
2096         hdr = &pcbinfo->localgrphashbase[
2097             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
2098
2099         LIST_FOREACH(grp, hdr, il_list) {
2100                 int i;
2101
2102                 for (i = 0; i < grp->il_inpcnt; ++i) {
2103                         if (grp->il_inp[i] != inp)
2104                                 continue;
2105
2106                         if (grp->il_inpcnt == 1) {
2107                                 /* Destroy this local group */
2108                                 inp_localgroup_destroy(grp);
2109                         } else {
2110                                 /* Pull up inpcbs */
2111                                 for (; i + 1 < grp->il_inpcnt; ++i)
2112                                         grp->il_inp[i] = grp->il_inp[i + 1];
2113                                 grp->il_inpcnt--;
2114                         }
2115                         return;
2116                 }
2117         }
2118 }
2119
2120 void
2121 in_pcbremwildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2122 {
2123         struct inpcontainer *ic;
2124         struct inpcontainerhead *head;
2125
2126         GET_PCBINFO_TOKEN(pcbinfo);
2127
2128         in_pcbremlocalgrphash_oncpu(inp, pcbinfo);
2129
2130         /* find bucket */
2131         head = &pcbinfo->wildcardhashbase[
2132             INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
2133
2134         LIST_FOREACH(ic, head, ic_list) {
2135                 if (ic->ic_inp == inp)
2136                         goto found;
2137         }
2138         REL_PCBINFO_TOKEN(pcbinfo);
2139         return;                 /* not found! */
2140
2141 found:
2142         LIST_REMOVE(ic, ic_list);       /* remove container from bucket chain */
2143         REL_PCBINFO_TOKEN(pcbinfo);
2144         kfree(ic, M_TEMP);              /* deallocate container */
2145 }
2146
2147 /*
2148  * Remove PCB from wildcard hash table.
2149  */
2150 void
2151 in_pcbremwildcardhash(struct inpcb *inp)
2152 {
2153         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2154
2155         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
2156             ("not in correct netisr"));
2157         KASSERT(inp->inp_flags & INP_WILDCARD, ("inp not wildcard"));
2158
2159         in_pcbremwildcardhash_oncpu(inp, pcbinfo);
2160         inp->inp_lgrpindex = -1;
2161         inp->inp_flags &= ~INP_WILDCARD;
2162 }
2163
2164 /*
2165  * Remove PCB from various lists.
2166  */
2167 void
2168 in_pcbremlists(struct inpcb *inp)
2169 {
2170         in_pcbremporthash(inp);
2171         if (inp->inp_flags & INP_WILDCARD) {
2172                 in_pcbremwildcardhash(inp);
2173         } else if (inp->inp_flags & INP_CONNECTED) {
2174                 in_pcbremconnhash(inp);
2175         }
2176
2177         if (inp->inp_flags & INP_ONLIST)
2178                 in_pcbofflist(inp);
2179 }
2180
2181 int
2182 prison_xinpcb(struct thread *td, struct inpcb *inp)
2183 {
2184         struct ucred *cr;
2185
2186         if (td->td_proc == NULL)
2187                 return (0);
2188         cr = td->td_proc->p_ucred;
2189         if (cr->cr_prison == NULL)
2190                 return (0);
2191         if (inp->inp_socket && inp->inp_socket->so_cred &&
2192             inp->inp_socket->so_cred->cr_prison &&
2193             cr->cr_prison == inp->inp_socket->so_cred->cr_prison)
2194                 return (0);
2195         return (1);
2196 }
2197
2198 int
2199 in_pcblist_global(SYSCTL_HANDLER_ARGS)
2200 {
2201         struct inpcbinfo *pcbinfo_arr = arg1;
2202         int pcbinfo_arrlen = arg2;
2203         struct inpcb *marker;
2204         int cpu, origcpu;
2205         int error, n;
2206
2207         KASSERT(pcbinfo_arrlen <= ncpus && pcbinfo_arrlen >= 1,
2208             ("invalid pcbinfo count %d", pcbinfo_arrlen));
2209
2210         /*
2211          * The process of preparing the TCB list is too time-consuming and
2212          * resource-intensive to repeat twice on every request.
2213          */
2214         n = 0;
2215         if (req->oldptr == NULL) {
2216                 for (cpu = 0; cpu < pcbinfo_arrlen; ++cpu)
2217                         n += pcbinfo_arr[cpu].ipi_count;
2218                 req->oldidx = (n + n/8 + 10) * sizeof(struct xinpcb);
2219                 return 0;
2220         }
2221
2222         if (req->newptr != NULL)
2223                 return EPERM;
2224
2225         marker = kmalloc(sizeof(struct inpcb), M_TEMP, M_WAITOK|M_ZERO);
2226         marker->inp_flags |= INP_PLACEMARKER;
2227
2228         /*
2229          * OK, now we're committed to doing something.  Re-fetch ipi_count
2230          * after obtaining the generation count.
2231          */
2232         error = 0;
2233         origcpu = mycpuid;
2234         for (cpu = 0; cpu < pcbinfo_arrlen && error == 0; ++cpu) {
2235                 struct inpcbinfo *pcbinfo = &pcbinfo_arr[cpu];
2236                 struct inpcb *inp;
2237                 struct xinpcb xi;
2238                 int i;
2239
2240                 lwkt_migratecpu(cpu);
2241
2242                 GET_PCBINFO_TOKEN(pcbinfo);
2243
2244                 n = pcbinfo->ipi_count;
2245
2246                 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
2247                 i = 0;
2248                 while ((inp = LIST_NEXT(marker, inp_list)) != NULL && i < n) {
2249                         LIST_REMOVE(marker, inp_list);
2250                         LIST_INSERT_AFTER(inp, marker, inp_list);
2251
2252                         if (inp->inp_flags & INP_PLACEMARKER)
2253                                 continue;
2254                         if (prison_xinpcb(req->td, inp))
2255                                 continue;
2256
2257                         bzero(&xi, sizeof xi);
2258                         xi.xi_len = sizeof xi;
2259                         bcopy(inp, &xi.xi_inp, sizeof *inp);
2260                         if (inp->inp_socket)
2261                                 sotoxsocket(inp->inp_socket, &xi.xi_socket);
2262                         if ((error = SYSCTL_OUT(req, &xi, sizeof xi)) != 0)
2263                                 break;
2264                         ++i;
2265                 }
2266                 LIST_REMOVE(marker, inp_list);
2267
2268                 REL_PCBINFO_TOKEN(pcbinfo);
2269
2270                 if (error == 0 && i < n) {
2271                         bzero(&xi, sizeof xi);
2272                         xi.xi_len = sizeof xi;
2273                         while (i < n) {
2274                                 error = SYSCTL_OUT(req, &xi, sizeof xi);
2275                                 if (error)
2276                                         break;
2277                                 ++i;
2278                         }
2279                 }
2280         }
2281
2282         lwkt_migratecpu(origcpu);
2283         kfree(marker, M_TEMP);
2284         return error;
2285 }
2286
2287 int
2288 in_pcblist_global_ncpus2(SYSCTL_HANDLER_ARGS)
2289 {
2290         return in_pcblist_global(oidp, arg1, ncpus2, req);
2291 }
2292
2293 void
2294 in_savefaddr(struct socket *so, const struct sockaddr *faddr)
2295 {
2296         struct sockaddr_in *sin;
2297
2298         KASSERT(faddr->sa_family == AF_INET,
2299             ("not AF_INET faddr %d", faddr->sa_family));
2300
2301         sin = kmalloc(sizeof(*sin), M_SONAME, M_WAITOK | M_ZERO);
2302         sin->sin_family = AF_INET;
2303         sin->sin_len = sizeof(*sin);
2304         sin->sin_port = ((const struct sockaddr_in *)faddr)->sin_port;
2305         sin->sin_addr = ((const struct sockaddr_in *)faddr)->sin_addr;
2306
2307         so->so_faddr = (struct sockaddr *)sin;
2308 }
2309
2310 void
2311 in_pcbportinfo_init(struct inpcbportinfo *portinfo, int hashsize,
2312     u_short offset)
2313 {
2314         memset(portinfo, 0, sizeof(*portinfo));
2315
2316         portinfo->offset = offset;
2317         portinfo->lastport = offset;
2318         portinfo->lastlow = offset;
2319         portinfo->lasthi = offset;
2320
2321         portinfo->porthashbase = hashinit(hashsize, M_PCB,
2322             &portinfo->porthashmask);
2323 }
2324
2325 void
2326 in_pcbportrange(u_short *hi0, u_short *lo0, u_short ofs, u_short step)
2327 {
2328         int hi, lo;
2329
2330         if (step == 1)
2331                 return;
2332
2333         hi = *hi0;
2334         lo = *lo0;
2335
2336         hi = rounddown2(hi, step);
2337         hi += ofs;
2338         if (hi > (int)*hi0)
2339                 hi -= step;
2340
2341         lo = roundup2(lo, step);
2342         lo -= (step - ofs);
2343         if (lo < (int)*lo0)
2344                 lo += step;
2345
2346         *hi0 = hi;
2347         *lo0 = lo;
2348 }
2349
2350 void
2351 in_pcbglobalinit(void)
2352 {
2353         int cpu;
2354
2355         in_pcbmarkers = kmalloc(ncpus * sizeof(struct inpcb), M_PCB,
2356             M_WAITOK | M_ZERO);
2357         in_pcbcontainer_markers = kmalloc(ncpus * sizeof(struct inpcontainer),
2358             M_PCB, M_WAITOK | M_ZERO);
2359
2360         for (cpu = 0; cpu < ncpus; ++cpu) {
2361                 struct inpcontainer *ic = &in_pcbcontainer_markers[cpu];
2362                 struct inpcb *marker = &in_pcbmarkers[cpu];
2363
2364                 marker->inp_flags |= INP_PLACEMARKER;
2365                 ic->ic_inp = marker;
2366         }
2367 }
2368
2369 struct inpcb *
2370 in_pcbmarker(int cpuid)
2371 {
2372         KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid));
2373         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
2374
2375         return &in_pcbmarkers[cpuid];
2376 }
2377
2378 struct inpcontainer *
2379 in_pcbcontainer_marker(int cpuid)
2380 {
2381         KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid));
2382         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
2383
2384         return &in_pcbcontainer_markers[cpuid];
2385 }
2386
2387 void
2388 in_pcbresetroute(struct inpcb *inp)
2389 {
2390         struct route *ro = &inp->inp_route;
2391
2392         if (ro->ro_rt != NULL)
2393                 RTFREE(ro->ro_rt);
2394         bzero(ro, sizeof(*ro));
2395 }
2396
2397 u_short
2398 in_pcblastport_down(volatile u_short *lastport, u_short first, u_short last,
2399     u_short step)
2400 {
2401         u_short lport;
2402
2403         for (;;) {
2404                 u_short olport;
2405
2406                 olport = *lastport;
2407                 lport = olport - step;
2408                 if (__predict_false(lport > first || lport < last))
2409                         lport = first;
2410                 if (atomic_cmpset_short(lastport, olport, lport))
2411                         break;
2412         }
2413         return lport;
2414 }
2415
2416 u_short
2417 in_pcblastport_up(volatile u_short *lastport, u_short first, u_short last,
2418     u_short step)
2419 {
2420         u_short lport;
2421
2422         for (;;) {
2423                 u_short olport;
2424
2425                 olport = *lastport;
2426                 lport = olport + step;
2427                 if (__predict_false(lport < first || lport > last))
2428                         lport = first;
2429                 if (atomic_cmpset_short(lastport, olport, lport))
2430                         break;
2431         }
2432         return lport;
2433 }