sys/netinet/in_pcb.c

   1 /*
   2  * Copyright (c) 2004 Jeffrey M. Hsu.  All rights reserved.
   3  * Copyright (c) 2004 The DragonFly Project.  All rights reserved.
   4  *
   5  * This code is derived from software contributed to The DragonFly Project
   6  * by Jeffrey M. Hsu.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the name of The DragonFly Project nor the names of its
  17  *    contributors may be used to endorse or promote products derived
  18  *    from this software without specific, prior written permission.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  23  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  24  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  25  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  26  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  27  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  28  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  29  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  30  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  */
  33
  34 /*
  35  * Copyright (c) 1982, 1986, 1991, 1993, 1995
  36  *      The Regents of the University of California.  All rights reserved.
  37  *
  38  * Redistribution and use in source and binary forms, with or without
  39  * modification, are permitted provided that the following conditions
  40  * are met:
  41  * 1. Redistributions of source code must retain the above copyright
  42  *    notice, this list of conditions and the following disclaimer.
  43  * 2. Redistributions in binary form must reproduce the above copyright
  44  *    notice, this list of conditions and the following disclaimer in the
  45  *    documentation and/or other materials provided with the distribution.
  46  * 3. Neither the name of the University nor the names of its contributors
  47  *    may be used to endorse or promote products derived from this software
  48  *    without specific prior written permission.
  49  *
  50  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  51  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  52  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  53  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  54  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  55  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  56  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  57  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  58  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  59  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  60  * SUCH DAMAGE.
  61  *
  62  *      @(#)in_pcb.c    8.4 (Berkeley) 5/24/95
  63  * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $
  64  */
  65
  66 #include "opt_inet6.h"
  67
  68 #include <sys/param.h>
  69 #include <sys/systm.h>
  70 #include <sys/malloc.h>
  71 #include <sys/mbuf.h>
  72 #include <sys/domain.h>
  73 #include <sys/protosw.h>
  74 #include <sys/socket.h>
  75 #include <sys/socketvar.h>
  76 #include <sys/proc.h>
  77 #include <sys/caps.h>
  78 #include <sys/jail.h>
  79 #include <sys/kernel.h>
  80 #include <sys/sysctl.h>
  81
  82 #include <sys/socketvar2.h>
  83 #include <sys/msgport2.h>
  84
  85 #include <machine/limits.h>
  86
  87 #include <net/if.h>
  88 #include <net/if_types.h>
  89 #include <net/route.h>
  90 #include <net/netisr2.h>
  91 #include <net/toeplitz2.h>
  92
  93 #include <netinet/in.h>
  94 #include <netinet/in_pcb.h>
  95 #include <netinet/in_var.h>
  96 #include <netinet/ip_var.h>
  97 #ifdef INET6
  98 #include <netinet/ip6.h>
  99 #include <netinet6/ip6_var.h>
 100 #endif /* INET6 */
 101
 102 #define INP_LOCALGROUP_SIZMIN   8
 103 #define INP_LOCALGROUP_SIZMAX   256
 104
 105 static struct inpcb *in_pcblookup_local(struct inpcbporthead *porthash,
 106                 struct in_addr laddr, u_int lport_arg, int wild_okay,
 107                 struct ucred *cred);
 108
 109 struct in_addr zeroin_addr;
 110
 111 /*
 112  * These configure the range of local port addresses assigned to
 113  * "unspecified" outgoing connections/packets/whatever.
 114  */
 115 int ipport_lowfirstauto = IPPORT_RESERVED - 1;  /* 1023 */
 116 int ipport_lowlastauto = IPPORT_RESERVEDSTART;  /* 600 */
 117
 118 int ipport_firstauto = IPPORT_RESERVED;         /* 1024 */
 119 int ipport_lastauto = IPPORT_USERRESERVED;      /* 5000 */
 120
 121 int ipport_hifirstauto = IPPORT_HIFIRSTAUTO;    /* 49152 */
 122 int ipport_hilastauto = IPPORT_HILASTAUTO;      /* 65535 */
 123
 124 #define RANGECHK(var, min, max) \
 125         if ((var) < (min)) { (var) = (min); } \
 126         else if ((var) > (max)) { (var) = (max); }
 127
 128 int udpencap_enable = 1;        /* enabled by default */
 129 int udpencap_port = 4500;       /* triggers decapsulation */
 130
 131 /*
 132  * Per-netisr inpcb markers.
 133  * NOTE: they should only be used in netisrs.
 134  */
 135 static struct inpcb             *in_pcbmarkers;
 136 static struct inpcontainer      *in_pcbcontainer_markers;
 137
 138 static int
 139 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
 140 {
 141         int error;
 142
 143         error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
 144         if (!error) {
 145                 RANGECHK(ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
 146                 RANGECHK(ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
 147
 148                 RANGECHK(ipport_firstauto, IPPORT_RESERVED, USHRT_MAX);
 149                 RANGECHK(ipport_lastauto, IPPORT_RESERVED, USHRT_MAX);
 150
 151                 RANGECHK(ipport_hifirstauto, IPPORT_RESERVED, USHRT_MAX);
 152                 RANGECHK(ipport_hilastauto, IPPORT_RESERVED, USHRT_MAX);
 153         }
 154         return (error);
 155 }
 156
 157 #undef RANGECHK
 158
 159 SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports");
 160
 161 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW,
 162            &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", "");
 163 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, CTLTYPE_INT|CTLFLAG_RW,
 164            &ipport_lowlastauto, 0, &sysctl_net_ipport_check, "I", "");
 165 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, CTLTYPE_INT|CTLFLAG_RW,
 166            &ipport_firstauto, 0, &sysctl_net_ipport_check, "I", "");
 167 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, CTLTYPE_INT|CTLFLAG_RW,
 168            &ipport_lastauto, 0, &sysctl_net_ipport_check, "I", "");
 169 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, CTLTYPE_INT|CTLFLAG_RW,
 170            &ipport_hifirstauto, 0, &sysctl_net_ipport_check, "I", "");
 171 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW,
 172            &ipport_hilastauto, 0, &sysctl_net_ipport_check, "I", "");
 173
 174 /* Initialized by ip_init() */
 175 int ip_porthash_trycount;
 176 SYSCTL_INT(_net_inet_ip, OID_AUTO, porthash_trycount, CTLFLAG_RW,
 177     &ip_porthash_trycount, 0,
 178     "Number of tries to find local port matching hash of 4-tuple");
 179
 180 /*
 181  * in_pcb.c: manage the Protocol Control Blocks.
 182  *
 183  * NOTE: It is assumed that most of these functions will be called from
 184  * a critical section.  XXX - There are, unfortunately, a few exceptions
 185  * to this rule that should be fixed.
 186  *
 187  * NOTE: The caller should initialize the cpu field to the cpu running the
 188  * protocol stack associated with this inpcbinfo.
 189  */
 190
 191 void
 192 in_pcbinfo_init(struct inpcbinfo *pcbinfo, int cpu, boolean_t shared)
 193 {
 194         KASSERT(cpu >= 0 && cpu < netisr_ncpus, ("invalid cpu%d", cpu));
 195         pcbinfo->cpu = cpu;
 196
 197         LIST_INIT(&pcbinfo->pcblisthead);
 198         pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), M_PCB,
 199                                     M_WAITOK | M_ZERO);
 200
 201         if (shared) {
 202                 pcbinfo->infotoken = kmalloc(sizeof(struct lwkt_token),
 203                     M_PCB, M_WAITOK);
 204                 lwkt_token_init(pcbinfo->infotoken, "infotoken");
 205         } else {
 206                 pcbinfo->infotoken = NULL;
 207         }
 208 }
 209
 210 void
 211 in_pcbportinfo_set(struct inpcbinfo *pcbinfo, struct inpcbportinfo *portinfo,
 212     int portinfo_cnt)
 213 {
 214
 215         KASSERT(portinfo_cnt > 0, ("invalid portinfo_cnt %d", portinfo_cnt));
 216         pcbinfo->portinfo = portinfo;
 217         pcbinfo->portinfo_cnt = portinfo_cnt;
 218 }
 219
 220 struct baddynamicports baddynamicports;
 221
 222 /*
 223  * Check if the specified port is invalid for dynamic allocation.
 224  */
 225 int
 226 in_baddynamic(u_int16_t port, u_int16_t proto)
 227 {
 228         switch (proto) {
 229         case IPPROTO_TCP:
 230                 return (DP_ISSET(baddynamicports.tcp, port));
 231         case IPPROTO_UDP:
 232                 return (DP_ISSET(baddynamicports.udp, port));
 233         default:
 234                 return (0);
 235         }
 236 }
 237
 238 void
 239 in_pcbonlist(struct inpcb *inp)
 240 {
 241         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 242
 243         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
 244             ("not in the correct netisr"));
 245         KASSERT((inp->inp_flags & INP_ONLIST) == 0, ("already on pcblist"));
 246         inp->inp_flags |= INP_ONLIST;
 247
 248         GET_PCBINFO_TOKEN(pcbinfo);
 249         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list);
 250         pcbinfo->ipi_count++;
 251         REL_PCBINFO_TOKEN(pcbinfo);
 252 }
 253
 254 void
 255 in_pcbofflist(struct inpcb *inp)
 256 {
 257         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 258
 259         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
 260             ("not in the correct netisr"));
 261         KASSERT(inp->inp_flags & INP_ONLIST, ("not on pcblist"));
 262         inp->inp_flags &= ~INP_ONLIST;
 263
 264         GET_PCBINFO_TOKEN(pcbinfo);
 265         LIST_REMOVE(inp, inp_list);
 266         KASSERT(pcbinfo->ipi_count > 0,
 267             ("invalid inpcb count %d", pcbinfo->ipi_count));
 268         pcbinfo->ipi_count--;
 269         REL_PCBINFO_TOKEN(pcbinfo);
 270 }
 271
 272 /*
 273  * Allocate a PCB and associate it with the socket.
 274  */
 275 int
 276 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
 277 {
 278         struct inpcb *inp;
 279
 280         inp = kmalloc(pcbinfo->ipi_size, M_PCB, M_WAITOK|M_ZERO|M_NULLOK);
 281         if (inp == NULL)
 282                 return (ENOMEM);
 283         inp->inp_lgrpindex = -1;
 284         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
 285         inp->inp_pcbinfo = pcbinfo;
 286         inp->inp_socket = so;
 287 #ifdef INET6
 288         if (INP_CHECK_SOCKAF(so, AF_INET6)) {
 289                 if (ip6_auto_flowlabel)
 290                         inp->inp_flags |= IN6P_AUTOFLOWLABEL;
 291                 inp->inp_af = AF_INET6;
 292         } else
 293 #endif
 294         inp->inp_af = AF_INET;
 295         soreference(so);
 296         so->so_pcb = inp;
 297
 298         in_pcbonlist(inp);
 299         return (0);
 300 }
 301
 302 /*
 303  * Unlink a pcb with the intention of moving it to another cpu with a
 304  * different pcbinfo.  While unlinked nothing should attempt to dereference
 305  * inp_pcbinfo, NULL it out so we assert if it does.
 306  */
 307 void
 308 in_pcbunlink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags)
 309 {
 310         KASSERT(inp->inp_pcbinfo == pcbinfo, ("pcbinfo mismatch"));
 311         KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0,
 312             ("already linked"));
 313
 314         in_pcbofflist(inp);
 315         inp->inp_pcbinfo = NULL;
 316 }
 317
 318 void
 319 in_pcbunlink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
 320 {
 321         in_pcbunlink_flags(inp, pcbinfo, INP_WILDCARD);
 322 }
 323
 324 /*
 325  * Relink a pcb into a new pcbinfo.
 326  */
 327 void
 328 in_pcblink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags)
 329 {
 330         KASSERT(inp->inp_pcbinfo == NULL, ("has pcbinfo"));
 331         KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0,
 332             ("already linked"));
 333
 334         inp->inp_pcbinfo = pcbinfo;
 335         in_pcbonlist(inp);
 336 }
 337
 338 void
 339 in_pcblink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
 340 {
 341         return in_pcblink_flags(inp, pcbinfo, INP_WILDCARD);
 342 }
 343
 344 static boolean_t
 345 in_pcbporthash_update(struct inpcbportinfo *portinfo,
 346     struct inpcb *inp, u_short lport, struct ucred *cred, int wild)
 347 {
 348         struct inpcbporthead *porthash;
 349
 350         /*
 351          * This has to be atomic.  If the porthash is shared across multiple
 352          * protocol threads, e.g. tcp and udp, then the token must be held.
 353          */
 354         porthash = in_pcbporthash_head(portinfo, lport);
 355         GET_PORTHASH_TOKEN(porthash);
 356
 357         if (in_pcblookup_local(porthash, inp->inp_laddr, lport, wild, cred)) {
 358                 REL_PORTHASH_TOKEN(porthash);
 359                 return FALSE;
 360         }
 361         inp->inp_lport = lport;
 362         in_pcbinsporthash(porthash, inp);
 363
 364         REL_PORTHASH_TOKEN(porthash);
 365         return TRUE;
 366 }
 367
 368 static int
 369 in_pcbsetlport(struct inpcb *inp, int wild, struct ucred *cred)
 370 {
 371         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 372         struct inpcbportinfo *portinfo;
 373         u_short first, last, lport, step, first0, last0;
 374         int count, error;
 375         int portinfo_first, portinfo_idx;
 376         uint32_t cut;
 377
 378         inp->inp_flags |= INP_ANONPORT;
 379
 380         step = pcbinfo->portinfo_cnt;
 381         portinfo_first = mycpuid % pcbinfo->portinfo_cnt;
 382         portinfo_idx = portinfo_first;
 383
 384         if (inp->inp_flags & INP_HIGHPORT) {
 385                 first0 = ipport_hifirstauto;    /* sysctl */
 386                 last0  = ipport_hilastauto;
 387         } else if (inp->inp_flags & INP_LOWPORT) {
 388                 if (cred &&
 389                     (error = caps_priv_check(cred, SYSCAP_NONET_RESPORT)))
 390                 {
 391                         inp->inp_laddr.s_addr = INADDR_ANY;
 392                         return error;
 393                 }
 394                 first0 = ipport_lowfirstauto;   /* 1023 */
 395                 last0  = ipport_lowlastauto;    /* 600 */
 396         } else {
 397                 first0 = ipport_firstauto;      /* sysctl */
 398                 last0  = ipport_lastauto;
 399         }
 400         if (first0 > last0) {
 401                 lport = last0;
 402                 last0 = first0;
 403                 first0 = lport;
 404         }
 405         KKASSERT(last0 >= first0);
 406
 407         cut = karc4random();
 408 loop:
 409         portinfo = &pcbinfo->portinfo[portinfo_idx];
 410         first = first0;
 411         last = last0;
 412
 413         /*
 414          * Simple check to ensure all ports are not used up causing
 415          * a deadlock here.
 416          */
 417         in_pcbportrange(&last, &first, portinfo->offset, step);
 418         lport = last - first;
 419         count = lport / step;
 420
 421         lport = rounddown(cut % lport, step) + first;
 422         KKASSERT(lport % step == portinfo->offset);
 423
 424         for (;;) {
 425                 if (count-- < 0) {      /* completely used? */
 426                         error = EADDRNOTAVAIL;
 427                         break;
 428                 }
 429
 430                 if (__predict_false(lport < first || lport > last)) {
 431                         lport = first;
 432                         KKASSERT(lport % step == portinfo->offset);
 433                 }
 434
 435                 if (in_pcbporthash_update(portinfo, inp, htons(lport),
 436                                           cred, wild)) {
 437                         error = 0;
 438                         break;
 439                 }
 440
 441                 lport += step;
 442                 KKASSERT(lport % step == portinfo->offset);
 443         }
 444
 445         if (error) {
 446                 /* Try next portinfo */
 447                 portinfo_idx++;
 448                 portinfo_idx %= pcbinfo->portinfo_cnt;
 449                 if (portinfo_idx != portinfo_first)
 450                         goto loop;
 451                 inp->inp_laddr.s_addr = INADDR_ANY;
 452         }
 453         return error;
 454 }
 455
 456 static __inline struct inpcbporthead *
 457 OBTAIN_LPORTHASH_TOKEN(struct inpcbinfo *pcbinfo, u_short lport)
 458 {
 459         struct inpcbportinfo *portinfo;
 460         struct inpcbporthead *porthash;
 461         u_short lport_ho = ntohs(lport);
 462
 463         /*
 464          * Locate the proper portinfo based on lport
 465          */
 466         portinfo = &pcbinfo->portinfo[lport_ho % pcbinfo->portinfo_cnt];
 467         KKASSERT((lport_ho % pcbinfo->portinfo_cnt) == portinfo->offset);
 468
 469         porthash = in_pcbporthash_head(portinfo, lport);
 470         GET_PORTHASH_TOKEN(porthash);
 471
 472         return porthash;
 473 }
 474
 475 static int
 476 in_pcbbind_laddr(struct sockaddr_in *sin, struct in_addr *laddr,
 477     struct thread *td)
 478 {
 479         struct sockaddr_in jsin;
 480
 481         if (!prison_replace_wildcards(td, (struct sockaddr *)sin))
 482                 return (EINVAL);
 483
 484         if (!IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
 485             sin->sin_addr.s_addr != INADDR_ANY) {
 486                 sin->sin_port = 0;              /* yech... */
 487                 bzero(&sin->sin_zero, sizeof sin->sin_zero);
 488                 if (ifa_ifwithaddr((struct sockaddr *)sin) == NULL)
 489                         return (EADDRNOTAVAIL);
 490         }
 491
 492         *laddr = sin->sin_addr;
 493
 494         jsin.sin_family = AF_INET;
 495         jsin.sin_addr.s_addr = laddr->s_addr;
 496         if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 497                 laddr->s_addr = INADDR_ANY;
 498                 return (EINVAL);
 499         }
 500         laddr->s_addr = jsin.sin_addr.s_addr;
 501
 502         return (0);
 503 }
 504
 505 static int
 506 in_pcbbind_laddrport_check(const struct socket *so, struct sockaddr_in *sin,
 507     struct inpcbporthead *porthash, int wild, struct ucred *cred,
 508     struct thread *td)
 509 {
 510         int reuseport = (so->so_options & SO_REUSEPORT);
 511         struct inpcb *t;
 512
 513         ASSERT_PORTHASH_TOKEN_HELD(porthash);
 514
 515         if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 516                 /*
 517                  * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
 518                  * allow complete duplication of binding if
 519                  * SO_REUSEPORT is set, or if SO_REUSEADDR is set
 520                  * and a multicast address is bound on both
 521                  * new and duplicated sockets.
 522                  */
 523                 if (so->so_options & SO_REUSEADDR)
 524                         reuseport = SO_REUSEADDR | SO_REUSEPORT;
 525         }
 526
 527         if (so->so_cred->cr_uid != 0 &&
 528             !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 529                 t = in_pcblookup_local(porthash, sin->sin_addr, sin->sin_port,
 530                                        INPLOOKUP_WILDCARD, cred);
 531                 if (t && t->inp_socket != so &&
 532                     (so->so_cred->cr_uid != t->inp_socket->so_cred->cr_uid))
 533                         return (EADDRINUSE);
 534         }
 535         if (cred && !prison_replace_wildcards(td, (struct sockaddr *)sin))
 536                 return (EADDRNOTAVAIL);
 537
 538         /*
 539          * When binding to a local port if the best match is against
 540          * an accepted socket we generally want to allow the binding.
 541          * This means that there is no longer any specific socket
 542          * bound or bound for listening.
 543          */
 544         t = in_pcblookup_local(porthash, sin->sin_addr, sin->sin_port,
 545                                wild, cred);
 546         if (t && t->inp_socket != so &&
 547             (reuseport & t->inp_socket->so_options) == 0 &&
 548             (t->inp_socket->so_state & SS_ACCEPTMECH) == 0)
 549                 return (EADDRINUSE);
 550
 551         return (0);
 552 }
 553
 554 int
 555 in_pcbsrcaddr_check(const struct inpcb *inp, struct sockaddr_in *sin,
 556     struct in_addr *laddr, struct thread *td)
 557 {
 558         const struct socket *so = inp->inp_socket;
 559         struct inpcbporthead *porthash;
 560         struct ucred *cred = NULL;
 561         int wild = 0;
 562         int error;
 563
 564         /* inp must be bound beforehand. */
 565         KKASSERT(inp->inp_lport != 0);
 566         KKASSERT(sin->sin_len == sizeof(*sin));
 567
 568         if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT)))
 569                 wild = 1;    /* neither SO_REUSEADDR nor SO_REUSEPORT is set */
 570         if (td->td_proc)
 571                 cred = td->td_proc->p_ucred;
 572
 573         /* Always use inp_lport */
 574         sin->sin_port = inp->inp_lport;
 575
 576         error = in_pcbbind_laddr(sin, laddr, td);
 577         if (error)
 578                 return (error);
 579
 580         if (IN_MULTICAST(ntohl(laddr->s_addr))) {
 581                 /* Unlike bind, multicast src address is not allowed. */
 582                 return (EINVAL);
 583         }
 584
 585         if (inp->inp_laddr.s_addr == laddr->s_addr) {
 586                 /*
 587                  * src address is same as what we bound to.
 588                  *
 589                  * inp_laddr == INADDR_ANY && srcaddr == INADDR_ANY
 590                  * is allowed, which does not really matter.
 591                  */
 592                 return (0);
 593         } else if (inp->inp_laddr.s_addr != INADDR_ANY &&
 594             !IN_MULTICAST(ntohl(inp->inp_laddr.s_addr))) {
 595                 /* Already bound to a specific address */
 596                 return (EINVAL);
 597         }
 598
 599         /*
 600          * This has to be atomic.  If the porthash is shared across
 601          * multiple protocol threads, e.g. tcp and udp then the token
 602          * must be held.
 603          */
 604         porthash = OBTAIN_LPORTHASH_TOKEN(inp->inp_pcbinfo, inp->inp_lport);
 605
 606         /*
 607          * Restore the sin_port whacked by in_pcbbind_ladddr();
 608          * sin->sin_port is checked by in_pcbbind_laddrport_check().
 609          */
 610         sin->sin_port = inp->inp_lport;
 611
 612         error = in_pcbbind_laddrport_check(so, sin, porthash, wild, cred, td);
 613         if (error)
 614                 laddr->s_addr = INADDR_ANY;
 615
 616         REL_PORTHASH_TOKEN(porthash);
 617         return (error);
 618 }
 619
 620 int
 621 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
 622 {
 623         const struct socket *so = inp->inp_socket;
 624         struct ucred *cred = NULL;
 625         int wild = 0;
 626
 627         if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
 628                 return (EINVAL);        /* already bound */
 629
 630         if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT)))
 631                 wild = 1;    /* neither SO_REUSEADDR nor SO_REUSEPORT is set */
 632         if (td->td_proc)
 633                 cred = td->td_proc->p_ucred;
 634
 635         if (nam != NULL) {
 636                 struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 637                 struct inpcbporthead *porthash;
 638                 u_short lport;
 639                 int error;
 640
 641                 if (nam->sa_len != sizeof *sin)
 642                         return (EINVAL);
 643 #ifdef notdef
 644                 /*
 645                  * We should check the family, but old programs
 646                  * incorrectly fail to initialize it.
 647                  */
 648                 if (sin->sin_family != AF_INET)
 649                         return (EAFNOSUPPORT);
 650 #endif
 651
 652                 /*
 653                  * Save sin_port for later use, since it will
 654                  * be whacked by in_pcbbind_laddr().
 655                  */
 656                 lport = sin->sin_port;
 657
 658                 error = in_pcbbind_laddr(sin, &inp->inp_laddr, td);
 659                 if (error)
 660                         return (error);
 661
 662                 if (lport == 0) {
 663                         /* Auto-select local port */
 664                         return in_pcbsetlport(inp, wild, cred);
 665                 }
 666
 667                 /* GROSS */
 668                 if (ntohs(lport) < IPPORT_RESERVED && cred &&
 669                     (error = caps_priv_check(cred, SYSCAP_NONET_RESPORT)))
 670                  {
 671                         inp->inp_laddr.s_addr = INADDR_ANY;
 672                         return (error);
 673                 }
 674
 675                 /*
 676                  * This has to be atomic.  If the porthash is shared across
 677                  * multiple protocol threads, e.g. tcp and udp then the token
 678                  * must be held.
 679                  */
 680                 porthash = OBTAIN_LPORTHASH_TOKEN(inp->inp_pcbinfo, lport);
 681
 682                 /*
 683                  * Restore the sin_port whacked by in_pcbbind_ladddr();
 684                  * sin->sin_port is checked by in_pcbbind_laddrport_check().
 685                  */
 686                 sin->sin_port = lport;
 687
 688                 error = in_pcbbind_laddrport_check(so, sin, porthash,
 689                                                    wild, cred, td);
 690                 if (error) {
 691                         inp->inp_laddr.s_addr = INADDR_ANY;
 692                         goto done;
 693                 }
 694
 695                 inp->inp_lport = lport;
 696                 in_pcbinsporthash(porthash, inp);
 697                 error = 0;
 698 done:
 699                 REL_PORTHASH_TOKEN(porthash);
 700                 return (error);
 701         } else {
 702                 struct sockaddr_in jsin;
 703
 704                 jsin.sin_family = AF_INET;
 705                 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 706                 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 707                         inp->inp_laddr.s_addr = INADDR_ANY;
 708                         return (EINVAL);
 709                 }
 710                 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 711
 712                 return in_pcbsetlport(inp, wild, cred);
 713         }
 714 }
 715
 716 /*
 717  * Lookup a PCB based on the local and remote address and port.
 718  *
 719  * This function is only used when scanning for a free port.
 720  */
 721 static struct inpcb *
 722 in_pcblookup_localremote(struct inpcbporthead *porthash, struct in_addr laddr,
 723                          u_short lport, struct in_addr faddr, u_short fport,
 724                          struct ucred *cred)
 725 {
 726         struct inpcb *inp;
 727         struct inpcbport *phd;
 728         struct inpcb *match = NULL;
 729         struct prison *pscan;
 730         struct prison *pr;
 731
 732         /*
 733          * If the porthashbase is shared across several cpus, it must
 734          * have been locked.
 735          */
 736         ASSERT_PORTHASH_TOKEN_HELD(porthash);
 737
 738         /*
 739          * Best fit PCB lookup.
 740          *
 741          * First see if this local port is in use by looking on the
 742          * port hash list.
 743          */
 744         LIST_FOREACH(phd, porthash, phd_hash) {
 745                 if (phd->phd_port == lport)
 746                         break;
 747         }
 748         if (phd != NULL) {
 749                 pr = cred ? cred->cr_prison : NULL;
 750
 751                 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
 752 #ifdef INET6
 753                         if (!INP_ISIPV4(inp))
 754                                 continue;
 755 #endif
 756                         if (inp->inp_laddr.s_addr == INADDR_ANY) {
 757                                 if (inp->inp_socket && inp->inp_socket->so_cred)
 758                                         pscan = inp->inp_socket->so_cred->cr_prison;
 759                                 else
 760                                         pscan = NULL;
 761                                 if (pr != pscan)
 762                                         continue;
 763                         } else {
 764                                 if (inp->inp_laddr.s_addr != laddr.s_addr)
 765                                         continue;
 766                         }
 767
 768                         if (inp->inp_faddr.s_addr != INADDR_ANY &&
 769                             inp->inp_faddr.s_addr != faddr.s_addr)
 770                                 continue;
 771
 772                         if (inp->inp_fport != 0 && inp->inp_fport != fport)
 773                                 continue;
 774
 775                         match = inp;
 776                         break;
 777                 }
 778         }
 779         return (match);
 780 }
 781
 782 static boolean_t
 783 in_pcbporthash_update4(struct inpcbportinfo *portinfo, struct inpcb *inp,
 784                        u_short lport, const struct sockaddr_in *sin,
 785                        struct ucred *cred)
 786 {
 787         struct inpcbporthead *porthash;
 788
 789         /*
 790          * This has to be atomic.  If the porthash is shared across multiple
 791          * protocol threads, e.g. tcp and udp, then the token must be held.
 792          */
 793         porthash = in_pcbporthash_head(portinfo, lport);
 794         GET_PORTHASH_TOKEN(porthash);
 795
 796         if (in_pcblookup_localremote(porthash, inp->inp_laddr, lport,
 797                                      sin->sin_addr, sin->sin_port, cred)) {
 798                 REL_PORTHASH_TOKEN(porthash);
 799                 return FALSE;
 800         }
 801         inp->inp_lport = lport;
 802         in_pcbinsporthash(porthash, inp);
 803
 804         REL_PORTHASH_TOKEN(porthash);
 805         return TRUE;
 806 }
 807
 808 int
 809 in_pcbbind_remote(struct inpcb *inp, const struct sockaddr *remote,
 810     struct thread *td)
 811 {
 812         struct proc *p = td->td_proc;
 813         const struct sockaddr_in *sin = (const struct sockaddr_in *)remote;
 814         struct sockaddr_in jsin;
 815         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 816         struct ucred *cred = NULL;
 817         u_short first, last, lport;
 818         int count, hash_count;
 819         int error, selfconn = 0;
 820         int cpuid = mycpuid;
 821         uint32_t hash_base = 0, hash;
 822
 823         ASSERT_NETISR_NCPUS(cpuid);
 824
 825         if (TAILQ_EMPTY(&in_ifaddrheads[cpuid])) /* XXX broken! */
 826                 return (EADDRNOTAVAIL);
 827
 828         KKASSERT(inp->inp_laddr.s_addr != INADDR_ANY);
 829         if (inp->inp_lport != 0)
 830                 return (EINVAL);        /* already bound */
 831
 832         KKASSERT(p);
 833         cred = p->p_ucred;
 834
 835         jsin.sin_family = AF_INET;
 836         jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 837         if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 838                 inp->inp_laddr.s_addr = INADDR_ANY;
 839                 return (EINVAL);
 840         }
 841         inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 842
 843         hash_count = ip_porthash_trycount;
 844         if (hash_count > 0) {
 845                 hash_base = toeplitz_piecemeal_addr(sin->sin_addr.s_addr) ^
 846                     toeplitz_piecemeal_addr(inp->inp_laddr.s_addr) ^
 847                     toeplitz_piecemeal_port(sin->sin_port);
 848         } else {
 849                 hash_count = 0;
 850         }
 851
 852         inp->inp_flags |= INP_ANONPORT;
 853
 854         if (inp->inp_flags & INP_HIGHPORT) {
 855                 first = ipport_hifirstauto;     /* sysctl */
 856                 last  = ipport_hilastauto;
 857         } else if (inp->inp_flags & INP_LOWPORT) {
 858                 if (cred &&
 859                     (error = caps_priv_check(cred, SYSCAP_NONET_RESPORT)))
 860                  {
 861                         inp->inp_laddr.s_addr = INADDR_ANY;
 862                         return (error);
 863                 }
 864                 first = ipport_lowfirstauto;    /* 1023 */
 865                 last = ipport_lowlastauto;      /* 600 */
 866         } else {
 867                 first = ipport_firstauto;       /* sysctl */
 868                 last  = ipport_lastauto;
 869         }
 870         if (first > last) {
 871                 lport = last;
 872                 last = first;
 873                 first = lport;
 874         }
 875         KKASSERT(last >= first);
 876
 877         count = last - first;
 878         lport = (karc4random() % count) + first;
 879         count += hash_count;
 880
 881         /*
 882          * Simple check to ensure all ports are not used up causing
 883          * a deadlock here.
 884          */
 885         for (;;) {
 886                 u_short lport_no;
 887
 888                 if (count-- < 0) {      /* completely used? */
 889                         error = EADDRNOTAVAIL;
 890                         break;
 891                 }
 892
 893                 if (__predict_false(lport < first || lport > last))
 894                         lport = first;
 895                 lport_no = htons(lport);
 896
 897                 /* This could happen on loopback interface */
 898                 if (__predict_false(sin->sin_port == lport_no &&
 899                     sin->sin_addr.s_addr == inp->inp_laddr.s_addr)) {
 900                         if (!selfconn) {
 901                                 ++count; /* don't count this try */
 902                                 selfconn = 1;
 903                         }
 904                         goto next;
 905                 }
 906
 907                 if (hash_count) {
 908                         --hash_count;
 909                         hash = hash_base ^
 910                             toeplitz_piecemeal_port(lport_no);
 911                         if (netisr_hashcpu(hash) != cpuid && hash_count)
 912                                 goto next;
 913                 }
 914
 915                 if (in_pcbporthash_update4(
 916                             &pcbinfo->portinfo[lport % pcbinfo->portinfo_cnt],
 917                             inp, lport_no, sin, cred)) {
 918                         error = 0;
 919                         break;
 920                 }
 921 next:
 922                 ++lport;
 923         }
 924
 925         if (error)
 926                 inp->inp_laddr.s_addr = INADDR_ANY;
 927         return (error);
 928 }
 929
 930 /*
 931  * Figure out the local interface address to pair against the requested
 932  * target address, as well as validate the target address.
 933  */
 934 int
 935 in_pcbladdr_find(struct inpcb *inp, struct sockaddr *nam,
 936                  struct sockaddr_in **plocal_sin, struct thread *td, int find)
 937 {
 938         struct in_ifaddr_container *iac;
 939         struct in_ifaddr *ia;
 940         struct ucred *cred = NULL;
 941         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 942         struct sockaddr *jsin;
 943         struct prison *pr;
 944         struct route *ro;
 945         int alloc_route = 0;
 946
 947         if (nam->sa_len != sizeof *sin)
 948                 return (EINVAL);
 949         if (sin->sin_family != AF_INET)
 950                 return (EAFNOSUPPORT);
 951         if (sin->sin_port == 0)
 952                 return (EADDRNOTAVAIL);
 953
 954         /*
 955          * Are we in a jail?
 956          */
 957         pr = NULL;
 958         if (td && td->td_proc && td->td_proc->p_ucred)
 959                 cred = td->td_proc->p_ucred;
 960         if (cred)
 961                 pr = cred->cr_prison;
 962
 963         /*
 964          * If the destination address is INADDR_ANY then use the primary
 965          * local address.
 966          *
 967          * If the supplied address is INADDR_BROADCAST, and the primary
 968          * interface supports broadcast, choose the broadcast address for
 969          * that interface.
 970          *
 971          * If jailed, locate an interface address acceptable to the jail.
 972          */
 973         if (sin->sin_addr.s_addr == INADDR_ANY) {
 974                 TAILQ_FOREACH(iac, &in_ifaddrheads[mycpuid], ia_link) {
 975                         ia = iac->ia;
 976                         if (pr == NULL ||
 977                             jailed_ip(pr, sintosa(&ia->ia_addr))) {
 978                                 sin->sin_addr = IA_SIN(ia)->sin_addr;
 979                                 break;
 980                         }
 981                 }
 982         } else if (sin->sin_addr.s_addr == (u_long)INADDR_BROADCAST) {
 983                 TAILQ_FOREACH(iac, &in_ifaddrheads[mycpuid], ia_link) {
 984                         ia = iac->ia;
 985                         if ((pr == NULL ||
 986                              jailed_ip(pr, sintosa(&ia->ia_addr))) &&
 987                             (iac->ia->ia_ifp->if_flags & IFF_BROADCAST)) {
 988                                 sin->sin_addr =
 989                                     satosin(&ia->ia_broadaddr)->sin_addr;
 990                                 break;
 991                         }
 992                 }
 993         }
 994
 995         /*
 996          * If asked to do a search, use the cached route or do a route table
 997          * lookup to try to find an acceptable local interface IP.
 998          */
 999         if (find == 0)
1000                 return 0;
1001
1002         ia = NULL;
1003
1004         /*
1005          * If we have a cached route, check to see if it is acceptable.
1006          * If not, free it.
1007          */
1008         ro = &inp->inp_route;
1009         if (ro->ro_rt &&
1010             (!(ro->ro_rt->rt_flags & RTF_UP) ||
1011              ro->ro_dst.sa_family != AF_INET ||
1012              satosin(&ro->ro_dst)->sin_addr.s_addr !=
1013                               sin->sin_addr.s_addr ||
1014              inp->inp_socket->so_options & SO_DONTROUTE)) {
1015                 RTFREE(ro->ro_rt);
1016                 ro->ro_rt = NULL;
1017         }
1018
1019         /*
1020          * If we do not have a route, construct one and do a lookup,
1021          * unless we are forbidden to do so.
1022          *
1023          * Note that we should check the address family of the cached
1024          * destination, in case of sharing the cache with IPv6.
1025          */
1026         if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/
1027             (ro->ro_rt == NULL || ro->ro_rt->rt_ifp == NULL)) {
1028                 bzero(&ro->ro_dst, sizeof(struct sockaddr_in));
1029                 ro->ro_dst.sa_family = AF_INET;
1030                 ro->ro_dst.sa_len = sizeof(struct sockaddr_in);
1031                 ((struct sockaddr_in *)&ro->ro_dst)->sin_addr = sin->sin_addr;
1032                 rtalloc(ro);
1033                 alloc_route = 1;
1034         }
1035
1036         /*
1037          * If we found a route, use the address corresponding to the
1038          * outgoing interface.
1039          *
1040          * If jailed, try to find a compatible address on the outgoing
1041          * interface.
1042          */
1043         if (ro->ro_rt) {
1044                 ia = ifatoia(ro->ro_rt->rt_ifa);
1045                 if (pr == NULL)
1046                         goto skip;
1047                 if (jailed_ip(pr, sintosa(&ia->ia_addr)))
1048                         goto skip;
1049                 TAILQ_FOREACH(iac, &in_ifaddrheads[mycpuid], ia_link) {
1050                         if (iac->ia->ia_ifp != ia->ia_ifp)
1051                                 continue;
1052                         ia = iac->ia;
1053                         if (jailed_ip(pr, sintosa(&ia->ia_addr)))
1054                                 goto skip;
1055                 }
1056                 ia = NULL;
1057         }
1058 skip:
1059
1060         /*
1061          * If the route didn't work or there was no route,
1062          * fall-back to the first address in in_ifaddrheads[].
1063          *
1064          * If jailed and this address is not available for
1065          * the jail, leave ia set to NULL.
1066          */
1067         if (ia == NULL) {
1068                 u_short fport = sin->sin_port;
1069
1070                 sin->sin_port = 0;
1071                 ia = ifatoia(ifa_ifwithdstaddr(sintosa(sin)));
1072                 if (ia && pr && !jailed_ip(pr, sintosa(&ia->ia_addr)))
1073                         ia = NULL;
1074
1075                 if (ia == NULL)
1076                         ia = ifatoia(ifa_ifwithnet(sintosa(sin)));
1077                 if (ia && pr && !jailed_ip(pr, sintosa(&ia->ia_addr)))
1078                         ia = NULL;
1079
1080                 sin->sin_port = fport;
1081                 if (ia == NULL && !TAILQ_EMPTY(&in_ifaddrheads[mycpuid]))
1082                         ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia;
1083
1084                 if (ia && pr && !jailed_ip(pr, sintosa(&ia->ia_addr)))
1085                         ia = NULL;
1086
1087                 if (pr == NULL && ia == NULL)
1088                         goto fail;
1089         }
1090
1091         /*
1092          * If the destination address is multicast and an outgoing
1093          * interface has been set as a multicast option, use the
1094          * address of that interface as our source address.
1095          */
1096         if (pr == NULL && IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
1097             inp->inp_moptions != NULL) {
1098                 struct ip_moptions *imo;
1099                 struct ifnet *ifp;
1100
1101                 imo = inp->inp_moptions;
1102                 if ((ifp = imo->imo_multicast_ifp) != NULL) {
1103                         struct in_ifaddr_container *iac;
1104
1105                         ia = NULL;
1106                         TAILQ_FOREACH(iac, &in_ifaddrheads[mycpuid], ia_link) {
1107                                 if (iac->ia->ia_ifp == ifp) {
1108                                         ia = iac->ia;
1109                                         break;
1110                                 }
1111                         }
1112                         if (ia == NULL)
1113                                 goto fail;
1114                 }
1115         }
1116
1117         /*
1118          * If we still don't have a local address, and are jailed,
1119          * use the jail's first non-localhost IP.  If there isn't
1120          * one, use the jail's first localhost IP.
1121          *
1122          * Don't do pcblookup call here; return interface in plocal_sin
1123          * and exit to caller, that will do the lookup.
1124          */
1125         if (ia == NULL && pr) {
1126                 jsin = prison_get_nonlocal(cred->cr_prison, AF_INET, NULL);
1127                 if (jsin == NULL)
1128                         jsin = prison_get_local(cred->cr_prison, AF_INET, NULL);
1129                 if (jsin)
1130                         *plocal_sin = satosin(jsin);
1131                 else
1132                         goto fail;
1133         } else if (ia) {
1134                 *plocal_sin = &ia->ia_addr;
1135         } else {
1136                 goto fail;
1137         }
1138         return (0);
1139 fail:
1140         if (alloc_route)
1141                 in_pcbresetroute(inp);
1142         return (EADDRNOTAVAIL);
1143 }
1144
1145 int
1146 in_pcbladdr(struct inpcb *inp, struct sockaddr *nam,
1147             struct sockaddr_in **plocal_sin, struct thread *td)
1148 {
1149         return in_pcbladdr_find(inp, nam, plocal_sin, td,
1150                                 (inp->inp_laddr.s_addr == INADDR_ANY));
1151 }
1152
1153 /*
1154  * Outer subroutine:
1155  * Connect from a socket to a specified address.
1156  * Both address and port must be specified in argument sin.
1157  * If don't have a local address for this socket yet,
1158  * then pick one.
1159  */
1160 int
1161 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
1162 {
1163         struct sockaddr_in *if_sin;
1164         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
1165         int error;
1166
1167         if_sin = NULL;  /* avoid gcc warnings */
1168
1169         /* Call inner routine to assign local interface address. */
1170         if ((error = in_pcbladdr(inp, nam, &if_sin, td)) != 0)
1171                 return (error);
1172
1173         if (in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port,
1174                               inp->inp_laddr.s_addr ?
1175                                 inp->inp_laddr : if_sin->sin_addr,
1176                               inp->inp_lport, FALSE, NULL) != NULL) {
1177                 return (EADDRINUSE);
1178         }
1179         if (inp->inp_laddr.s_addr == INADDR_ANY) {
1180                 if (inp->inp_lport == 0) {
1181                         error = in_pcbbind(inp, NULL, td);
1182                         if (error)
1183                                 return (error);
1184                 }
1185                 inp->inp_laddr = if_sin->sin_addr;
1186         }
1187         inp->inp_faddr = sin->sin_addr;
1188         inp->inp_fport = sin->sin_port;
1189         in_pcbinsconnhash(inp);
1190         return (0);
1191 }
1192
1193 void
1194 in_pcbdisconnect(struct inpcb *inp)
1195 {
1196
1197         in_pcbremconnhash(inp);
1198         inp->inp_faddr.s_addr = INADDR_ANY;
1199         inp->inp_fport = 0;
1200 }
1201
1202 void
1203 in_pcbdetach(struct inpcb *inp)
1204 {
1205         struct socket *so = inp->inp_socket;
1206         struct inpcbinfo *ipi = inp->inp_pcbinfo;
1207
1208         inp->inp_gencnt = ++ipi->ipi_gencnt;
1209         KKASSERT((so->so_state & SS_ASSERTINPROG) == 0);
1210         in_pcbremlists(inp);
1211         so->so_pcb = NULL;
1212         sofree(so);                     /* remove pcb ref */
1213         if (inp->inp_options)
1214                 m_free(inp->inp_options);
1215         if (inp->inp_route.ro_rt)
1216                 rtfree(inp->inp_route.ro_rt);
1217         ip_freemoptions(inp->inp_moptions);
1218         kfree(inp, M_PCB);
1219 }
1220
1221 /*
1222  * The socket may have an invalid PCB, i.e. NULL.  For example, a TCP
1223  * socket received RST.
1224  */
1225 static int
1226 in_setsockaddr(struct socket *so, struct sockaddr **nam)
1227 {
1228         struct inpcb *inp;
1229         struct sockaddr_in *sin;
1230
1231         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
1232         inp = so->so_pcb;
1233         if (!inp)
1234                 return (ECONNRESET);
1235
1236         sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1237         sin->sin_family = AF_INET;
1238         sin->sin_len = sizeof *sin;
1239         sin->sin_port = inp->inp_lport;
1240         sin->sin_addr = inp->inp_laddr;
1241
1242         *nam = (struct sockaddr *)sin;
1243         return (0);
1244 }
1245
1246 void
1247 in_setsockaddr_dispatch(netmsg_t msg)
1248 {
1249         int error;
1250
1251         error = in_setsockaddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1252         lwkt_replymsg(&msg->lmsg, error);
1253 }
1254
1255 /*
1256  * The socket may have an invalid PCB, i.e. NULL.  For example, a TCP
1257  * socket received RST.
1258  */
1259 int
1260 in_setpeeraddr(struct socket *so, struct sockaddr **nam)
1261 {
1262         struct inpcb *inp;
1263         struct sockaddr_in *sin;
1264
1265         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
1266         inp = so->so_pcb;
1267         if (!inp)
1268                 return (ECONNRESET);
1269
1270         sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1271         sin->sin_family = AF_INET;
1272         sin->sin_len = sizeof *sin;
1273         sin->sin_port = inp->inp_fport;
1274         sin->sin_addr = inp->inp_faddr;
1275
1276         *nam = (struct sockaddr *)sin;
1277         return (0);
1278 }
1279
1280 void
1281 in_setpeeraddr_dispatch(netmsg_t msg)
1282 {
1283         int error;
1284
1285         error = in_setpeeraddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1286         lwkt_replymsg(&msg->lmsg, error);
1287 }
1288
1289 void
1290 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int err,
1291     inp_notify_t notify)
1292 {
1293         struct inpcb *inp, *marker;
1294
1295         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1296             ("not in the correct netisr"));
1297         marker = in_pcbmarker();
1298
1299         /*
1300          * NOTE:
1301          * - If INP_PLACEMARKER is set we must ignore the rest of the
1302          *   structure and skip it.
1303          * - It is safe to nuke inpcbs here, since we are in their own
1304          *   netisr.
1305          */
1306         GET_PCBINFO_TOKEN(pcbinfo);
1307
1308         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1309         while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1310                 LIST_REMOVE(marker, inp_list);
1311                 LIST_INSERT_AFTER(inp, marker, inp_list);
1312
1313                 if (inp->inp_flags & INP_PLACEMARKER)
1314                         continue;
1315 #ifdef INET6
1316                 if (!INP_ISIPV4(inp))
1317                         continue;
1318 #endif
1319                 if (inp->inp_faddr.s_addr != faddr.s_addr ||
1320                     inp->inp_socket == NULL)
1321                         continue;
1322                 (*notify)(inp, err);            /* can remove inp from list! */
1323         }
1324         LIST_REMOVE(marker, inp_list);
1325
1326         REL_PCBINFO_TOKEN(pcbinfo);
1327 }
1328
1329 void
1330 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1331 {
1332         struct inpcb *inp, *marker;
1333
1334         /*
1335          * We only need to make sure that we are in netisr0, where all
1336          * multicast operation happen.  We could check inpcbinfo which
1337          * does not belong to netisr0 by holding the inpcbinfo's token.
1338          * In this case, the pcbinfo must be able to be shared, i.e.
1339          * pcbinfo->infotoken is not NULL.
1340          */
1341         ASSERT_NETISR0;
1342         KASSERT(pcbinfo->cpu == 0 || pcbinfo->infotoken != NULL,
1343             ("pcbinfo could not be shared"));
1344
1345         /*
1346          * Get a marker for the current netisr (netisr0).
1347          *
1348          * It is possible that the multicast address deletion blocks,
1349          * which could cause temporary token releasing.  So we use
1350          * inpcb marker here to get a coherent view of the inpcb list.
1351          *
1352          * While, on the other hand, moptions are only added and deleted
1353          * in netisr0, so we would not see staled moption or miss moption
1354          * even if the token was released due to the blocking multicast
1355          * address deletion.
1356          */
1357         marker = in_pcbmarker();
1358
1359         GET_PCBINFO_TOKEN(pcbinfo);
1360
1361         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1362         while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1363                 struct ip_moptions *imo;
1364
1365                 LIST_REMOVE(marker, inp_list);
1366                 LIST_INSERT_AFTER(inp, marker, inp_list);
1367
1368                 if (inp->inp_flags & INP_PLACEMARKER)
1369                         continue;
1370                 imo = inp->inp_moptions;
1371                 if (INP_ISIPV4(inp) && imo != NULL) {
1372                         int i, gap;
1373
1374                         /*
1375                          * Unselect the outgoing interface if it is being
1376                          * detached.
1377                          */
1378                         if (imo->imo_multicast_ifp == ifp)
1379                                 imo->imo_multicast_ifp = NULL;
1380
1381                         /*
1382                          * Drop multicast group membership if we joined
1383                          * through the interface being detached.
1384                          */
1385                         for (i = 0, gap = 0; i < imo->imo_num_memberships;
1386                             i++) {
1387                                 if (imo->imo_membership[i]->inm_ifp == ifp) {
1388                                         /*
1389                                          * NOTE:
1390                                          * This could block and the pcbinfo
1391                                          * token could be passively released.
1392                                          */
1393                                         in_delmulti(imo->imo_membership[i]);
1394                                         gap++;
1395                                 } else if (gap != 0)
1396                                         imo->imo_membership[i - gap] =
1397                                             imo->imo_membership[i];
1398                         }
1399                         imo->imo_num_memberships -= gap;
1400                 }
1401         }
1402         LIST_REMOVE(marker, inp_list);
1403
1404         REL_PCBINFO_TOKEN(pcbinfo);
1405 }
1406
1407 /*
1408  * Check for alternatives when higher level complains
1409  * about service problems.  For now, invalidate cached
1410  * routing information.  If the route was created dynamically
1411  * (by a redirect), time to try a default gateway again.
1412  */
1413 void
1414 in_losing(struct inpcb *inp)
1415 {
1416         struct rtentry *rt;
1417         struct rt_addrinfo rtinfo;
1418
1419         if ((rt = inp->inp_route.ro_rt)) {
1420                 bzero(&rtinfo, sizeof(struct rt_addrinfo));
1421                 rtinfo.rti_info[RTAX_DST] = rt_key(rt);
1422                 rtinfo.rti_info[RTAX_GATEWAY] = rt->rt_gateway;
1423                 rtinfo.rti_info[RTAX_NETMASK] = rt_mask(rt);
1424                 rtinfo.rti_flags = rt->rt_flags;
1425                 rt_missmsg(RTM_LOSING, &rtinfo, rt->rt_flags, 0);
1426                 if (rt->rt_flags & RTF_DYNAMIC) {
1427                         rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway,
1428                             rt_mask(rt), rt->rt_flags, NULL);
1429                 }
1430                 inp->inp_route.ro_rt = NULL;
1431                 rtfree(rt);
1432                 /*
1433                  * A new route can be allocated
1434                  * the next time output is attempted.
1435                  */
1436         }
1437 }
1438
1439 /*
1440  * After a routing change, flush old routing
1441  * and allocate a (hopefully) better one.
1442  */
1443 void
1444 in_rtchange(struct inpcb *inp, int err)
1445 {
1446         if (inp->inp_route.ro_rt) {
1447                 rtfree(inp->inp_route.ro_rt);
1448                 inp->inp_route.ro_rt = NULL;
1449                 /*
1450                  * A new route can be allocated the next time
1451                  * output is attempted.
1452                  */
1453         }
1454 }
1455
1456 /*
1457  * Lookup a PCB based on the local address and port.
1458  *
1459  * This function is only used when scanning for a free port.
1460  */
1461 static struct inpcb *
1462 in_pcblookup_local(struct inpcbporthead *porthash, struct in_addr laddr,
1463                    u_int lport_arg, int wild_okay, struct ucred *cred)
1464 {
1465         struct prison *pscan;
1466         struct prison *pr;
1467         struct inpcb *inp;
1468         int matchwild = 3, wildcard;
1469         u_short lport = lport_arg;
1470         struct inpcbport *phd;
1471         struct inpcb *match = NULL;
1472
1473         /*
1474          * If the porthashbase is shared across several cpus, it must
1475          * have been locked.
1476          */
1477         ASSERT_PORTHASH_TOKEN_HELD(porthash);
1478
1479         /*
1480          * Best fit PCB lookup.
1481          *
1482          * First see if this local port is in use by looking on the
1483          * port hash list.
1484          */
1485         LIST_FOREACH(phd, porthash, phd_hash) {
1486                 if (phd->phd_port == lport)
1487                         break;
1488         }
1489         if (phd != NULL) {
1490                 pr = cred ? cred->cr_prison : NULL;
1491
1492                 /*
1493                  * Port is in use by one or more PCBs. Look for best
1494                  * fit.
1495                  *
1496                  * If in a prison we may wish to allow the jail to override
1497                  * a wildcard listen on the host.  Since the jail forces its
1498                  * own wildcard listens to a specific set of jail IPs, this
1499                  * override allows most services on the host to remain as
1500                  * they were and still be 'jail friendly'.
1501                  */
1502                 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
1503                         wildcard = 0;
1504 #ifdef INET6
1505                         if (!INP_ISIPV4(inp))
1506                                 continue;
1507 #endif
1508                         if (inp->inp_faddr.s_addr != INADDR_ANY)
1509                                 wildcard++;
1510
1511                         /*
1512                          * Prison are independent of each other in terms
1513                          * of allowing bindings.  This can result in multiple
1514                          * overloaded bindings which in_pcblookup_pkthash()
1515                          * will have to sort out.
1516                          *
1517                          * Allow wildcarded entries to co-exist with specific
1518                          * entries.  Specific entries override wildcarded
1519                          * entries.
1520                          */
1521                         if (inp->inp_socket && inp->inp_socket->so_cred)
1522                                 pscan = inp->inp_socket->so_cred->cr_prison;
1523                         else
1524                                 pscan = NULL;
1525                         if (pr != pscan)
1526                                 continue;
1527                         if (inp->inp_laddr.s_addr == INADDR_ANY) {
1528                                 if (laddr.s_addr != INADDR_ANY)
1529                                         wildcard++;
1530                         } else {
1531                                 if (laddr.s_addr == INADDR_ANY)
1532                                         wildcard++;
1533                                 else if (inp->inp_laddr.s_addr != laddr.s_addr)
1534                                         continue;
1535                         }
1536                         if (wildcard && !wild_okay)
1537                                 continue;
1538                         if (wildcard < matchwild) {
1539                                 match = inp;
1540                                 matchwild = wildcard;
1541                                 if (matchwild == 0)
1542                                         break;
1543                         }
1544                 }
1545         }
1546         return (match);
1547 }
1548
1549 struct inpcb *
1550 in_pcblocalgroup_last(const struct inpcbinfo *pcbinfo,
1551     const struct inpcb *inp)
1552 {
1553         const struct inp_localgrphead *hdr;
1554         const struct inp_localgroup *grp;
1555         int i;
1556
1557         if (pcbinfo->localgrphashbase == NULL)
1558                 return NULL;
1559
1560         GET_PCBINFO_TOKEN(pcbinfo);
1561
1562         hdr = &pcbinfo->localgrphashbase[
1563             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
1564
1565         LIST_FOREACH(grp, hdr, il_list) {
1566                 if (grp->il_af == inp->inp_af &&
1567                     grp->il_lport == inp->inp_lport &&
1568                     memcmp(&grp->il_dependladdr,
1569                         &inp->inp_inc.inc_ie.ie_dependladdr,
1570                         sizeof(grp->il_dependladdr)) == 0) {
1571                         break;
1572                 }
1573         }
1574         if (grp == NULL || grp->il_inpcnt == 1) {
1575                 REL_PCBINFO_TOKEN(pcbinfo);
1576                 return NULL;
1577         }
1578
1579         KASSERT(grp->il_inpcnt >= 2,
1580             ("invalid localgroup inp count %d", grp->il_inpcnt));
1581         for (i = 0; i < grp->il_inpcnt; ++i) {
1582                 if (grp->il_inp[i] == inp) {
1583                         int last = grp->il_inpcnt - 1;
1584
1585                         if (i == last)
1586                                 last = grp->il_inpcnt - 2;
1587                         REL_PCBINFO_TOKEN(pcbinfo);
1588                         return grp->il_inp[last];
1589                 }
1590         }
1591         REL_PCBINFO_TOKEN(pcbinfo);
1592         return NULL;
1593 }
1594
1595 static struct inpcb *
1596 inp_localgroup_lookup(const struct inpcbinfo *pcbinfo,
1597     struct in_addr laddr, uint16_t lport, uint32_t pkt_hash)
1598 {
1599         struct inpcb *local_wild;
1600         struct inpcb *jinp;
1601         struct inpcb *jinp_wild;
1602         struct inpcb *inp;
1603         const struct inp_localgrphead *hdr;
1604         const struct inp_localgroup *grp;
1605         struct sockaddr_in jsin;
1606         struct prison *pr;
1607         struct ucred *cred;
1608         int idx;
1609         int net_listen_ov_local;
1610         int net_listen_ov_wild;
1611
1612         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1613
1614         hdr = &pcbinfo->localgrphashbase[
1615             INP_PCBLOCALGRPHASH(lport, pcbinfo->localgrphashmask)];
1616
1617         /*
1618          * Order of socket selection:
1619          * 1. non-wild.
1620          * 2. wild.
1621          *
1622          * NOTE: Local group does not contain jailed sockets
1623          */
1624         jsin.sin_family = AF_INET;
1625         jsin.sin_addr.s_addr = laddr.s_addr;
1626
1627         jinp = NULL;
1628         jinp_wild = NULL;
1629         local_wild = NULL;
1630         net_listen_ov_local = 0;
1631         net_listen_ov_wild = 0;
1632
1633         LIST_FOREACH(grp, hdr, il_list) {
1634 #ifdef INET6
1635                 if (grp->il_af != AF_INET)
1636                         continue;
1637 #endif
1638                 if (grp->il_lport != lport)
1639                         continue;
1640
1641                 /*
1642                  * look for a match
1643                  */
1644                 idx = netisr_hashlsb(pkt_hash) % grp->il_inpcnt;
1645                 inp = grp->il_inp[idx];
1646
1647                 /*
1648                  * Modulo-N is used here, which greatly reduces
1649                  * completion queue token contention, thus more
1650                  * cpu time is saved.
1651                  */
1652                 if (grp->il_jailed) {
1653                         if (inp->inp_socket == NULL)
1654                                 continue;
1655                         cred = inp->inp_socket->so_cred;
1656                         if (cred == NULL)
1657                                 continue;
1658                         pr = cred->cr_prison;
1659                         if (pr == NULL)
1660                                 continue;
1661                         if (!jailed_ip(pr, (struct sockaddr *)&jsin))
1662                                 continue;
1663                         if (grp->il_laddr.s_addr == laddr.s_addr) {
1664                                 jinp = inp;
1665                                 if (PRISON_CAP_ISSET(pr->pr_caps, PRISON_CAP_NET_LISTEN_OVERRIDE))
1666                                         net_listen_ov_local = 1;
1667
1668                         } else if (grp->il_laddr.s_addr == INADDR_ANY &&
1669                                    jinp_wild == NULL) {
1670                                 jinp_wild = inp;
1671                                 if (PRISON_CAP_ISSET(pr->pr_caps, PRISON_CAP_NET_LISTEN_OVERRIDE))
1672                                         net_listen_ov_wild = 1;
1673                         }
1674                 } else {
1675                         if (grp->il_laddr.s_addr == laddr.s_addr) {
1676                                 return inp;
1677                         } else if (grp->il_laddr.s_addr == INADDR_ANY) {
1678                                 local_wild = inp;
1679                         }
1680                 }
1681         }
1682
1683         if (net_listen_ov_local)
1684                 return jinp;
1685         if (net_listen_ov_wild)
1686                 return jinp_wild;
1687         if (local_wild)
1688                 return (local_wild);
1689         if (jinp)
1690                 return (jinp);
1691         return (jinp_wild);
1692 }
1693
1694 /*
1695  * Lookup PCB in hash list.
1696  *
1697  * This is used to match incoming packets to a pcb
1698  */
1699 struct inpcb *
1700 in_pcblookup_pkthash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1701     u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1702     boolean_t wildcard, struct ifnet *ifp, const struct mbuf *m)
1703 {
1704         struct inpcbhead *head;
1705         struct inpcb *inp, *jinp=NULL;
1706         u_short fport = fport_arg, lport = lport_arg;
1707
1708         /*
1709          * First look for an exact match.
1710          */
1711         head = &pcbinfo->hashbase[INP_PCBCONNHASH(faddr.s_addr, fport,
1712                                                   laddr.s_addr, lport,
1713                                                   pcbinfo->hashmask)];
1714         LIST_FOREACH(inp, head, inp_hash) {
1715 #ifdef INET6
1716                 if (!INP_ISIPV4(inp))
1717                         continue;
1718 #endif
1719                 if (in_hosteq(inp->inp_faddr, faddr) &&
1720                     in_hosteq(inp->inp_laddr, laddr) &&
1721                     inp->inp_fport == fport && inp->inp_lport == lport) {
1722                         /*
1723                          * Found specific address, host overrides jailed
1724                          * inpcb.
1725                          */
1726                         if (inp->inp_socket == NULL ||
1727                             inp->inp_socket->so_cred->cr_prison == NULL) {
1728                                 return (inp);
1729                         }
1730                         if (jinp == NULL)
1731                                 jinp = inp;
1732                 }
1733         }
1734         if (jinp != NULL)
1735                 return (jinp);
1736
1737         /*
1738          * We generally get here for connections to wildcarded listeners.
1739          * Any wildcarded listeners in jails must be restricted to the
1740          * jailed IPs only.
1741          */
1742         if (wildcard) {
1743                 struct inpcb *local_wild = NULL;
1744                 struct inpcb *jinp_wild = NULL;
1745                 struct inpcontainer *ic;
1746                 struct inpcontainerhead *chead;
1747                 struct sockaddr_in jsin;
1748                 struct ucred *cred;
1749                 struct prison *pr;
1750                 int net_listen_ov_local = 0;
1751                 int net_listen_ov_wild = 0;
1752
1753                 GET_PCBINFO_TOKEN(pcbinfo);
1754
1755                 /*
1756                  * Check local group first.  When present, the localgroup
1757                  * hash utilizes the same non-jailed-vs/jailed priortization
1758                  * that the normal wildcardhash does.
1759                  */
1760                 if (pcbinfo->localgrphashbase != NULL &&
1761                     m != NULL && (m->m_flags & M_HASH)) {
1762                         inp = inp_localgroup_lookup(pcbinfo, laddr, lport,
1763                                                     m->m_pkthdr.hash);
1764                         if (inp != NULL) {
1765                                 REL_PCBINFO_TOKEN(pcbinfo);
1766                                 return inp;
1767                         }
1768                 }
1769
1770                 /*
1771                  * Order of socket selection:
1772                  *
1773                  * 1. non-jailed, non-wild.
1774                  * 2. non-jailed, wild.         (allow_listen_override on)
1775                  * 3. jailed, non-wild.
1776                  * 4. jailed, wild.
1777                  * 5. non-jailed, wild.         (allow_listen_override off)
1778                  *
1779                  * NOTE: jailed wildcards are still restricted to the jail
1780                  *       IPs.
1781                  *
1782                  * NOTE: (1) and (3) already handled above.
1783                  */
1784                 jsin.sin_family = AF_INET;
1785                 chead = &pcbinfo->wildcardhashbase[
1786                     INP_PCBWILDCARDHASH(lport, pcbinfo->wildcardhashmask)];
1787
1788                 LIST_FOREACH(ic, chead, ic_list) {
1789                         inp = ic->ic_inp;
1790                         if (inp->inp_flags & INP_PLACEMARKER)
1791                                 continue;
1792
1793                         /*
1794                          * Basic validation
1795                          */
1796 #ifdef INET6
1797                         if (!INP_ISIPV4(inp))
1798                                 continue;
1799 #endif
1800                         if (inp->inp_lport != lport)
1801                                 continue;
1802
1803                         /*
1804                          * Calculate prison, setup jsin for jailed_ip()
1805                          * check.
1806                          */
1807                         jsin.sin_addr.s_addr = laddr.s_addr;
1808                         pr = NULL;
1809                         cred = NULL;
1810                         if (inp->inp_socket) {
1811                                 cred = inp->inp_socket->so_cred;
1812                                 if (cred)
1813                                         pr = cred->cr_prison;
1814                         }
1815
1816                         /*
1817                          * Assign jinp, jinp_wild, and local_wild as
1818                          * appropriate, track whether the jail supports
1819                          * listen overrides.
1820                          */
1821                         if (pr) {
1822                                 if (!jailed_ip(pr, (struct sockaddr *)&jsin))
1823                                         continue;
1824                                 if (inp->inp_laddr.s_addr == laddr.s_addr &&
1825                                     jinp == NULL) {
1826                                         jinp = inp;
1827                                         if (PRISON_CAP_ISSET(pr->pr_caps, PRISON_CAP_NET_LISTEN_OVERRIDE))
1828                                                 net_listen_ov_local = 1;
1829                                 }
1830                                 if (inp->inp_laddr.s_addr == INADDR_ANY &&
1831                                     jinp_wild == NULL) {
1832                                         jinp_wild = inp;
1833                                         if (PRISON_CAP_ISSET(pr->pr_caps, PRISON_CAP_NET_LISTEN_OVERRIDE))
1834                                                 net_listen_ov_wild = 1;
1835                                 }
1836                         } else {
1837                                 if (inp->inp_laddr.s_addr == laddr.s_addr) {
1838                                         REL_PCBINFO_TOKEN(pcbinfo);
1839                                         return (inp);
1840                                 }
1841                                 if (inp->inp_laddr.s_addr == INADDR_ANY)
1842                                         local_wild = inp;
1843                         }
1844                 }
1845
1846                 REL_PCBINFO_TOKEN(pcbinfo);
1847
1848                 if (net_listen_ov_local)
1849                         return jinp;
1850                 if (net_listen_ov_wild)
1851                         return jinp_wild;
1852                 if (local_wild)
1853                         return (local_wild);
1854                 if (jinp)
1855                         return (jinp);
1856                 return (jinp_wild);
1857         }
1858
1859         /*
1860          * Not found.
1861          */
1862         return (NULL);
1863 }
1864
1865 struct inpcb *
1866 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1867     u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1868     boolean_t wildcard, struct ifnet *ifp)
1869 {
1870         return in_pcblookup_pkthash(pcbinfo, faddr, fport_arg,
1871             laddr, lport_arg, wildcard, ifp, NULL);
1872 }
1873
1874 /*
1875  * Insert PCB into connection hash table.
1876  */
1877 void
1878 in_pcbinsconnhash(struct inpcb *inp)
1879 {
1880         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1881         struct inpcbhead *bucket;
1882         u_int32_t hashkey_faddr, hashkey_laddr;
1883
1884 #ifdef INET6
1885         if (INP_ISIPV6(inp)) {
1886                 hashkey_faddr = inp->in6p_faddr.s6_addr32[3] /* XXX JH */;
1887                 hashkey_laddr = inp->in6p_laddr.s6_addr32[3] /* XXX JH */;
1888         } else {
1889 #endif
1890                 hashkey_faddr = inp->inp_faddr.s_addr;
1891                 hashkey_laddr = inp->inp_laddr.s_addr;
1892 #ifdef INET6
1893         }
1894 #endif
1895
1896         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1897             ("not in the correct netisr"));
1898         ASSERT_INP_NOTINHASH(inp);
1899         inp->inp_flags |= INP_CONNECTED;
1900
1901         /*
1902          * Insert into the connection hash table.
1903          */
1904         bucket = &pcbinfo->hashbase[INP_PCBCONNHASH(hashkey_faddr,
1905             inp->inp_fport, hashkey_laddr, inp->inp_lport, pcbinfo->hashmask)];
1906         LIST_INSERT_HEAD(bucket, inp, inp_hash);
1907 }
1908
1909 /*
1910  * Remove PCB from connection hash table.
1911  */
1912 void
1913 in_pcbremconnhash(struct inpcb *inp)
1914 {
1915         struct inpcbinfo *pcbinfo __debugvar = inp->inp_pcbinfo;
1916
1917         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1918             ("not in the correct netisr"));
1919         KASSERT(inp->inp_flags & INP_CONNECTED, ("inp not connected"));
1920
1921         LIST_REMOVE(inp, inp_hash);
1922         inp->inp_flags &= ~INP_CONNECTED;
1923 }
1924
1925 /*
1926  * Insert PCB into port hash table.
1927  */
1928 void
1929 in_pcbinsporthash(struct inpcbporthead *pcbporthash, struct inpcb *inp)
1930 {
1931         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1932         struct inpcbport *phd;
1933
1934         /*
1935          * If the porthashbase is shared across several cpus, it must
1936          * have been locked.
1937          */
1938         ASSERT_PORTHASH_TOKEN_HELD(pcbporthash);
1939
1940         /*
1941          * Insert into the port hash table.
1942          */
1943
1944         /* Go through port list and look for a head for this lport. */
1945         LIST_FOREACH(phd, pcbporthash, phd_hash) {
1946                 if (phd->phd_port == inp->inp_lport)
1947                         break;
1948         }
1949
1950         /* If none exists, use saved one and tack it on. */
1951         if (phd == NULL) {
1952                 KKASSERT(pcbinfo->portsave != NULL);
1953                 phd = pcbinfo->portsave;
1954                 pcbinfo->portsave = NULL;
1955                 phd->phd_port = inp->inp_lport;
1956                 LIST_INIT(&phd->phd_pcblist);
1957                 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
1958         }
1959
1960         inp->inp_porthash = pcbporthash;
1961         inp->inp_phd = phd;
1962         LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
1963
1964         /*
1965          * Malloc one inpcbport for later use.  It is safe to use
1966          * "wait" malloc here (port token would be released, if
1967          * malloc ever blocked), since all changes to the porthash
1968          * are done.
1969          */
1970         if (pcbinfo->portsave == NULL) {
1971                 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave),
1972                                             M_PCB, M_INTWAIT | M_ZERO);
1973         }
1974 }
1975
1976 void
1977 in_pcbinsporthash_lport(struct inpcb *inp)
1978 {
1979         struct inpcbporthead *porthash;
1980
1981         porthash = OBTAIN_LPORTHASH_TOKEN(inp->inp_pcbinfo, inp->inp_lport);
1982         in_pcbinsporthash(porthash, inp);
1983         REL_PORTHASH_TOKEN(porthash);
1984 }
1985
1986 void
1987 in_pcbremporthash(struct inpcb *inp)
1988 {
1989         struct inpcbporthead *porthash;
1990         struct inpcbport *phd;
1991
1992         if (inp->inp_phd == NULL)
1993                 return;
1994         KASSERT(inp->inp_lport != 0, ("inpcb has no lport"));
1995
1996         porthash = inp->inp_porthash;
1997         KASSERT(porthash != NULL, ("no porthash"));
1998
1999         GET_PORTHASH_TOKEN(porthash);
2000
2001         phd = inp->inp_phd;
2002         LIST_REMOVE(inp, inp_portlist);
2003         if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
2004                 LIST_REMOVE(phd, phd_hash);
2005                 kfree(phd, M_PCB);
2006         }
2007
2008         REL_PORTHASH_TOKEN(porthash);
2009
2010         inp->inp_phd = NULL;
2011         /* NOTE: Don't whack inp_lport, which may be used later */
2012 }
2013
2014 static struct inp_localgroup *
2015 inp_localgroup_alloc(u_char af, uint16_t port,
2016     const union in_dependaddr *addr, int size)
2017 {
2018         struct inp_localgroup *grp;
2019
2020         grp = kmalloc(__offsetof(struct inp_localgroup, il_inp[size]),
2021                       M_TEMP, M_INTWAIT | M_ZERO);
2022         grp->il_af = af;
2023         grp->il_lport = port;
2024         grp->il_dependladdr = *addr;
2025         grp->il_inpsiz = size;
2026
2027         return grp;
2028 }
2029
2030 static void
2031 inp_localgroup_free(struct inp_localgroup *grp)
2032 {
2033         kfree(grp, M_TEMP);
2034 }
2035
2036 static void
2037 inp_localgroup_destroy(struct inp_localgroup *grp)
2038 {
2039         LIST_REMOVE(grp, il_list);
2040         inp_localgroup_free(grp);
2041 }
2042
2043 static void
2044 inp_localgroup_copy(struct inp_localgroup *grp,
2045                     const struct inp_localgroup *old_grp)
2046 {
2047         int i;
2048
2049         KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
2050             ("invalid new local group size %d and old local group count %d",
2051              grp->il_inpsiz, old_grp->il_inpcnt));
2052         for (i = 0; i < old_grp->il_inpcnt; ++i)
2053                 grp->il_inp[i] = old_grp->il_inp[i];
2054         grp->il_inpcnt = old_grp->il_inpcnt;
2055 }
2056
2057 static void
2058 in_pcbinslocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2059 {
2060         struct inp_localgrphead *hdr;
2061         struct inp_localgroup *grp, *grp_alloc = NULL;
2062         u_char isjailed;
2063         int i, idx;
2064
2065         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
2066
2067         if (pcbinfo->localgrphashbase == NULL)
2068                 return;
2069
2070         /*
2071          * Further separate groups by whether the inp is jailed or not.
2072          * This allows the inp_localgroup_lookup() code to manage port
2073          * overloading between jails and non-jails.
2074          *
2075          * XXX all jails are collected into one group, which works fine
2076          *     as we expect the jails to be listening on different addresses.
2077          *     If this changes in the future we may have to break the groups
2078          *     up by prison pointer as well.
2079          */
2080         if (inp->inp_socket && inp->inp_socket->so_cred)
2081                 isjailed = jailed(inp->inp_socket->so_cred);
2082         else
2083                 isjailed = 0;
2084
2085         hdr = &pcbinfo->localgrphashbase[
2086             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
2087
2088 again:
2089         LIST_FOREACH(grp, hdr, il_list) {
2090                 if (grp->il_af == inp->inp_af &&
2091                     grp->il_lport == inp->inp_lport &&
2092                     grp->il_jailed == isjailed &&
2093                     memcmp(&grp->il_dependladdr,
2094                            &inp->inp_inc.inc_ie.ie_dependladdr,
2095                            sizeof(grp->il_dependladdr)) == 0) {
2096                         break;
2097                 }
2098         }
2099         if (grp == NULL) {
2100                 /*
2101                  * Create a new local group
2102                  */
2103                 if (grp_alloc == NULL) {
2104                         grp_alloc = inp_localgroup_alloc(inp->inp_af,
2105                             inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
2106                             INP_LOCALGROUP_SIZMIN);
2107                         /*
2108                          * Local group allocation could block and the
2109                          * local group w/ the same property might have
2110                          * been added by others when we were blocked;
2111                          * check again.
2112                          */
2113                         goto again;
2114                 } else {
2115                         /* Local group has been allocated; link it */
2116                         grp = grp_alloc;
2117                         grp->il_jailed = isjailed;
2118                         grp_alloc = NULL;
2119                         LIST_INSERT_HEAD(hdr, grp, il_list);
2120                 }
2121         } else if (grp->il_inpcnt == grp->il_inpsiz) {
2122 #if 0
2123                 /*
2124                  * REMOVED - Ensure that all entries are placed in the
2125                  *           localgroup so jail operations can be
2126                  *           deterministic on a il_lport basis.
2127                  */
2128                 if (grp->il_inpsiz >= INP_LOCALGROUP_SIZMAX) {
2129                         static int limit_logged = 0;
2130
2131                         if (!limit_logged) {
2132                                 limit_logged = 1;
2133                                 kprintf("local group port %d, "
2134                                     "limit reached\n", ntohs(grp->il_lport));
2135                         }
2136                         if (grp_alloc != NULL) {
2137                                 /*
2138                                  * This would happen if the local group
2139                                  * w/ the same property was expanded when
2140                                  * our local group allocation blocked.
2141                                  */
2142                                 inp_localgroup_free(grp_alloc);
2143                         }
2144                         return;
2145                 }
2146 #endif
2147
2148                 /*
2149                  * Expand this local group
2150                  */
2151                 if (grp_alloc == NULL ||
2152                     grp->il_inpcnt >= grp_alloc->il_inpsiz) {
2153                         if (grp_alloc != NULL)
2154                                 inp_localgroup_free(grp_alloc);
2155                         grp_alloc = inp_localgroup_alloc(grp->il_af,
2156                             grp->il_lport, &grp->il_dependladdr,
2157                             grp->il_inpsiz * 2);
2158                         /*
2159                          * Local group allocation could block and the
2160                          * local group w/ the same property might have
2161                          * been expanded by others when we were blocked;
2162                          * check again.
2163                          */
2164                         goto again;
2165                 }
2166
2167                 /*
2168                  * Save the old local group, link the new one, and then
2169                  * destroy the old local group
2170                  */
2171                 inp_localgroup_copy(grp_alloc, grp);
2172                 LIST_INSERT_HEAD(hdr, grp_alloc, il_list);
2173                 inp_localgroup_destroy(grp);
2174
2175                 grp = grp_alloc;
2176                 grp->il_jailed = isjailed;
2177                 grp_alloc = NULL;
2178         } else {
2179                 /*
2180                  * Found the local group
2181                  */
2182                 if (grp_alloc != NULL) {
2183                         /*
2184                          * This would happen if the local group w/ the
2185                          * same property was added or expanded when our
2186                          * local group allocation blocked.
2187                          */
2188                         inp_localgroup_free(grp_alloc);
2189                         grp_alloc = NULL;
2190                 }
2191         }
2192
2193         KASSERT(grp->il_inpcnt < grp->il_inpsiz,
2194             ("invalid local group size %d and count %d",
2195              grp->il_inpsiz, grp->il_inpcnt));
2196
2197         /*
2198          * Keep the local group sorted by the inpcb local group index
2199          * in ascending order.
2200          *
2201          * This eases the multi-process userland application which uses
2202          * SO_REUSEPORT sockets and binds process to the owner cpu of
2203          * the SO_REUSEPORT socket:
2204          * If we didn't sort the local group by the inpcb local group
2205          * index and one of the process owning an inpcb in this local
2206          * group restarted, e.g. crashed and restarted by watchdog,
2207          * other processes owning a inpcb in this local group would have
2208          * to detect that event, refetch its socket's owner cpu, and
2209          * re-bind.
2210          */
2211         idx = grp->il_inpcnt;
2212         for (i = 0; i < idx; ++i) {
2213                 struct inpcb *oinp = grp->il_inp[i];
2214
2215                 if (oinp->inp_lgrpindex > i) {
2216                         if (inp->inp_lgrpindex < 0) {
2217                                 inp->inp_lgrpindex = i;
2218                         } else if (inp->inp_lgrpindex != i) {
2219                                 if (bootverbose) {
2220                                         kprintf("inp %p: grpidx %d, "
2221                                             "assigned to %d, cpu%d\n",
2222                                             inp, inp->inp_lgrpindex, i,
2223                                             mycpuid);
2224                                 }
2225                         }
2226                         grp->il_inp[i] = inp;
2227
2228                         /* Pull down inpcbs */
2229                         for (; i < grp->il_inpcnt; ++i) {
2230                                 struct inpcb *oinp1 = grp->il_inp[i + 1];
2231
2232                                 grp->il_inp[i + 1] = oinp;
2233                                 oinp = oinp1;
2234                         }
2235                         grp->il_inpcnt++;
2236                         return;
2237                 }
2238         }
2239
2240         if (inp->inp_lgrpindex < 0) {
2241                 inp->inp_lgrpindex = idx;
2242         } else if (inp->inp_lgrpindex != idx) {
2243                 if (bootverbose) {
2244                         kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n",
2245                             inp, inp->inp_lgrpindex, idx, mycpuid);
2246                 }
2247         }
2248         grp->il_inp[idx] = inp;
2249         grp->il_inpcnt++;
2250 }
2251
2252 void
2253 in_pcbinswildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2254 {
2255         struct inpcontainer *ic;
2256         struct inpcontainerhead *bucket;
2257
2258         GET_PCBINFO_TOKEN(pcbinfo);
2259
2260         in_pcbinslocalgrphash_oncpu(inp, pcbinfo);
2261
2262         bucket = &pcbinfo->wildcardhashbase[
2263             INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
2264
2265         ic = kmalloc(sizeof(struct inpcontainer), M_TEMP, M_INTWAIT);
2266         ic->ic_inp = inp;
2267         LIST_INSERT_HEAD(bucket, ic, ic_list);
2268
2269         REL_PCBINFO_TOKEN(pcbinfo);
2270 }
2271
2272 /*
2273  * Insert PCB into wildcard hash table.
2274  */
2275 void
2276 in_pcbinswildcardhash(struct inpcb *inp)
2277 {
2278         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2279
2280         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
2281             ("not in correct netisr"));
2282         ASSERT_INP_NOTINHASH(inp);
2283         inp->inp_flags |= INP_WILDCARD;
2284
2285         in_pcbinswildcardhash_oncpu(inp, pcbinfo);
2286 }
2287
2288 static void
2289 in_pcbremlocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2290 {
2291         struct inp_localgrphead *hdr;
2292         struct inp_localgroup *grp;
2293
2294         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
2295
2296         if (pcbinfo->localgrphashbase == NULL)
2297                 return;
2298
2299         hdr = &pcbinfo->localgrphashbase[
2300             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
2301
2302         LIST_FOREACH(grp, hdr, il_list) {
2303                 int i;
2304
2305                 for (i = 0; i < grp->il_inpcnt; ++i) {
2306                         if (grp->il_inp[i] != inp)
2307                                 continue;
2308
2309                         if (grp->il_inpcnt == 1) {
2310                                 /* Destroy this local group */
2311                                 inp_localgroup_destroy(grp);
2312                         } else {
2313                                 /* Pull up inpcbs */
2314                                 for (; i + 1 < grp->il_inpcnt; ++i)
2315                                         grp->il_inp[i] = grp->il_inp[i + 1];
2316                                 grp->il_inpcnt--;
2317                         }
2318                         return;
2319                 }
2320         }
2321 }
2322
2323 void
2324 in_pcbremwildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2325 {
2326         struct inpcontainer *ic;
2327         struct inpcontainerhead *head;
2328
2329         GET_PCBINFO_TOKEN(pcbinfo);
2330
2331         in_pcbremlocalgrphash_oncpu(inp, pcbinfo);
2332
2333         /* find bucket */
2334         head = &pcbinfo->wildcardhashbase[
2335             INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
2336
2337         LIST_FOREACH(ic, head, ic_list) {
2338                 if (ic->ic_inp == inp)
2339                         goto found;
2340         }
2341         REL_PCBINFO_TOKEN(pcbinfo);
2342         return;                 /* not found! */
2343
2344 found:
2345         LIST_REMOVE(ic, ic_list);       /* remove container from bucket chain */
2346         REL_PCBINFO_TOKEN(pcbinfo);
2347         kfree(ic, M_TEMP);              /* deallocate container */
2348 }
2349
2350 /*
2351  * Remove PCB from wildcard hash table.
2352  */
2353 void
2354 in_pcbremwildcardhash(struct inpcb *inp)
2355 {
2356         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2357
2358         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
2359             ("not in correct netisr"));
2360         KASSERT(inp->inp_flags & INP_WILDCARD, ("inp not wildcard"));
2361
2362         in_pcbremwildcardhash_oncpu(inp, pcbinfo);
2363         inp->inp_lgrpindex = -1;
2364         inp->inp_flags &= ~INP_WILDCARD;
2365 }
2366
2367 /*
2368  * Remove PCB from various lists.
2369  */
2370 void
2371 in_pcbremlists(struct inpcb *inp)
2372 {
2373         in_pcbremporthash(inp);
2374         if (inp->inp_flags & INP_WILDCARD) {
2375                 in_pcbremwildcardhash(inp);
2376         } else if (inp->inp_flags & INP_CONNECTED) {
2377                 in_pcbremconnhash(inp);
2378         }
2379
2380         if (inp->inp_flags & INP_ONLIST)
2381                 in_pcbofflist(inp);
2382 }
2383
2384 int
2385 prison_xinpcb(struct thread *td, struct inpcb *inp)
2386 {
2387         struct ucred *cr;
2388
2389         if (td->td_proc == NULL)
2390                 return (0);
2391         cr = td->td_proc->p_ucred;
2392         if (cr->cr_prison == NULL)
2393                 return (0);
2394         if (inp->inp_socket && inp->inp_socket->so_cred &&
2395             inp->inp_socket->so_cred->cr_prison &&
2396             cr->cr_prison == inp->inp_socket->so_cred->cr_prison)
2397                 return (0);
2398         return (1);
2399 }
2400
2401 int
2402 in_pcblist_range(SYSCTL_HANDLER_ARGS)
2403 {
2404         struct inpcbinfo *pcbinfo_arr = arg1;
2405         int pcbinfo_arrlen = arg2;
2406         struct inpcb *marker;
2407         int cpu, origcpu;
2408         int error, n;
2409
2410         KASSERT(pcbinfo_arrlen <= netisr_ncpus && pcbinfo_arrlen >= 1,
2411             ("invalid pcbinfo count %d", pcbinfo_arrlen));
2412
2413         /*
2414          * The process of preparing the TCB list is too time-consuming and
2415          * resource-intensive to repeat twice on every request.
2416          */
2417         n = 0;
2418         if (req->oldptr == NULL) {
2419                 for (cpu = 0; cpu < pcbinfo_arrlen; ++cpu)
2420                         n += pcbinfo_arr[cpu].ipi_count;
2421                 req->oldidx = (n + n/8 + 10) * sizeof(struct xinpcb);
2422                 return 0;
2423         }
2424
2425         if (req->newptr != NULL)
2426                 return EPERM;
2427
2428         marker = kmalloc(sizeof(struct inpcb), M_TEMP, M_WAITOK|M_ZERO);
2429         marker->inp_flags |= INP_PLACEMARKER;
2430
2431         /*
2432          * OK, now we're committed to doing something.  Re-fetch ipi_count
2433          * after obtaining the generation count.
2434          */
2435         error = 0;
2436         origcpu = mycpuid;
2437         for (cpu = 0; cpu < pcbinfo_arrlen && error == 0; ++cpu) {
2438                 struct inpcbinfo *pcbinfo = &pcbinfo_arr[cpu];
2439                 struct inpcb *inp;
2440                 struct xinpcb xi;
2441                 int i;
2442
2443                 lwkt_migratecpu(cpu);
2444
2445                 GET_PCBINFO_TOKEN(pcbinfo);
2446
2447                 n = pcbinfo->ipi_count;
2448
2449                 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
2450                 i = 0;
2451                 while ((inp = LIST_NEXT(marker, inp_list)) != NULL && i < n) {
2452                         LIST_REMOVE(marker, inp_list);
2453                         LIST_INSERT_AFTER(inp, marker, inp_list);
2454
2455                         if (inp->inp_flags & INP_PLACEMARKER)
2456                                 continue;
2457                         if (prison_xinpcb(req->td, inp))
2458                                 continue;
2459
2460                         bzero(&xi, sizeof xi);
2461                         xi.xi_len = sizeof xi;
2462                         bcopy(inp, &xi.xi_inp, sizeof *inp);
2463                         if (inp->inp_socket)
2464                                 sotoxsocket(inp->inp_socket, &xi.xi_socket);
2465                         if ((error = SYSCTL_OUT(req, &xi, sizeof xi)) != 0)
2466                                 break;
2467                         ++i;
2468                 }
2469                 LIST_REMOVE(marker, inp_list);
2470
2471                 REL_PCBINFO_TOKEN(pcbinfo);
2472
2473                 if (error == 0 && i < n) {
2474                         bzero(&xi, sizeof xi);
2475                         xi.xi_len = sizeof xi;
2476                         while (i < n) {
2477                                 error = SYSCTL_OUT(req, &xi, sizeof xi);
2478                                 if (error)
2479                                         break;
2480                                 ++i;
2481                         }
2482                 }
2483         }
2484
2485         lwkt_migratecpu(origcpu);
2486         kfree(marker, M_TEMP);
2487         return error;
2488 }
2489
2490 int
2491 in_pcblist_ncpus(SYSCTL_HANDLER_ARGS)
2492 {
2493
2494         return (in_pcblist_range(oidp, arg1, netisr_ncpus, req));
2495 }
2496
2497 void
2498 in_savefaddr(struct socket *so, const struct sockaddr *faddr)
2499 {
2500         struct sockaddr_in *sin;
2501
2502         KASSERT(faddr->sa_family == AF_INET,
2503             ("not AF_INET faddr %d", faddr->sa_family));
2504
2505         sin = kmalloc(sizeof(*sin), M_SONAME, M_WAITOK | M_ZERO);
2506         sin->sin_family = AF_INET;
2507         sin->sin_len = sizeof(*sin);
2508         sin->sin_port = ((const struct sockaddr_in *)faddr)->sin_port;
2509         sin->sin_addr = ((const struct sockaddr_in *)faddr)->sin_addr;
2510
2511         so->so_faddr = (struct sockaddr *)sin;
2512 }
2513
2514 void
2515 in_pcbportinfo_init(struct inpcbportinfo *portinfo, int hashsize,
2516     u_short offset)
2517 {
2518         memset(portinfo, 0, sizeof(*portinfo));
2519
2520         portinfo->offset = offset;
2521         portinfo->porthashbase = phashinit(hashsize, M_PCB,
2522             &portinfo->porthashcnt);
2523 }
2524
2525 void
2526 in_pcbportrange(u_short *hi0, u_short *lo0, u_short ofs, u_short step)
2527 {
2528         int hi, lo;
2529
2530         if (step == 1)
2531                 return;
2532
2533         hi = *hi0;
2534         lo = *lo0;
2535
2536         hi = rounddown(hi, step);
2537         hi += ofs;
2538         if (hi > (int)*hi0)
2539                 hi -= step;
2540
2541         lo = roundup(lo, step);
2542         lo -= (step - ofs);
2543         if (lo < (int)*lo0)
2544                 lo += step;
2545
2546         *hi0 = hi;
2547         *lo0 = lo;
2548 }
2549
2550 void
2551 in_pcbglobalinit(void)
2552 {
2553         int cpu;
2554
2555         in_pcbmarkers = kmalloc(netisr_ncpus * sizeof(struct inpcb), M_PCB,
2556             M_WAITOK | M_ZERO);
2557         in_pcbcontainer_markers =
2558             kmalloc(netisr_ncpus * sizeof(struct inpcontainer), M_PCB,
2559             M_WAITOK | M_ZERO);
2560
2561         for (cpu = 0; cpu < netisr_ncpus; ++cpu) {
2562                 struct inpcontainer *ic = &in_pcbcontainer_markers[cpu];
2563                 struct inpcb *marker = &in_pcbmarkers[cpu];
2564
2565                 marker->inp_flags |= INP_PLACEMARKER;
2566                 ic->ic_inp = marker;
2567         }
2568 }
2569
2570 struct inpcb *
2571 in_pcbmarker(void)
2572 {
2573
2574         ASSERT_NETISR_NCPUS(mycpuid);
2575         return &in_pcbmarkers[mycpuid];
2576 }
2577
2578 struct inpcontainer *
2579 in_pcbcontainer_marker(void)
2580 {
2581
2582         ASSERT_NETISR_NCPUS(mycpuid);
2583         return &in_pcbcontainer_markers[mycpuid];
2584 }
2585
2586 void
2587 in_pcbresetroute(struct inpcb *inp)
2588 {
2589         struct route *ro = &inp->inp_route;
2590
2591         if (ro->ro_rt != NULL)
2592                 RTFREE(ro->ro_rt);
2593         bzero(ro, sizeof(*ro));
2594 }