sys/netinet/in_pcb.c

   1 /*
   2  * Copyright (c) 2004 Jeffrey M. Hsu.  All rights reserved.
   3  * Copyright (c) 2004 The DragonFly Project.  All rights reserved.
   4  *
   5  * This code is derived from software contributed to The DragonFly Project
   6  * by Jeffrey M. Hsu.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the name of The DragonFly Project nor the names of its
  17  *    contributors may be used to endorse or promote products derived
  18  *    from this software without specific, prior written permission.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  23  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  24  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  25  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  26  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  27  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  28  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  29  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  30  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  */
  33
  34 /*
  35  * Copyright (c) 1982, 1986, 1991, 1993, 1995
  36  *      The Regents of the University of California.  All rights reserved.
  37  *
  38  * Redistribution and use in source and binary forms, with or without
  39  * modification, are permitted provided that the following conditions
  40  * are met:
  41  * 1. Redistributions of source code must retain the above copyright
  42  *    notice, this list of conditions and the following disclaimer.
  43  * 2. Redistributions in binary form must reproduce the above copyright
  44  *    notice, this list of conditions and the following disclaimer in the
  45  *    documentation and/or other materials provided with the distribution.
  46  * 3. Neither the name of the University nor the names of its contributors
  47  *    may be used to endorse or promote products derived from this software
  48  *    without specific prior written permission.
  49  *
  50  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  51  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  52  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  53  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  54  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  55  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  56  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  57  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  58  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  59  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  60  * SUCH DAMAGE.
  61  *
  62  *      @(#)in_pcb.c    8.4 (Berkeley) 5/24/95
  63  * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $
  64  */
  65
  66 #include "opt_ipsec.h"
  67 #include "opt_inet6.h"
  68
  69 #include <sys/param.h>
  70 #include <sys/systm.h>
  71 #include <sys/malloc.h>
  72 #include <sys/mbuf.h>
  73 #include <sys/domain.h>
  74 #include <sys/protosw.h>
  75 #include <sys/socket.h>
  76 #include <sys/socketvar.h>
  77 #include <sys/proc.h>
  78 #include <sys/priv.h>
  79 #include <sys/jail.h>
  80 #include <sys/kernel.h>
  81 #include <sys/sysctl.h>
  82
  83 #include <sys/thread2.h>
  84 #include <sys/socketvar2.h>
  85 #include <sys/msgport2.h>
  86
  87 #include <machine/limits.h>
  88
  89 #include <net/if.h>
  90 #include <net/if_types.h>
  91 #include <net/route.h>
  92 #include <net/netisr2.h>
  93 #include <net/toeplitz2.h>
  94
  95 #include <netinet/in.h>
  96 #include <netinet/in_pcb.h>
  97 #include <netinet/in_var.h>
  98 #include <netinet/ip_var.h>
  99 #ifdef INET6
 100 #include <netinet/ip6.h>
 101 #include <netinet6/ip6_var.h>
 102 #endif /* INET6 */
 103
 104 #ifdef IPSEC
 105 #include <netinet6/ipsec.h>
 106 #include <netproto/key/key.h>
 107 #include <netproto/ipsec/esp_var.h>
 108 #endif
 109
 110 #ifdef FAST_IPSEC
 111 #if defined(IPSEC) || defined(IPSEC_ESP)
 112 #error "Bad idea: don't compile with both IPSEC and FAST_IPSEC!"
 113 #endif
 114
 115 #include <netproto/ipsec/ipsec.h>
 116 #include <netproto/ipsec/key.h>
 117 #define IPSEC
 118 #endif /* FAST_IPSEC */
 119
 120 #define INP_LOCALGROUP_SIZMIN   8
 121 #define INP_LOCALGROUP_SIZMAX   256
 122
 123 static struct inpcb *in_pcblookup_local(struct inpcbporthead *porthash,
 124                 struct in_addr laddr, u_int lport_arg, int wild_okay,
 125                 struct ucred *cred);
 126
 127 struct in_addr zeroin_addr;
 128
 129 /*
 130  * These configure the range of local port addresses assigned to
 131  * "unspecified" outgoing connections/packets/whatever.
 132  */
 133 int ipport_lowfirstauto = IPPORT_RESERVED - 1;  /* 1023 */
 134 int ipport_lowlastauto = IPPORT_RESERVEDSTART;  /* 600 */
 135
 136 int ipport_firstauto = IPPORT_RESERVED;         /* 1024 */
 137 int ipport_lastauto = IPPORT_USERRESERVED;      /* 5000 */
 138
 139 int ipport_hifirstauto = IPPORT_HIFIRSTAUTO;    /* 49152 */
 140 int ipport_hilastauto = IPPORT_HILASTAUTO;      /* 65535 */
 141
 142 #define RANGECHK(var, min, max) \
 143         if ((var) < (min)) { (var) = (min); } \
 144         else if ((var) > (max)) { (var) = (max); }
 145
 146 int udpencap_enable = 1;        /* enabled by default */
 147 int udpencap_port = 4500;       /* triggers decapsulation */
 148
 149 /*
 150  * Per-netisr inpcb markers.
 151  * NOTE: they should only be used in netisrs.
 152  */
 153 static struct inpcb             *in_pcbmarkers;
 154 static struct inpcontainer      *in_pcbcontainer_markers;
 155
 156 static int
 157 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
 158 {
 159         int error;
 160
 161         error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
 162         if (!error) {
 163                 RANGECHK(ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
 164                 RANGECHK(ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
 165
 166                 RANGECHK(ipport_firstauto, IPPORT_RESERVED, USHRT_MAX);
 167                 RANGECHK(ipport_lastauto, IPPORT_RESERVED, USHRT_MAX);
 168
 169                 RANGECHK(ipport_hifirstauto, IPPORT_RESERVED, USHRT_MAX);
 170                 RANGECHK(ipport_hilastauto, IPPORT_RESERVED, USHRT_MAX);
 171         }
 172         return (error);
 173 }
 174
 175 #undef RANGECHK
 176
 177 SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports");
 178
 179 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW,
 180            &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", "");
 181 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, CTLTYPE_INT|CTLFLAG_RW,
 182            &ipport_lowlastauto, 0, &sysctl_net_ipport_check, "I", "");
 183 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, CTLTYPE_INT|CTLFLAG_RW,
 184            &ipport_firstauto, 0, &sysctl_net_ipport_check, "I", "");
 185 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, CTLTYPE_INT|CTLFLAG_RW,
 186            &ipport_lastauto, 0, &sysctl_net_ipport_check, "I", "");
 187 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, CTLTYPE_INT|CTLFLAG_RW,
 188            &ipport_hifirstauto, 0, &sysctl_net_ipport_check, "I", "");
 189 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW,
 190            &ipport_hilastauto, 0, &sysctl_net_ipport_check, "I", "");
 191
 192 static int ip_porthash_trycount = 15;
 193 SYSCTL_INT(_net_inet_ip, OID_AUTO, porthash_trycount, CTLFLAG_RW,
 194     &ip_porthash_trycount, 0,
 195     "Number of tries to find local port matching hash of 4-tuple");
 196
 197 /*
 198  * in_pcb.c: manage the Protocol Control Blocks.
 199  *
 200  * NOTE: It is assumed that most of these functions will be called from
 201  * a critical section.  XXX - There are, unfortunately, a few exceptions
 202  * to this rule that should be fixed.
 203  *
 204  * NOTE: The caller should initialize the cpu field to the cpu running the
 205  * protocol stack associated with this inpcbinfo.
 206  */
 207
 208 void
 209 in_pcbinfo_init(struct inpcbinfo *pcbinfo, int cpu, boolean_t shared)
 210 {
 211         KASSERT(cpu >= 0 && cpu < ncpus, ("invalid cpu%d", cpu));
 212         pcbinfo->cpu = cpu;
 213
 214         LIST_INIT(&pcbinfo->pcblisthead);
 215         pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), M_PCB,
 216                                     M_WAITOK | M_ZERO);
 217
 218         if (shared) {
 219                 pcbinfo->infotoken = kmalloc(sizeof(struct lwkt_token),
 220                     M_PCB, M_WAITOK);
 221                 lwkt_token_init(pcbinfo->infotoken, "infotoken");
 222         } else {
 223                 pcbinfo->infotoken = NULL;
 224         }
 225 }
 226
 227 void
 228 in_pcbportinfo_set(struct inpcbinfo *pcbinfo, struct inpcbportinfo *portinfo,
 229     int portinfo_cnt)
 230 {
 231
 232         KASSERT(portinfo_cnt > 0, ("invalid portinfo_cnt %d", portinfo_cnt));
 233         pcbinfo->portinfo = portinfo;
 234         pcbinfo->portinfo_cnt = portinfo_cnt;
 235 }
 236
 237 struct baddynamicports baddynamicports;
 238
 239 /*
 240  * Check if the specified port is invalid for dynamic allocation.
 241  */
 242 int
 243 in_baddynamic(u_int16_t port, u_int16_t proto)
 244 {
 245         switch (proto) {
 246         case IPPROTO_TCP:
 247                 return (DP_ISSET(baddynamicports.tcp, port));
 248         case IPPROTO_UDP:
 249 #ifdef IPSEC
 250                 /* Cannot preset this as it is a sysctl */
 251                 if (port == udpencap_port)
 252                         return (1);
 253 #endif
 254                 return (DP_ISSET(baddynamicports.udp, port));
 255         default:
 256                 return (0);
 257         }
 258 }
 259
 260 void
 261 in_pcbonlist(struct inpcb *inp)
 262 {
 263         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 264
 265         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
 266             ("not in the correct netisr"));
 267         KASSERT((inp->inp_flags & INP_ONLIST) == 0, ("already on pcblist"));
 268         inp->inp_flags |= INP_ONLIST;
 269
 270         GET_PCBINFO_TOKEN(pcbinfo);
 271         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list);
 272         pcbinfo->ipi_count++;
 273         REL_PCBINFO_TOKEN(pcbinfo);
 274 }
 275
 276 void
 277 in_pcbofflist(struct inpcb *inp)
 278 {
 279         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 280
 281         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
 282             ("not in the correct netisr"));
 283         KASSERT(inp->inp_flags & INP_ONLIST, ("not on pcblist"));
 284         inp->inp_flags &= ~INP_ONLIST;
 285
 286         GET_PCBINFO_TOKEN(pcbinfo);
 287         LIST_REMOVE(inp, inp_list);
 288         KASSERT(pcbinfo->ipi_count > 0,
 289             ("invalid inpcb count %d", pcbinfo->ipi_count));
 290         pcbinfo->ipi_count--;
 291         REL_PCBINFO_TOKEN(pcbinfo);
 292 }
 293
 294 /*
 295  * Allocate a PCB and associate it with the socket.
 296  */
 297 int
 298 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
 299 {
 300         struct inpcb *inp;
 301 #ifdef IPSEC
 302         int error;
 303 #endif
 304
 305         inp = kmalloc(pcbinfo->ipi_size, M_PCB, M_WAITOK|M_ZERO|M_NULLOK);
 306         if (inp == NULL)
 307                 return (ENOMEM);
 308         inp->inp_lgrpindex = -1;
 309         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
 310         inp->inp_pcbinfo = pcbinfo;
 311         inp->inp_socket = so;
 312 #ifdef IPSEC
 313         error = ipsec_init_policy(so, &inp->inp_sp);
 314         if (error != 0) {
 315                 kfree(inp, M_PCB);
 316                 return (error);
 317         }
 318 #endif
 319 #ifdef INET6
 320         if (INP_CHECK_SOCKAF(so, AF_INET6)) {
 321                 if (ip6_auto_flowlabel)
 322                         inp->inp_flags |= IN6P_AUTOFLOWLABEL;
 323                 inp->inp_af = AF_INET6;
 324         } else
 325 #endif
 326         inp->inp_af = AF_INET;
 327         soreference(so);
 328         so->so_pcb = inp;
 329
 330         in_pcbonlist(inp);
 331         return (0);
 332 }
 333
 334 /*
 335  * Unlink a pcb with the intention of moving it to another cpu with a
 336  * different pcbinfo.  While unlinked nothing should attempt to dereference
 337  * inp_pcbinfo, NULL it out so we assert if it does.
 338  */
 339 void
 340 in_pcbunlink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags)
 341 {
 342         KASSERT(inp->inp_pcbinfo == pcbinfo, ("pcbinfo mismatch"));
 343         KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0,
 344             ("already linked"));
 345
 346         in_pcbofflist(inp);
 347         inp->inp_pcbinfo = NULL;
 348 }
 349
 350 void
 351 in_pcbunlink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
 352 {
 353         in_pcbunlink_flags(inp, pcbinfo, INP_WILDCARD);
 354 }
 355
 356 /*
 357  * Relink a pcb into a new pcbinfo.
 358  */
 359 void
 360 in_pcblink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags)
 361 {
 362         KASSERT(inp->inp_pcbinfo == NULL, ("has pcbinfo"));
 363         KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0,
 364             ("already linked"));
 365
 366         inp->inp_pcbinfo = pcbinfo;
 367         in_pcbonlist(inp);
 368 }
 369
 370 void
 371 in_pcblink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
 372 {
 373         return in_pcblink_flags(inp, pcbinfo, INP_WILDCARD);
 374 }
 375
 376 static boolean_t
 377 in_pcbporthash_update(struct inpcbportinfo *portinfo,
 378     struct inpcb *inp, u_short lport, struct ucred *cred, int wild)
 379 {
 380         struct inpcbporthead *porthash;
 381
 382         /*
 383          * This has to be atomic.  If the porthash is shared across multiple
 384          * protocol threads, e.g. tcp and udp, then the token must be held.
 385          */
 386         porthash = in_pcbporthash_head(portinfo, lport);
 387         GET_PORTHASH_TOKEN(porthash);
 388
 389         if (in_pcblookup_local(porthash, inp->inp_laddr, lport,
 390             wild, cred) != NULL) {
 391                 REL_PORTHASH_TOKEN(porthash);
 392                 return FALSE;
 393         }
 394         inp->inp_lport = lport;
 395         in_pcbinsporthash(porthash, inp);
 396
 397         REL_PORTHASH_TOKEN(porthash);
 398         return TRUE;
 399 }
 400
 401 static int
 402 in_pcbsetlport(struct inpcb *inp, int wild, struct ucred *cred)
 403 {
 404         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 405         struct inpcbportinfo *portinfo;
 406         u_short first, last, lport, step;
 407         u_short *lastport;
 408         int count, error;
 409         int portinfo_first, portinfo_idx;
 410
 411         inp->inp_flags |= INP_ANONPORT;
 412
 413         step = pcbinfo->portinfo_cnt;
 414         portinfo_first = mycpuid % pcbinfo->portinfo_cnt;
 415         portinfo_idx = portinfo_first;
 416 loop:
 417         portinfo = &pcbinfo->portinfo[portinfo_idx];
 418
 419         if (inp->inp_flags & INP_HIGHPORT) {
 420                 first = ipport_hifirstauto;     /* sysctl */
 421                 last  = ipport_hilastauto;
 422                 lastport = &portinfo->lasthi;
 423         } else if (inp->inp_flags & INP_LOWPORT) {
 424                 if (cred &&
 425                     (error =
 426                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 427                         inp->inp_laddr.s_addr = INADDR_ANY;
 428                         return error;
 429                 }
 430                 first = ipport_lowfirstauto;    /* 1023 */
 431                 last  = ipport_lowlastauto;     /* 600 */
 432                 lastport = &portinfo->lastlow;
 433         } else {
 434                 first = ipport_firstauto;       /* sysctl */
 435                 last  = ipport_lastauto;
 436                 lastport = &portinfo->lastport;
 437         }
 438
 439         /*
 440          * Simple check to ensure all ports are not used up causing
 441          * a deadlock here.
 442          *
 443          * We split the two cases (up and down) so that the direction
 444          * is not being tested on each round of the loop.
 445          */
 446         if (first > last) {
 447                 /*
 448                  * counting down
 449                  */
 450                 in_pcbportrange(&first, &last, portinfo->offset, step);
 451                 count = (first - last) / step;
 452
 453                 for (;;) {
 454                         if (count-- < 0) {      /* completely used? */
 455                                 error = EADDRNOTAVAIL;
 456                                 break;
 457                         }
 458                         lport = in_pcblastport_down(lastport,
 459                             first, last, step);
 460                         KKASSERT((lport % pcbinfo->portinfo_cnt) ==
 461                             portinfo->offset);
 462                         lport = htons(lport);
 463
 464                         if (in_pcbporthash_update(portinfo, inp, lport,
 465                             cred, wild)) {
 466                                 error = 0;
 467                                 break;
 468                         }
 469                 }
 470         } else {
 471                 /*
 472                  * counting up
 473                  */
 474                 in_pcbportrange(&last, &first, portinfo->offset, step);
 475                 count = (last - first) / step;
 476
 477                 for (;;) {
 478                         if (count-- < 0) {      /* completely used? */
 479                                 error = EADDRNOTAVAIL;
 480                                 break;
 481                         }
 482                         lport = in_pcblastport_up(lastport, first, last, step);
 483                         KKASSERT((lport % pcbinfo->portinfo_cnt) ==
 484                             portinfo->offset);
 485                         lport = htons(lport);
 486
 487                         if (in_pcbporthash_update(portinfo, inp, lport,
 488                             cred, wild)) {
 489                                 error = 0;
 490                                 break;
 491                         }
 492                 }
 493         }
 494
 495         if (error) {
 496                 /* Try next portinfo */
 497                 portinfo_idx++;
 498                 portinfo_idx %= pcbinfo->portinfo_cnt;
 499                 if (portinfo_idx != portinfo_first)
 500                         goto loop;
 501                 inp->inp_laddr.s_addr = INADDR_ANY;
 502         }
 503         return error;
 504 }
 505
 506 int
 507 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
 508 {
 509         struct socket *so = inp->inp_socket;
 510         struct sockaddr_in jsin;
 511         struct ucred *cred = NULL;
 512         int wild = 0;
 513
 514         if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */
 515                 return (EADDRNOTAVAIL);
 516         if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
 517                 return (EINVAL);        /* already bound */
 518
 519         if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT)))
 520                 wild = 1;    /* neither SO_REUSEADDR nor SO_REUSEPORT is set */
 521         if (td->td_proc)
 522                 cred = td->td_proc->p_ucred;
 523
 524         if (nam != NULL) {
 525                 struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 526                 struct inpcbinfo *pcbinfo;
 527                 struct inpcbportinfo *portinfo;
 528                 struct inpcbporthead *porthash;
 529                 struct inpcb *t;
 530                 u_short lport, lport_ho;
 531                 int reuseport = (so->so_options & SO_REUSEPORT);
 532                 int error;
 533
 534                 if (nam->sa_len != sizeof *sin)
 535                         return (EINVAL);
 536 #ifdef notdef
 537                 /*
 538                  * We should check the family, but old programs
 539                  * incorrectly fail to initialize it.
 540                  */
 541                 if (sin->sin_family != AF_INET)
 542                         return (EAFNOSUPPORT);
 543 #endif
 544                 if (!prison_replace_wildcards(td, nam))
 545                         return (EINVAL);
 546
 547                 lport = sin->sin_port;
 548                 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 549                         /*
 550                          * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
 551                          * allow complete duplication of binding if
 552                          * SO_REUSEPORT is set, or if SO_REUSEADDR is set
 553                          * and a multicast address is bound on both
 554                          * new and duplicated sockets.
 555                          */
 556                         if (so->so_options & SO_REUSEADDR)
 557                                 reuseport = SO_REUSEADDR | SO_REUSEPORT;
 558                 } else if (sin->sin_addr.s_addr != INADDR_ANY) {
 559                         sin->sin_port = 0;              /* yech... */
 560                         bzero(&sin->sin_zero, sizeof sin->sin_zero);
 561                         if (ifa_ifwithaddr((struct sockaddr *)sin) == NULL)
 562                                 return (EADDRNOTAVAIL);
 563                 }
 564
 565                 inp->inp_laddr = sin->sin_addr;
 566
 567                 jsin.sin_family = AF_INET;
 568                 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 569                 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 570                         inp->inp_laddr.s_addr = INADDR_ANY;
 571                         return (EINVAL);
 572                 }
 573                 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 574
 575                 if (lport == 0) {
 576                         /* Auto-select local port */
 577                         return in_pcbsetlport(inp, wild, cred);
 578                 }
 579                 lport_ho = ntohs(lport);
 580
 581                 /* GROSS */
 582                 if (lport_ho < IPPORT_RESERVED && cred &&
 583                     (error =
 584                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 585                         inp->inp_laddr.s_addr = INADDR_ANY;
 586                         return (error);
 587                 }
 588
 589                 /*
 590                  * Locate the proper portinfo based on lport
 591                  */
 592                 pcbinfo = inp->inp_pcbinfo;
 593                 portinfo =
 594                     &pcbinfo->portinfo[lport_ho % pcbinfo->portinfo_cnt];
 595                 KKASSERT((lport_ho % pcbinfo->portinfo_cnt) ==
 596                     portinfo->offset);
 597
 598                 /*
 599                  * This has to be atomic.  If the porthash is shared across
 600                  * multiple protocol threads, e.g. tcp and udp then the token
 601                  * must be held.
 602                  */
 603                 porthash = in_pcbporthash_head(portinfo, lport);
 604                 GET_PORTHASH_TOKEN(porthash);
 605
 606                 if (so->so_cred->cr_uid != 0 &&
 607                     !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 608                         t = in_pcblookup_local(porthash, sin->sin_addr, lport,
 609                             INPLOOKUP_WILDCARD, cred);
 610                         if (t &&
 611                             (so->so_cred->cr_uid !=
 612                              t->inp_socket->so_cred->cr_uid)) {
 613                                 inp->inp_laddr.s_addr = INADDR_ANY;
 614                                 error = EADDRINUSE;
 615                                 goto done;
 616                         }
 617                 }
 618                 if (cred && !prison_replace_wildcards(td, nam)) {
 619                         inp->inp_laddr.s_addr = INADDR_ANY;
 620                         error = EADDRNOTAVAIL;
 621                         goto done;
 622                 }
 623                 t = in_pcblookup_local(porthash, sin->sin_addr, lport,
 624                     wild, cred);
 625                 if (t && !(reuseport & t->inp_socket->so_options)) {
 626                         inp->inp_laddr.s_addr = INADDR_ANY;
 627                         error = EADDRINUSE;
 628                         goto done;
 629                 }
 630                 inp->inp_lport = lport;
 631                 in_pcbinsporthash(porthash, inp);
 632                 error = 0;
 633 done:
 634                 REL_PORTHASH_TOKEN(porthash);
 635                 return (error);
 636         } else {
 637                 jsin.sin_family = AF_INET;
 638                 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 639                 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 640                         inp->inp_laddr.s_addr = INADDR_ANY;
 641                         return (EINVAL);
 642                 }
 643                 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 644
 645                 return in_pcbsetlport(inp, wild, cred);
 646         }
 647 }
 648
 649 static struct inpcb *
 650 in_pcblookup_localremote(struct inpcbporthead *porthash, struct in_addr laddr,
 651     u_short lport, struct in_addr faddr, u_short fport, struct ucred *cred)
 652 {
 653         struct inpcb *inp;
 654         struct inpcbport *phd;
 655         struct inpcb *match = NULL;
 656
 657         /*
 658          * If the porthashbase is shared across several cpus, it must
 659          * have been locked.
 660          */
 661         ASSERT_PORTHASH_TOKEN_HELD(porthash);
 662
 663         /*
 664          * Best fit PCB lookup.
 665          *
 666          * First see if this local port is in use by looking on the
 667          * port hash list.
 668          */
 669         LIST_FOREACH(phd, porthash, phd_hash) {
 670                 if (phd->phd_port == lport)
 671                         break;
 672         }
 673         if (phd != NULL) {
 674                 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
 675 #ifdef INET6
 676                         if (!INP_ISIPV4(inp))
 677                                 continue;
 678 #endif
 679                         if (inp->inp_laddr.s_addr != INADDR_ANY &&
 680                             inp->inp_laddr.s_addr != laddr.s_addr)
 681                                 continue;
 682
 683                         if (inp->inp_faddr.s_addr != INADDR_ANY &&
 684                             inp->inp_faddr.s_addr != faddr.s_addr)
 685                                 continue;
 686
 687                         if (inp->inp_fport != 0 && inp->inp_fport != fport)
 688                                 continue;
 689
 690                         if (cred == NULL ||
 691                             cred->cr_prison ==
 692                             inp->inp_socket->so_cred->cr_prison) {
 693                                 match = inp;
 694                                 break;
 695                         }
 696                 }
 697         }
 698         return (match);
 699 }
 700
 701 static boolean_t
 702 in_pcbporthash_update4(struct inpcbportinfo *portinfo,
 703     struct inpcb *inp, u_short lport, const struct sockaddr_in *sin,
 704     struct ucred *cred)
 705 {
 706         struct inpcbporthead *porthash;
 707
 708         /*
 709          * This has to be atomic.  If the porthash is shared across multiple
 710          * protocol threads, e.g. tcp and udp, then the token must be held.
 711          */
 712         porthash = in_pcbporthash_head(portinfo, lport);
 713         GET_PORTHASH_TOKEN(porthash);
 714
 715         if (in_pcblookup_localremote(porthash, inp->inp_laddr,
 716             lport, sin->sin_addr, sin->sin_port, cred) != NULL) {
 717                 REL_PORTHASH_TOKEN(porthash);
 718                 return FALSE;
 719         }
 720         inp->inp_lport = lport;
 721         in_pcbinsporthash(porthash, inp);
 722
 723         REL_PORTHASH_TOKEN(porthash);
 724         return TRUE;
 725 }
 726
 727 int
 728 in_pcbbind_remote(struct inpcb *inp, const struct sockaddr *remote,
 729     struct thread *td)
 730 {
 731         struct proc *p = td->td_proc;
 732         u_short *lastport;
 733         const struct sockaddr_in *sin = (const struct sockaddr_in *)remote;
 734         struct sockaddr_in jsin;
 735         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 736         struct inpcbportinfo *portinfo;
 737         struct ucred *cred = NULL;
 738         u_short first, last, lport, step;
 739         int count, error, selfconn;
 740         int portinfo_first, portinfo_idx;
 741         int hash_cpu, hash_count, hash_count0;
 742         uint32_t hash_base = 0, hash;
 743
 744         if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */
 745                 return (EADDRNOTAVAIL);
 746
 747         KKASSERT(inp->inp_laddr.s_addr != INADDR_ANY);
 748         if (inp->inp_lport != 0)
 749                 return (EINVAL);        /* already bound */
 750
 751         KKASSERT(p);
 752         cred = p->p_ucred;
 753
 754         jsin.sin_family = AF_INET;
 755         jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 756         if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 757                 inp->inp_laddr.s_addr = INADDR_ANY;
 758                 return (EINVAL);
 759         }
 760         inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 761
 762         hash_count0 = ip_porthash_trycount;
 763         if (hash_count0 > 0) {
 764                 hash_base = toeplitz_piecemeal_addr(sin->sin_addr.s_addr) ^
 765                     toeplitz_piecemeal_addr(inp->inp_laddr.s_addr) ^
 766                     toeplitz_piecemeal_port(sin->sin_port);
 767         } else {
 768                 hash_count0 = 0;
 769         }
 770
 771         inp->inp_flags |= INP_ANONPORT;
 772
 773         step = pcbinfo->portinfo_cnt;
 774         portinfo_first = mycpuid % pcbinfo->portinfo_cnt;
 775         portinfo_idx = portinfo_first;
 776 loop:
 777         hash_cpu = portinfo_idx % netisr_ncpus;
 778         portinfo = &pcbinfo->portinfo[portinfo_idx];
 779         selfconn = 0;
 780
 781         if (inp->inp_flags & INP_HIGHPORT) {
 782                 first = ipport_hifirstauto;     /* sysctl */
 783                 last  = ipport_hilastauto;
 784                 lastport = &portinfo->lasthi;
 785         } else if (inp->inp_flags & INP_LOWPORT) {
 786                 if (cred &&
 787                     (error =
 788                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 789                         inp->inp_laddr.s_addr = INADDR_ANY;
 790                         return (error);
 791                 }
 792                 first = ipport_lowfirstauto;    /* 1023 */
 793                 last  = ipport_lowlastauto;     /* 600 */
 794                 lastport = &portinfo->lastlow;
 795         } else {
 796                 first = ipport_firstauto;       /* sysctl */
 797                 last  = ipport_lastauto;
 798                 lastport = &portinfo->lastport;
 799         }
 800
 801         /* This could happen on loopback interface */
 802 #define IS_SELFCONNECT(inp, lport, sin)                 \
 803         (__predict_false((sin)->sin_port == (lport) &&  \
 804          (sin)->sin_addr.s_addr == (inp)->inp_laddr.s_addr))
 805
 806         /*
 807          * Simple check to ensure all ports are not used up causing
 808          * a deadlock here.
 809          *
 810          * We split the two cases (up and down) so that the direction
 811          * is not being tested on each round of the loop.
 812          */
 813         hash_count = hash_count0;
 814         if (first > last) {
 815                 /*
 816                  * counting down
 817                  */
 818                 in_pcbportrange(&first, &last, portinfo->offset, step);
 819                 count = ((first - last) / step) + hash_count;
 820
 821                 for (;;) {
 822                         if (count-- < 0) {      /* completely used? */
 823                                 error = EADDRNOTAVAIL;
 824                                 break;
 825                         }
 826
 827                         lport = in_pcblastport_down(lastport, first, last,
 828                             step);
 829                         KKASSERT((lport % pcbinfo->portinfo_cnt) ==
 830                             portinfo->offset);
 831                         lport = htons(lport);
 832                         if (IS_SELFCONNECT(inp, lport, sin)) {
 833                                 if (!selfconn) {
 834                                         ++count; /* don't count this try */
 835                                         selfconn = 1;
 836                                 }
 837                                 continue;
 838                         }
 839
 840                         if (hash_count) {
 841                                 --hash_count;
 842                                 hash = hash_base ^
 843                                     toeplitz_piecemeal_port(lport);
 844                                 if (netisr_hashcpu(hash) != hash_cpu &&
 845                                     hash_count)
 846                                         continue;
 847                         }
 848
 849                         if (in_pcbporthash_update4(portinfo,
 850                             inp, lport, sin, cred)) {
 851                                 error = 0;
 852                                 break;
 853                         }
 854                 }
 855         } else {
 856                 /*
 857                  * counting up
 858                  */
 859                 in_pcbportrange(&last, &first, portinfo->offset, step);
 860                 count = ((last - first) / step) + hash_count;
 861
 862                 for (;;) {
 863                         if (count-- < 0) {      /* completely used? */
 864                                 error = EADDRNOTAVAIL;
 865                                 break;
 866                         }
 867
 868                         lport = in_pcblastport_up(lastport, first, last, step);
 869                         KKASSERT((lport % pcbinfo->portinfo_cnt) ==
 870                             portinfo->offset);
 871                         lport = htons(lport);
 872                         if (IS_SELFCONNECT(inp, lport, sin)) {
 873                                 if (!selfconn) {
 874                                         ++count; /* don't count this try */
 875                                         selfconn = 1;
 876                                 }
 877                                 continue;
 878                         }
 879
 880                         if (hash_count) {
 881                                 --hash_count;
 882                                 hash = hash_base ^
 883                                     toeplitz_piecemeal_port(lport);
 884                                 if (netisr_hashcpu(hash) != hash_cpu &&
 885                                     hash_count)
 886                                         continue;
 887                         }
 888
 889                         if (in_pcbporthash_update4(portinfo,
 890                             inp, lport, sin, cred)) {
 891                                 error = 0;
 892                                 break;
 893                         }
 894                 }
 895         }
 896
 897 #undef IS_SELFCONNECT
 898
 899         if (error) {
 900                 /* Try next portinfo */
 901                 portinfo_idx++;
 902                 portinfo_idx %= pcbinfo->portinfo_cnt;
 903                 if (portinfo_idx != portinfo_first)
 904                         goto loop;
 905                 inp->inp_laddr.s_addr = INADDR_ANY;
 906         }
 907         return error;
 908 }
 909
 910 /*
 911  *   Transform old in_pcbconnect() into an inner subroutine for new
 912  *   in_pcbconnect(): Do some validity-checking on the remote
 913  *   address (in mbuf 'nam') and then determine local host address
 914  *   (i.e., which interface) to use to access that remote host.
 915  *
 916  *   This preserves definition of in_pcbconnect(), while supporting a
 917  *   slightly different version for T/TCP.  (This is more than
 918  *   a bit of a kludge, but cleaning up the internal interfaces would
 919  *   have forced minor changes in every protocol).
 920  */
 921 int
 922 in_pcbladdr_find(struct inpcb *inp, struct sockaddr *nam,
 923     struct sockaddr_in **plocal_sin, struct thread *td, int find)
 924 {
 925         struct in_ifaddr *ia;
 926         struct ucred *cred = NULL;
 927         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 928         struct sockaddr *jsin;
 929         int jailed = 0, alloc_route = 0;
 930
 931         if (nam->sa_len != sizeof *sin)
 932                 return (EINVAL);
 933         if (sin->sin_family != AF_INET)
 934                 return (EAFNOSUPPORT);
 935         if (sin->sin_port == 0)
 936                 return (EADDRNOTAVAIL);
 937         if (td && td->td_proc && td->td_proc->p_ucred)
 938                 cred = td->td_proc->p_ucred;
 939         if (cred && cred->cr_prison)
 940                 jailed = 1;
 941         if (!TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) {
 942                 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia;
 943                 /*
 944                  * If the destination address is INADDR_ANY,
 945                  * use the primary local address.
 946                  * If the supplied address is INADDR_BROADCAST,
 947                  * and the primary interface supports broadcast,
 948                  * choose the broadcast address for that interface.
 949                  */
 950                 if (sin->sin_addr.s_addr == INADDR_ANY)
 951                         sin->sin_addr = IA_SIN(ia)->sin_addr;
 952                 else if (sin->sin_addr.s_addr == (u_long)INADDR_BROADCAST &&
 953                     (ia->ia_ifp->if_flags & IFF_BROADCAST))
 954                         sin->sin_addr = satosin(&ia->ia_broadaddr)->sin_addr;
 955         }
 956         if (find) {
 957                 struct route *ro;
 958
 959                 ia = NULL;
 960                 /*
 961                  * If route is known or can be allocated now,
 962                  * our src addr is taken from the i/f, else punt.
 963                  * Note that we should check the address family of the cached
 964                  * destination, in case of sharing the cache with IPv6.
 965                  */
 966                 ro = &inp->inp_route;
 967                 if (ro->ro_rt &&
 968                     (!(ro->ro_rt->rt_flags & RTF_UP) ||
 969                      ro->ro_dst.sa_family != AF_INET ||
 970                      satosin(&ro->ro_dst)->sin_addr.s_addr !=
 971                                       sin->sin_addr.s_addr ||
 972                      inp->inp_socket->so_options & SO_DONTROUTE)) {
 973                         RTFREE(ro->ro_rt);
 974                         ro->ro_rt = NULL;
 975                 }
 976                 if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/
 977                     (ro->ro_rt == NULL ||
 978                     ro->ro_rt->rt_ifp == NULL)) {
 979                         /* No route yet, so try to acquire one */
 980                         bzero(&ro->ro_dst, sizeof(struct sockaddr_in));
 981                         ro->ro_dst.sa_family = AF_INET;
 982                         ro->ro_dst.sa_len = sizeof(struct sockaddr_in);
 983                         ((struct sockaddr_in *) &ro->ro_dst)->sin_addr =
 984                                 sin->sin_addr;
 985                         rtalloc(ro);
 986                         alloc_route = 1;
 987                 }
 988                 /*
 989                  * If we found a route, use the address
 990                  * corresponding to the outgoing interface
 991                  * unless it is the loopback (in case a route
 992                  * to our address on another net goes to loopback).
 993                  */
 994                 if (ro->ro_rt &&
 995                     !(ro->ro_rt->rt_ifp->if_flags & IFF_LOOPBACK)) {
 996                         if (jailed) {
 997                                 if (jailed_ip(cred->cr_prison,
 998                                     ro->ro_rt->rt_ifa->ifa_addr)) {
 999                                         ia = ifatoia(ro->ro_rt->rt_ifa);
1000                                 }
1001                         } else {
1002                                 ia = ifatoia(ro->ro_rt->rt_ifa);
1003                         }
1004                 }
1005                 if (ia == NULL) {
1006                         u_short fport = sin->sin_port;
1007
1008                         sin->sin_port = 0;
1009                         ia = ifatoia(ifa_ifwithdstaddr(sintosa(sin)));
1010                         if (ia && jailed && !jailed_ip(cred->cr_prison,
1011                             sintosa(&ia->ia_addr)))
1012                                 ia = NULL;
1013                         if (ia == NULL)
1014                                 ia = ifatoia(ifa_ifwithnet(sintosa(sin)));
1015                         if (ia && jailed && !jailed_ip(cred->cr_prison,
1016                             sintosa(&ia->ia_addr)))
1017                                 ia = NULL;
1018                         sin->sin_port = fport;
1019                         if (ia == NULL &&
1020                             !TAILQ_EMPTY(&in_ifaddrheads[mycpuid]))
1021                                 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia;
1022                         if (ia && jailed && !jailed_ip(cred->cr_prison,
1023                             sintosa(&ia->ia_addr)))
1024                                 ia = NULL;
1025
1026                         if (!jailed && ia == NULL)
1027                                 goto fail;
1028                 }
1029                 /*
1030                  * If the destination address is multicast and an outgoing
1031                  * interface has been set as a multicast option, use the
1032                  * address of that interface as our source address.
1033                  */
1034                 if (!jailed && IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
1035                     inp->inp_moptions != NULL) {
1036                         struct ip_moptions *imo;
1037                         struct ifnet *ifp;
1038
1039                         imo = inp->inp_moptions;
1040                         if ((ifp = imo->imo_multicast_ifp) != NULL) {
1041                                 struct in_ifaddr_container *iac;
1042
1043                                 ia = NULL;
1044                                 TAILQ_FOREACH(iac,
1045                                 &in_ifaddrheads[mycpuid], ia_link) {
1046                                         if (iac->ia->ia_ifp == ifp) {
1047                                                 ia = iac->ia;
1048                                                 break;
1049                                         }
1050                                 }
1051                                 if (ia == NULL)
1052                                         goto fail;
1053                         }
1054                 }
1055                 /*
1056                  * Don't do pcblookup call here; return interface in plocal_sin
1057                  * and exit to caller, that will do the lookup.
1058                  */
1059                 if (ia == NULL && jailed) {
1060                         if ((jsin = prison_get_nonlocal(
1061                                 cred->cr_prison, AF_INET, NULL)) != NULL ||
1062                             (jsin = prison_get_local(
1063                                 cred->cr_prison, AF_INET, NULL)) != NULL) {
1064                                 *plocal_sin = satosin(jsin);
1065                         } else {
1066                                 /* IPv6 only Jail */
1067                                 goto fail;
1068                         }
1069                 } else {
1070                         *plocal_sin = &ia->ia_addr;
1071                 }
1072         }
1073         return (0);
1074 fail:
1075         if (alloc_route)
1076                 in_pcbresetroute(inp);
1077         return (EADDRNOTAVAIL);
1078 }
1079
1080 int
1081 in_pcbladdr(struct inpcb *inp, struct sockaddr *nam,
1082     struct sockaddr_in **plocal_sin, struct thread *td)
1083 {
1084         return in_pcbladdr_find(inp, nam, plocal_sin, td,
1085             (inp->inp_laddr.s_addr == INADDR_ANY));
1086 }
1087
1088 /*
1089  * Outer subroutine:
1090  * Connect from a socket to a specified address.
1091  * Both address and port must be specified in argument sin.
1092  * If don't have a local address for this socket yet,
1093  * then pick one.
1094  */
1095 int
1096 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
1097 {
1098         struct sockaddr_in *if_sin;
1099         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
1100         int error;
1101
1102         if_sin = NULL;  /* avoid gcc warnings */
1103
1104         /* Call inner routine to assign local interface address. */
1105         if ((error = in_pcbladdr(inp, nam, &if_sin, td)) != 0)
1106                 return (error);
1107
1108         if (in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port,
1109                               inp->inp_laddr.s_addr ?
1110                                 inp->inp_laddr : if_sin->sin_addr,
1111                               inp->inp_lport, FALSE, NULL) != NULL) {
1112                 return (EADDRINUSE);
1113         }
1114         if (inp->inp_laddr.s_addr == INADDR_ANY) {
1115                 if (inp->inp_lport == 0) {
1116                         error = in_pcbbind(inp, NULL, td);
1117                         if (error)
1118                                 return (error);
1119                 }
1120                 inp->inp_laddr = if_sin->sin_addr;
1121         }
1122         inp->inp_faddr = sin->sin_addr;
1123         inp->inp_fport = sin->sin_port;
1124         in_pcbinsconnhash(inp);
1125         return (0);
1126 }
1127
1128 void
1129 in_pcbdisconnect(struct inpcb *inp)
1130 {
1131
1132         in_pcbremconnhash(inp);
1133         inp->inp_faddr.s_addr = INADDR_ANY;
1134         inp->inp_fport = 0;
1135 }
1136
1137 void
1138 in_pcbdetach(struct inpcb *inp)
1139 {
1140         struct socket *so = inp->inp_socket;
1141         struct inpcbinfo *ipi = inp->inp_pcbinfo;
1142
1143 #ifdef IPSEC
1144         ipsec4_delete_pcbpolicy(inp);
1145 #endif /*IPSEC*/
1146         inp->inp_gencnt = ++ipi->ipi_gencnt;
1147         KKASSERT((so->so_state & SS_ASSERTINPROG) == 0);
1148         in_pcbremlists(inp);
1149         so->so_pcb = NULL;
1150         sofree(so);                     /* remove pcb ref */
1151         if (inp->inp_options)
1152                 m_free(inp->inp_options);
1153         if (inp->inp_route.ro_rt)
1154                 rtfree(inp->inp_route.ro_rt);
1155         ip_freemoptions(inp->inp_moptions);
1156         kfree(inp, M_PCB);
1157 }
1158
1159 /*
1160  * The socket may have an invalid PCB, i.e. NULL.  For example, a TCP
1161  * socket received RST.
1162  */
1163 static int
1164 in_setsockaddr(struct socket *so, struct sockaddr **nam)
1165 {
1166         struct inpcb *inp;
1167         struct sockaddr_in *sin;
1168
1169         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
1170         inp = so->so_pcb;
1171         if (!inp)
1172                 return (ECONNRESET);
1173
1174         sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1175         sin->sin_family = AF_INET;
1176         sin->sin_len = sizeof *sin;
1177         sin->sin_port = inp->inp_lport;
1178         sin->sin_addr = inp->inp_laddr;
1179
1180         *nam = (struct sockaddr *)sin;
1181         return (0);
1182 }
1183
1184 void
1185 in_setsockaddr_dispatch(netmsg_t msg)
1186 {
1187         int error;
1188
1189         error = in_setsockaddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1190         lwkt_replymsg(&msg->lmsg, error);
1191 }
1192
1193 /*
1194  * The socket may have an invalid PCB, i.e. NULL.  For example, a TCP
1195  * socket received RST.
1196  */
1197 int
1198 in_setpeeraddr(struct socket *so, struct sockaddr **nam)
1199 {
1200         struct inpcb *inp;
1201         struct sockaddr_in *sin;
1202
1203         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
1204         inp = so->so_pcb;
1205         if (!inp)
1206                 return (ECONNRESET);
1207
1208         sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1209         sin->sin_family = AF_INET;
1210         sin->sin_len = sizeof *sin;
1211         sin->sin_port = inp->inp_fport;
1212         sin->sin_addr = inp->inp_faddr;
1213
1214         *nam = (struct sockaddr *)sin;
1215         return (0);
1216 }
1217
1218 void
1219 in_setpeeraddr_dispatch(netmsg_t msg)
1220 {
1221         int error;
1222
1223         error = in_setpeeraddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1224         lwkt_replymsg(&msg->lmsg, error);
1225 }
1226
1227 void
1228 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int err,
1229     inp_notify_t notify)
1230 {
1231         struct inpcb *inp, *marker;
1232
1233         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1234             ("not in the correct netisr"));
1235         marker = &in_pcbmarkers[mycpuid];
1236
1237         /*
1238          * NOTE:
1239          * - If INP_PLACEMARKER is set we must ignore the rest of the
1240          *   structure and skip it.
1241          * - It is safe to nuke inpcbs here, since we are in their own
1242          *   netisr.
1243          */
1244         GET_PCBINFO_TOKEN(pcbinfo);
1245
1246         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1247         while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1248                 LIST_REMOVE(marker, inp_list);
1249                 LIST_INSERT_AFTER(inp, marker, inp_list);
1250
1251                 if (inp->inp_flags & INP_PLACEMARKER)
1252                         continue;
1253 #ifdef INET6
1254                 if (!INP_ISIPV4(inp))
1255                         continue;
1256 #endif
1257                 if (inp->inp_faddr.s_addr != faddr.s_addr ||
1258                     inp->inp_socket == NULL)
1259                         continue;
1260                 (*notify)(inp, err);            /* can remove inp from list! */
1261         }
1262         LIST_REMOVE(marker, inp_list);
1263
1264         REL_PCBINFO_TOKEN(pcbinfo);
1265 }
1266
1267 void
1268 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1269 {
1270         struct inpcb *inp, *marker;
1271
1272         /*
1273          * We only need to make sure that we are in netisr0, where all
1274          * multicast operation happen.  We could check inpcbinfo which
1275          * does not belong to netisr0 by holding the inpcbinfo's token.
1276          * In this case, the pcbinfo must be able to be shared, i.e.
1277          * pcbinfo->infotoken is not NULL.
1278          */
1279         ASSERT_IN_NETISR(0);
1280         KASSERT(pcbinfo->cpu == 0 || pcbinfo->infotoken != NULL,
1281             ("pcbinfo could not be shared"));
1282
1283         /*
1284          * Get a marker for the current netisr (netisr0).
1285          *
1286          * It is possible that the multicast address deletion blocks,
1287          * which could cause temporary token releasing.  So we use
1288          * inpcb marker here to get a coherent view of the inpcb list.
1289          *
1290          * While, on the other hand, moptions are only added and deleted
1291          * in netisr0, so we would not see staled moption or miss moption
1292          * even if the token was released due to the blocking multicast
1293          * address deletion.
1294          */
1295         marker = &in_pcbmarkers[mycpuid];
1296
1297         GET_PCBINFO_TOKEN(pcbinfo);
1298
1299         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1300         while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1301                 struct ip_moptions *imo;
1302
1303                 LIST_REMOVE(marker, inp_list);
1304                 LIST_INSERT_AFTER(inp, marker, inp_list);
1305
1306                 if (inp->inp_flags & INP_PLACEMARKER)
1307                         continue;
1308                 imo = inp->inp_moptions;
1309                 if (INP_ISIPV4(inp) && imo != NULL) {
1310                         int i, gap;
1311
1312                         /*
1313                          * Unselect the outgoing interface if it is being
1314                          * detached.
1315                          */
1316                         if (imo->imo_multicast_ifp == ifp)
1317                                 imo->imo_multicast_ifp = NULL;
1318
1319                         /*
1320                          * Drop multicast group membership if we joined
1321                          * through the interface being detached.
1322                          */
1323                         for (i = 0, gap = 0; i < imo->imo_num_memberships;
1324                             i++) {
1325                                 if (imo->imo_membership[i]->inm_ifp == ifp) {
1326                                         /*
1327                                          * NOTE:
1328                                          * This could block and the pcbinfo
1329                                          * token could be passively released.
1330                                          */
1331                                         in_delmulti(imo->imo_membership[i]);
1332                                         gap++;
1333                                 } else if (gap != 0)
1334                                         imo->imo_membership[i - gap] =
1335                                             imo->imo_membership[i];
1336                         }
1337                         imo->imo_num_memberships -= gap;
1338                 }
1339         }
1340         LIST_REMOVE(marker, inp_list);
1341
1342         REL_PCBINFO_TOKEN(pcbinfo);
1343 }
1344
1345 /*
1346  * Check for alternatives when higher level complains
1347  * about service problems.  For now, invalidate cached
1348  * routing information.  If the route was created dynamically
1349  * (by a redirect), time to try a default gateway again.
1350  */
1351 void
1352 in_losing(struct inpcb *inp)
1353 {
1354         struct rtentry *rt;
1355         struct rt_addrinfo rtinfo;
1356
1357         if ((rt = inp->inp_route.ro_rt)) {
1358                 bzero(&rtinfo, sizeof(struct rt_addrinfo));
1359                 rtinfo.rti_info[RTAX_DST] = rt_key(rt);
1360                 rtinfo.rti_info[RTAX_GATEWAY] = rt->rt_gateway;
1361                 rtinfo.rti_info[RTAX_NETMASK] = rt_mask(rt);
1362                 rtinfo.rti_flags = rt->rt_flags;
1363                 rt_missmsg(RTM_LOSING, &rtinfo, rt->rt_flags, 0);
1364                 if (rt->rt_flags & RTF_DYNAMIC) {
1365                         rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway,
1366                             rt_mask(rt), rt->rt_flags, NULL);
1367                 }
1368                 inp->inp_route.ro_rt = NULL;
1369                 rtfree(rt);
1370                 /*
1371                  * A new route can be allocated
1372                  * the next time output is attempted.
1373                  */
1374         }
1375 }
1376
1377 /*
1378  * After a routing change, flush old routing
1379  * and allocate a (hopefully) better one.
1380  */
1381 void
1382 in_rtchange(struct inpcb *inp, int err)
1383 {
1384         if (inp->inp_route.ro_rt) {
1385                 rtfree(inp->inp_route.ro_rt);
1386                 inp->inp_route.ro_rt = NULL;
1387                 /*
1388                  * A new route can be allocated the next time
1389                  * output is attempted.
1390                  */
1391         }
1392 }
1393
1394 /*
1395  * Lookup a PCB based on the local address and port.
1396  */
1397 static struct inpcb *
1398 in_pcblookup_local(struct inpcbporthead *porthash, struct in_addr laddr,
1399                    u_int lport_arg, int wild_okay, struct ucred *cred)
1400 {
1401         struct inpcb *inp;
1402         int matchwild = 3, wildcard;
1403         u_short lport = lport_arg;
1404         struct inpcbport *phd;
1405         struct inpcb *match = NULL;
1406
1407         /*
1408          * If the porthashbase is shared across several cpus, it must
1409          * have been locked.
1410          */
1411         ASSERT_PORTHASH_TOKEN_HELD(porthash);
1412
1413         /*
1414          * Best fit PCB lookup.
1415          *
1416          * First see if this local port is in use by looking on the
1417          * port hash list.
1418          */
1419         LIST_FOREACH(phd, porthash, phd_hash) {
1420                 if (phd->phd_port == lport)
1421                         break;
1422         }
1423         if (phd != NULL) {
1424                 /*
1425                  * Port is in use by one or more PCBs. Look for best
1426                  * fit.
1427                  */
1428                 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
1429                         wildcard = 0;
1430 #ifdef INET6
1431                         if (!INP_ISIPV4(inp))
1432                                 continue;
1433 #endif
1434                         if (inp->inp_faddr.s_addr != INADDR_ANY)
1435                                 wildcard++;
1436                         if (inp->inp_laddr.s_addr != INADDR_ANY) {
1437                                 if (laddr.s_addr == INADDR_ANY)
1438                                         wildcard++;
1439                                 else if (inp->inp_laddr.s_addr != laddr.s_addr)
1440                                         continue;
1441                         } else {
1442                                 if (laddr.s_addr != INADDR_ANY)
1443                                         wildcard++;
1444                         }
1445                         if (wildcard && !wild_okay)
1446                                 continue;
1447                         if (wildcard < matchwild &&
1448                             (cred == NULL ||
1449                              cred->cr_prison ==
1450                                         inp->inp_socket->so_cred->cr_prison)) {
1451                                 match = inp;
1452                                 matchwild = wildcard;
1453                                 if (matchwild == 0) {
1454                                         break;
1455                                 }
1456                         }
1457                 }
1458         }
1459         return (match);
1460 }
1461
1462 struct inpcb *
1463 in_pcblocalgroup_last(const struct inpcbinfo *pcbinfo,
1464     const struct inpcb *inp)
1465 {
1466         const struct inp_localgrphead *hdr;
1467         const struct inp_localgroup *grp;
1468         int i;
1469
1470         if (pcbinfo->localgrphashbase == NULL)
1471                 return NULL;
1472
1473         GET_PCBINFO_TOKEN(pcbinfo);
1474
1475         hdr = &pcbinfo->localgrphashbase[
1476             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
1477
1478         LIST_FOREACH(grp, hdr, il_list) {
1479                 if (grp->il_af == inp->inp_af &&
1480                     grp->il_lport == inp->inp_lport &&
1481                     memcmp(&grp->il_dependladdr,
1482                         &inp->inp_inc.inc_ie.ie_dependladdr,
1483                         sizeof(grp->il_dependladdr)) == 0) {
1484                         break;
1485                 }
1486         }
1487         if (grp == NULL || grp->il_inpcnt == 1) {
1488                 REL_PCBINFO_TOKEN(pcbinfo);
1489                 return NULL;
1490         }
1491
1492         KASSERT(grp->il_inpcnt >= 2,
1493             ("invalid localgroup inp count %d", grp->il_inpcnt));
1494         for (i = 0; i < grp->il_inpcnt; ++i) {
1495                 if (grp->il_inp[i] == inp) {
1496                         int last = grp->il_inpcnt - 1;
1497
1498                         if (i == last)
1499                                 last = grp->il_inpcnt - 2;
1500                         REL_PCBINFO_TOKEN(pcbinfo);
1501                         return grp->il_inp[last];
1502                 }
1503         }
1504         REL_PCBINFO_TOKEN(pcbinfo);
1505         return NULL;
1506 }
1507
1508 static struct inpcb *
1509 inp_localgroup_lookup(const struct inpcbinfo *pcbinfo,
1510     struct in_addr laddr, uint16_t lport, uint32_t pkt_hash)
1511 {
1512         struct inpcb *local_wild = NULL;
1513         const struct inp_localgrphead *hdr;
1514         const struct inp_localgroup *grp;
1515
1516         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1517
1518         hdr = &pcbinfo->localgrphashbase[
1519             INP_PCBLOCALGRPHASH(lport, pcbinfo->localgrphashmask)];
1520
1521         /*
1522          * Order of socket selection:
1523          * 1. non-wild.
1524          * 2. wild.
1525          *
1526          * NOTE: Local group does not contain jailed sockets
1527          */
1528         LIST_FOREACH(grp, hdr, il_list) {
1529 #ifdef INET6
1530                 if (grp->il_af != AF_INET)
1531                         continue;
1532 #endif
1533                 if (grp->il_lport == lport) {
1534                         int idx;
1535
1536                         /*
1537                          * Modulo-N is used here, which greatly reduces
1538                          * completion queue token contention, thus more
1539                          * cpu time is saved.
1540                          */
1541                         idx = netisr_hashlsb(pkt_hash) % grp->il_inpcnt;
1542                         if (grp->il_laddr.s_addr == laddr.s_addr)
1543                                 return grp->il_inp[idx];
1544                         else if (grp->il_laddr.s_addr == INADDR_ANY)
1545                                 local_wild = grp->il_inp[idx];
1546                 }
1547         }
1548         if (local_wild != NULL)
1549                 return local_wild;
1550         return NULL;
1551 }
1552
1553 /*
1554  * Lookup PCB in hash list.
1555  */
1556 struct inpcb *
1557 in_pcblookup_pkthash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1558     u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1559     boolean_t wildcard, struct ifnet *ifp, const struct mbuf *m)
1560 {
1561         struct inpcbhead *head;
1562         struct inpcb *inp, *jinp=NULL;
1563         u_short fport = fport_arg, lport = lport_arg;
1564
1565         /*
1566          * First look for an exact match.
1567          */
1568         head = &pcbinfo->hashbase[INP_PCBCONNHASH(faddr.s_addr, fport,
1569             laddr.s_addr, lport, pcbinfo->hashmask)];
1570         LIST_FOREACH(inp, head, inp_hash) {
1571 #ifdef INET6
1572                 if (!INP_ISIPV4(inp))
1573                         continue;
1574 #endif
1575                 if (in_hosteq(inp->inp_faddr, faddr) &&
1576                     in_hosteq(inp->inp_laddr, laddr) &&
1577                     inp->inp_fport == fport && inp->inp_lport == lport) {
1578                         /* found */
1579                         if (inp->inp_socket == NULL ||
1580                             inp->inp_socket->so_cred->cr_prison == NULL) {
1581                                 return (inp);
1582                         } else {
1583                                 if  (jinp == NULL)
1584                                         jinp = inp;
1585                         }
1586                 }
1587         }
1588         if (jinp != NULL)
1589                 return (jinp);
1590
1591         if (wildcard) {
1592                 struct inpcb *local_wild = NULL;
1593                 struct inpcb *jinp_wild = NULL;
1594                 struct inpcontainer *ic;
1595                 struct inpcontainerhead *chead;
1596                 struct sockaddr_in jsin;
1597                 struct ucred *cred;
1598
1599                 GET_PCBINFO_TOKEN(pcbinfo);
1600
1601                 /*
1602                  * Check local group first
1603                  */
1604                 if (pcbinfo->localgrphashbase != NULL &&
1605                     m != NULL && (m->m_flags & M_HASH) &&
1606                     !(ifp && ifp->if_type == IFT_FAITH)) {
1607                         inp = inp_localgroup_lookup(pcbinfo,
1608                             laddr, lport, m->m_pkthdr.hash);
1609                         if (inp != NULL) {
1610                                 REL_PCBINFO_TOKEN(pcbinfo);
1611                                 return inp;
1612                         }
1613                 }
1614
1615                 /*
1616                  * Order of socket selection:
1617                  * 1. non-jailed, non-wild.
1618                  * 2. non-jailed, wild.
1619                  * 3. jailed, non-wild.
1620                  * 4. jailed, wild.
1621                  */
1622                 jsin.sin_family = AF_INET;
1623                 chead = &pcbinfo->wildcardhashbase[
1624                     INP_PCBWILDCARDHASH(lport, pcbinfo->wildcardhashmask)];
1625                 LIST_FOREACH(ic, chead, ic_list) {
1626                         inp = ic->ic_inp;
1627                         if (inp->inp_flags & INP_PLACEMARKER)
1628                                 continue;
1629
1630                         jsin.sin_addr.s_addr = laddr.s_addr;
1631 #ifdef INET6
1632                         if (!INP_ISIPV4(inp))
1633                                 continue;
1634 #endif
1635                         if (inp->inp_socket != NULL)
1636                                 cred = inp->inp_socket->so_cred;
1637                         else
1638                                 cred = NULL;
1639                         if (cred != NULL && jailed(cred)) {
1640                                 if (jinp != NULL)
1641                                         continue;
1642                                 else
1643                                         if (!jailed_ip(cred->cr_prison,
1644                                             (struct sockaddr *)&jsin))
1645                                                 continue;
1646                         }
1647                         if (inp->inp_lport == lport) {
1648                                 if (ifp && ifp->if_type == IFT_FAITH &&
1649                                     !(inp->inp_flags & INP_FAITH))
1650                                         continue;
1651                                 if (inp->inp_laddr.s_addr == laddr.s_addr) {
1652                                         if (cred != NULL && jailed(cred)) {
1653                                                 jinp = inp;
1654                                         } else {
1655                                                 REL_PCBINFO_TOKEN(pcbinfo);
1656                                                 return (inp);
1657                                         }
1658                                 }
1659                                 if (inp->inp_laddr.s_addr == INADDR_ANY) {
1660                                         if (cred != NULL && jailed(cred))
1661                                                 jinp_wild = inp;
1662                                         else
1663                                                 local_wild = inp;
1664                                 }
1665                         }
1666                 }
1667
1668                 REL_PCBINFO_TOKEN(pcbinfo);
1669
1670                 if (local_wild != NULL)
1671                         return (local_wild);
1672                 if (jinp != NULL)
1673                         return (jinp);
1674                 return (jinp_wild);
1675         }
1676
1677         /*
1678          * Not found.
1679          */
1680         return (NULL);
1681 }
1682
1683 struct inpcb *
1684 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1685     u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1686     boolean_t wildcard, struct ifnet *ifp)
1687 {
1688         return in_pcblookup_pkthash(pcbinfo, faddr, fport_arg,
1689             laddr, lport_arg, wildcard, ifp, NULL);
1690 }
1691
1692 /*
1693  * Insert PCB into connection hash table.
1694  */
1695 void
1696 in_pcbinsconnhash(struct inpcb *inp)
1697 {
1698         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1699         struct inpcbhead *bucket;
1700         u_int32_t hashkey_faddr, hashkey_laddr;
1701
1702 #ifdef INET6
1703         if (INP_ISIPV6(inp)) {
1704                 hashkey_faddr = inp->in6p_faddr.s6_addr32[3] /* XXX JH */;
1705                 hashkey_laddr = inp->in6p_laddr.s6_addr32[3] /* XXX JH */;
1706         } else {
1707 #endif
1708                 hashkey_faddr = inp->inp_faddr.s_addr;
1709                 hashkey_laddr = inp->inp_laddr.s_addr;
1710 #ifdef INET6
1711         }
1712 #endif
1713
1714         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1715             ("not in the correct netisr"));
1716         ASSERT_INP_NOTINHASH(inp);
1717         inp->inp_flags |= INP_CONNECTED;
1718
1719         /*
1720          * Insert into the connection hash table.
1721          */
1722         bucket = &pcbinfo->hashbase[INP_PCBCONNHASH(hashkey_faddr,
1723             inp->inp_fport, hashkey_laddr, inp->inp_lport, pcbinfo->hashmask)];
1724         LIST_INSERT_HEAD(bucket, inp, inp_hash);
1725 }
1726
1727 /*
1728  * Remove PCB from connection hash table.
1729  */
1730 void
1731 in_pcbremconnhash(struct inpcb *inp)
1732 {
1733         struct inpcbinfo *pcbinfo __debugvar = inp->inp_pcbinfo;
1734
1735         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1736             ("not in the correct netisr"));
1737         KASSERT(inp->inp_flags & INP_CONNECTED, ("inp not connected"));
1738
1739         LIST_REMOVE(inp, inp_hash);
1740         inp->inp_flags &= ~INP_CONNECTED;
1741 }
1742
1743 /*
1744  * Insert PCB into port hash table.
1745  */
1746 void
1747 in_pcbinsporthash(struct inpcbporthead *pcbporthash, struct inpcb *inp)
1748 {
1749         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1750         struct inpcbport *phd;
1751
1752         /*
1753          * If the porthashbase is shared across several cpus, it must
1754          * have been locked.
1755          */
1756         ASSERT_PORTHASH_TOKEN_HELD(pcbporthash);
1757
1758         /*
1759          * Insert into the port hash table.
1760          */
1761
1762         /* Go through port list and look for a head for this lport. */
1763         LIST_FOREACH(phd, pcbporthash, phd_hash) {
1764                 if (phd->phd_port == inp->inp_lport)
1765                         break;
1766         }
1767
1768         /* If none exists, use saved one and tack it on. */
1769         if (phd == NULL) {
1770                 KKASSERT(pcbinfo->portsave != NULL);
1771                 phd = pcbinfo->portsave;
1772                 pcbinfo->portsave = NULL;
1773                 phd->phd_port = inp->inp_lport;
1774                 LIST_INIT(&phd->phd_pcblist);
1775                 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
1776         }
1777
1778         inp->inp_porthash = pcbporthash;
1779         inp->inp_phd = phd;
1780         LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
1781
1782         /*
1783          * Malloc one inpcbport for later use.  It is safe to use
1784          * "wait" malloc here (port token would be released, if
1785          * malloc ever blocked), since all changes to the porthash
1786          * are done.
1787          */
1788         if (pcbinfo->portsave == NULL) {
1789                 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave),
1790                                             M_PCB, M_INTWAIT | M_ZERO);
1791         }
1792 }
1793
1794 void
1795 in_pcbinsporthash_lport(struct inpcb *inp)
1796 {
1797         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1798         struct inpcbportinfo *portinfo;
1799         struct inpcbporthead *porthash;
1800         u_short lport_ho;
1801
1802         /* Locate the proper portinfo based on lport */
1803         lport_ho = ntohs(inp->inp_lport);
1804         portinfo = &pcbinfo->portinfo[lport_ho % pcbinfo->portinfo_cnt];
1805         KKASSERT((lport_ho % pcbinfo->portinfo_cnt) == portinfo->offset);
1806
1807         porthash = in_pcbporthash_head(portinfo, inp->inp_lport);
1808         GET_PORTHASH_TOKEN(porthash);
1809         in_pcbinsporthash(porthash, inp);
1810         REL_PORTHASH_TOKEN(porthash);
1811 }
1812
1813 void
1814 in_pcbremporthash(struct inpcb *inp)
1815 {
1816         struct inpcbporthead *porthash;
1817         struct inpcbport *phd;
1818
1819         if (inp->inp_phd == NULL)
1820                 return;
1821         KASSERT(inp->inp_lport != 0, ("inpcb has no lport"));
1822
1823         porthash = inp->inp_porthash;
1824         KASSERT(porthash != NULL, ("no porthash"));
1825
1826         GET_PORTHASH_TOKEN(porthash);
1827
1828         phd = inp->inp_phd;
1829         LIST_REMOVE(inp, inp_portlist);
1830         if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
1831                 LIST_REMOVE(phd, phd_hash);
1832                 kfree(phd, M_PCB);
1833         }
1834
1835         REL_PORTHASH_TOKEN(porthash);
1836
1837         inp->inp_phd = NULL;
1838         /* NOTE: Don't whack inp_lport, which may be used later */
1839 }
1840
1841 static struct inp_localgroup *
1842 inp_localgroup_alloc(u_char af, uint16_t port,
1843     const union in_dependaddr *addr, int size)
1844 {
1845         struct inp_localgroup *grp;
1846
1847         grp = kmalloc(__offsetof(struct inp_localgroup, il_inp[size]),
1848             M_TEMP, M_INTWAIT | M_ZERO);
1849         grp->il_af = af;
1850         grp->il_lport = port;
1851         grp->il_dependladdr = *addr;
1852         grp->il_inpsiz = size;
1853
1854         return grp;
1855 }
1856
1857 static void
1858 inp_localgroup_free(struct inp_localgroup *grp)
1859 {
1860         kfree(grp, M_TEMP);
1861 }
1862
1863 static void
1864 inp_localgroup_destroy(struct inp_localgroup *grp)
1865 {
1866         LIST_REMOVE(grp, il_list);
1867         inp_localgroup_free(grp);
1868 }
1869
1870 static void
1871 inp_localgroup_copy(struct inp_localgroup *grp,
1872     const struct inp_localgroup *old_grp)
1873 {
1874         int i;
1875
1876         KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
1877             ("invalid new local group size %d and old local group count %d",
1878              grp->il_inpsiz, old_grp->il_inpcnt));
1879         for (i = 0; i < old_grp->il_inpcnt; ++i)
1880                 grp->il_inp[i] = old_grp->il_inp[i];
1881         grp->il_inpcnt = old_grp->il_inpcnt;
1882 }
1883
1884 static void
1885 in_pcbinslocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
1886 {
1887         struct inp_localgrphead *hdr;
1888         struct inp_localgroup *grp, *grp_alloc = NULL;
1889         struct ucred *cred;
1890         int i, idx;
1891
1892         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1893
1894         if (pcbinfo->localgrphashbase == NULL)
1895                 return;
1896
1897         /*
1898          * XXX don't allow jailed socket to join local group
1899          */
1900         if (inp->inp_socket != NULL)
1901                 cred = inp->inp_socket->so_cred;
1902         else
1903                 cred = NULL;
1904         if (cred != NULL && jailed(cred))
1905                 return;
1906
1907         hdr = &pcbinfo->localgrphashbase[
1908             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
1909
1910 again:
1911         LIST_FOREACH(grp, hdr, il_list) {
1912                 if (grp->il_af == inp->inp_af &&
1913                     grp->il_lport == inp->inp_lport &&
1914                     memcmp(&grp->il_dependladdr,
1915                         &inp->inp_inc.inc_ie.ie_dependladdr,
1916                         sizeof(grp->il_dependladdr)) == 0) {
1917                         break;
1918                 }
1919         }
1920         if (grp == NULL) {
1921                 /*
1922                  * Create a new local group
1923                  */
1924                 if (grp_alloc == NULL) {
1925                         grp_alloc = inp_localgroup_alloc(inp->inp_af,
1926                             inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
1927                             INP_LOCALGROUP_SIZMIN);
1928                         /*
1929                          * Local group allocation could block and the
1930                          * local group w/ the same property might have
1931                          * been added by others when we were blocked;
1932                          * check again.
1933                          */
1934                         goto again;
1935                 } else {
1936                         /* Local group has been allocated; link it */
1937                         grp = grp_alloc;
1938                         grp_alloc = NULL;
1939                         LIST_INSERT_HEAD(hdr, grp, il_list);
1940                 }
1941         } else if (grp->il_inpcnt == grp->il_inpsiz) {
1942                 if (grp->il_inpsiz >= INP_LOCALGROUP_SIZMAX) {
1943                         static int limit_logged = 0;
1944
1945                         if (!limit_logged) {
1946                                 limit_logged = 1;
1947                                 kprintf("local group port %d, "
1948                                     "limit reached\n", ntohs(grp->il_lport));
1949                         }
1950                         if (grp_alloc != NULL) {
1951                                 /*
1952                                  * This would happen if the local group
1953                                  * w/ the same property was expanded when
1954                                  * our local group allocation blocked.
1955                                  */
1956                                 inp_localgroup_free(grp_alloc);
1957                         }
1958                         return;
1959                 }
1960
1961                 /*
1962                  * Expand this local group
1963                  */
1964                 if (grp_alloc == NULL ||
1965                     grp->il_inpcnt >= grp_alloc->il_inpsiz) {
1966                         if (grp_alloc != NULL)
1967                                 inp_localgroup_free(grp_alloc);
1968                         grp_alloc = inp_localgroup_alloc(grp->il_af,
1969                             grp->il_lport, &grp->il_dependladdr,
1970                             grp->il_inpsiz * 2);
1971                         /*
1972                          * Local group allocation could block and the
1973                          * local group w/ the same property might have
1974                          * been expanded by others when we were blocked;
1975                          * check again.
1976                          */
1977                         goto again;
1978                 }
1979
1980                 /*
1981                  * Save the old local group, link the new one, and then
1982                  * destroy the old local group
1983                  */
1984                 inp_localgroup_copy(grp_alloc, grp);
1985                 LIST_INSERT_HEAD(hdr, grp_alloc, il_list);
1986                 inp_localgroup_destroy(grp);
1987
1988                 grp = grp_alloc;
1989                 grp_alloc = NULL;
1990         } else {
1991                 /*
1992                  * Found the local group
1993                  */
1994                 if (grp_alloc != NULL) {
1995                         /*
1996                          * This would happen if the local group w/ the
1997                          * same property was added or expanded when our
1998                          * local group allocation blocked.
1999                          */
2000                         inp_localgroup_free(grp_alloc);
2001                         grp_alloc = NULL;
2002                 }
2003         }
2004
2005         KASSERT(grp->il_inpcnt < grp->il_inpsiz,
2006             ("invalid local group size %d and count %d",
2007              grp->il_inpsiz, grp->il_inpcnt));
2008
2009         /*
2010          * Keep the local group sorted by the inpcb local group index
2011          * in ascending order.
2012          *
2013          * This eases the multi-process userland application which uses
2014          * SO_REUSEPORT sockets and binds process to the owner cpu of
2015          * the SO_REUSEPORT socket:
2016          * If we didn't sort the local group by the inpcb local group
2017          * index and one of the process owning an inpcb in this local
2018          * group restarted, e.g. crashed and restarted by watchdog,
2019          * other processes owning a inpcb in this local group would have
2020          * to detect that event, refetch its socket's owner cpu, and
2021          * re-bind.
2022          */
2023         idx = grp->il_inpcnt;
2024         for (i = 0; i < idx; ++i) {
2025                 struct inpcb *oinp = grp->il_inp[i];
2026
2027                 if (oinp->inp_lgrpindex > i) {
2028                         if (inp->inp_lgrpindex < 0) {
2029                                 inp->inp_lgrpindex = i;
2030                         } else if (inp->inp_lgrpindex != i) {
2031                                 if (bootverbose) {
2032                                         kprintf("inp %p: grpidx %d, "
2033                                             "assigned to %d, cpu%d\n",
2034                                             inp, inp->inp_lgrpindex, i,
2035                                             mycpuid);
2036                                 }
2037                         }
2038                         grp->il_inp[i] = inp;
2039
2040                         /* Pull down inpcbs */
2041                         for (; i < grp->il_inpcnt; ++i) {
2042                                 struct inpcb *oinp1 = grp->il_inp[i + 1];
2043
2044                                 grp->il_inp[i + 1] = oinp;
2045                                 oinp = oinp1;
2046                         }
2047                         grp->il_inpcnt++;
2048                         return;
2049                 }
2050         }
2051
2052         if (inp->inp_lgrpindex < 0) {
2053                 inp->inp_lgrpindex = idx;
2054         } else if (inp->inp_lgrpindex != idx) {
2055                 if (bootverbose) {
2056                         kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n",
2057                             inp, inp->inp_lgrpindex, idx, mycpuid);
2058                 }
2059         }
2060         grp->il_inp[idx] = inp;
2061         grp->il_inpcnt++;
2062 }
2063
2064 void
2065 in_pcbinswildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2066 {
2067         struct inpcontainer *ic;
2068         struct inpcontainerhead *bucket;
2069
2070         GET_PCBINFO_TOKEN(pcbinfo);
2071
2072         in_pcbinslocalgrphash_oncpu(inp, pcbinfo);
2073
2074         bucket = &pcbinfo->wildcardhashbase[
2075             INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
2076
2077         ic = kmalloc(sizeof(struct inpcontainer), M_TEMP, M_INTWAIT);
2078         ic->ic_inp = inp;
2079         LIST_INSERT_HEAD(bucket, ic, ic_list);
2080
2081         REL_PCBINFO_TOKEN(pcbinfo);
2082 }
2083
2084 /*
2085  * Insert PCB into wildcard hash table.
2086  */
2087 void
2088 in_pcbinswildcardhash(struct inpcb *inp)
2089 {
2090         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2091
2092         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
2093             ("not in correct netisr"));
2094         ASSERT_INP_NOTINHASH(inp);
2095         inp->inp_flags |= INP_WILDCARD;
2096
2097         in_pcbinswildcardhash_oncpu(inp, pcbinfo);
2098 }
2099
2100 static void
2101 in_pcbremlocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2102 {
2103         struct inp_localgrphead *hdr;
2104         struct inp_localgroup *grp;
2105
2106         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
2107
2108         if (pcbinfo->localgrphashbase == NULL)
2109                 return;
2110
2111         hdr = &pcbinfo->localgrphashbase[
2112             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
2113
2114         LIST_FOREACH(grp, hdr, il_list) {
2115                 int i;
2116
2117                 for (i = 0; i < grp->il_inpcnt; ++i) {
2118                         if (grp->il_inp[i] != inp)
2119                                 continue;
2120
2121                         if (grp->il_inpcnt == 1) {
2122                                 /* Destroy this local group */
2123                                 inp_localgroup_destroy(grp);
2124                         } else {
2125                                 /* Pull up inpcbs */
2126                                 for (; i + 1 < grp->il_inpcnt; ++i)
2127                                         grp->il_inp[i] = grp->il_inp[i + 1];
2128                                 grp->il_inpcnt--;
2129                         }
2130                         return;
2131                 }
2132         }
2133 }
2134
2135 void
2136 in_pcbremwildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2137 {
2138         struct inpcontainer *ic;
2139         struct inpcontainerhead *head;
2140
2141         GET_PCBINFO_TOKEN(pcbinfo);
2142
2143         in_pcbremlocalgrphash_oncpu(inp, pcbinfo);
2144
2145         /* find bucket */
2146         head = &pcbinfo->wildcardhashbase[
2147             INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
2148
2149         LIST_FOREACH(ic, head, ic_list) {
2150                 if (ic->ic_inp == inp)
2151                         goto found;
2152         }
2153         REL_PCBINFO_TOKEN(pcbinfo);
2154         return;                 /* not found! */
2155
2156 found:
2157         LIST_REMOVE(ic, ic_list);       /* remove container from bucket chain */
2158         REL_PCBINFO_TOKEN(pcbinfo);
2159         kfree(ic, M_TEMP);              /* deallocate container */
2160 }
2161
2162 /*
2163  * Remove PCB from wildcard hash table.
2164  */
2165 void
2166 in_pcbremwildcardhash(struct inpcb *inp)
2167 {
2168         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2169
2170         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
2171             ("not in correct netisr"));
2172         KASSERT(inp->inp_flags & INP_WILDCARD, ("inp not wildcard"));
2173
2174         in_pcbremwildcardhash_oncpu(inp, pcbinfo);
2175         inp->inp_lgrpindex = -1;
2176         inp->inp_flags &= ~INP_WILDCARD;
2177 }
2178
2179 /*
2180  * Remove PCB from various lists.
2181  */
2182 void
2183 in_pcbremlists(struct inpcb *inp)
2184 {
2185         in_pcbremporthash(inp);
2186         if (inp->inp_flags & INP_WILDCARD) {
2187                 in_pcbremwildcardhash(inp);
2188         } else if (inp->inp_flags & INP_CONNECTED) {
2189                 in_pcbremconnhash(inp);
2190         }
2191
2192         if (inp->inp_flags & INP_ONLIST)
2193                 in_pcbofflist(inp);
2194 }
2195
2196 int
2197 prison_xinpcb(struct thread *td, struct inpcb *inp)
2198 {
2199         struct ucred *cr;
2200
2201         if (td->td_proc == NULL)
2202                 return (0);
2203         cr = td->td_proc->p_ucred;
2204         if (cr->cr_prison == NULL)
2205                 return (0);
2206         if (inp->inp_socket && inp->inp_socket->so_cred &&
2207             inp->inp_socket->so_cred->cr_prison &&
2208             cr->cr_prison == inp->inp_socket->so_cred->cr_prison)
2209                 return (0);
2210         return (1);
2211 }
2212
2213 int
2214 in_pcblist_range(SYSCTL_HANDLER_ARGS)
2215 {
2216         struct inpcbinfo *pcbinfo_arr = arg1;
2217         int pcbinfo_arrlen = arg2;
2218         struct inpcb *marker;
2219         int cpu, origcpu;
2220         int error, n;
2221
2222         KASSERT(pcbinfo_arrlen <= ncpus && pcbinfo_arrlen >= 1,
2223             ("invalid pcbinfo count %d", pcbinfo_arrlen));
2224
2225         /*
2226          * The process of preparing the TCB list is too time-consuming and
2227          * resource-intensive to repeat twice on every request.
2228          */
2229         n = 0;
2230         if (req->oldptr == NULL) {
2231                 for (cpu = 0; cpu < pcbinfo_arrlen; ++cpu)
2232                         n += pcbinfo_arr[cpu].ipi_count;
2233                 req->oldidx = (n + n/8 + 10) * sizeof(struct xinpcb);
2234                 return 0;
2235         }
2236
2237         if (req->newptr != NULL)
2238                 return EPERM;
2239
2240         marker = kmalloc(sizeof(struct inpcb), M_TEMP, M_WAITOK|M_ZERO);
2241         marker->inp_flags |= INP_PLACEMARKER;
2242
2243         /*
2244          * OK, now we're committed to doing something.  Re-fetch ipi_count
2245          * after obtaining the generation count.
2246          */
2247         error = 0;
2248         origcpu = mycpuid;
2249         for (cpu = 0; cpu < pcbinfo_arrlen && error == 0; ++cpu) {
2250                 struct inpcbinfo *pcbinfo = &pcbinfo_arr[cpu];
2251                 struct inpcb *inp;
2252                 struct xinpcb xi;
2253                 int i;
2254
2255                 lwkt_migratecpu(cpu);
2256
2257                 GET_PCBINFO_TOKEN(pcbinfo);
2258
2259                 n = pcbinfo->ipi_count;
2260
2261                 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
2262                 i = 0;
2263                 while ((inp = LIST_NEXT(marker, inp_list)) != NULL && i < n) {
2264                         LIST_REMOVE(marker, inp_list);
2265                         LIST_INSERT_AFTER(inp, marker, inp_list);
2266
2267                         if (inp->inp_flags & INP_PLACEMARKER)
2268                                 continue;
2269                         if (prison_xinpcb(req->td, inp))
2270                                 continue;
2271
2272                         bzero(&xi, sizeof xi);
2273                         xi.xi_len = sizeof xi;
2274                         bcopy(inp, &xi.xi_inp, sizeof *inp);
2275                         if (inp->inp_socket)
2276                                 sotoxsocket(inp->inp_socket, &xi.xi_socket);
2277                         if ((error = SYSCTL_OUT(req, &xi, sizeof xi)) != 0)
2278                                 break;
2279                         ++i;
2280                 }
2281                 LIST_REMOVE(marker, inp_list);
2282
2283                 REL_PCBINFO_TOKEN(pcbinfo);
2284
2285                 if (error == 0 && i < n) {
2286                         bzero(&xi, sizeof xi);
2287                         xi.xi_len = sizeof xi;
2288                         while (i < n) {
2289                                 error = SYSCTL_OUT(req, &xi, sizeof xi);
2290                                 if (error)
2291                                         break;
2292                                 ++i;
2293                         }
2294                 }
2295         }
2296
2297         lwkt_migratecpu(origcpu);
2298         kfree(marker, M_TEMP);
2299         return error;
2300 }
2301
2302 int
2303 in_pcblist_ncpus(SYSCTL_HANDLER_ARGS)
2304 {
2305
2306         return (in_pcblist_range(oidp, arg1, netisr_ncpus, req));
2307 }
2308
2309 void
2310 in_savefaddr(struct socket *so, const struct sockaddr *faddr)
2311 {
2312         struct sockaddr_in *sin;
2313
2314         KASSERT(faddr->sa_family == AF_INET,
2315             ("not AF_INET faddr %d", faddr->sa_family));
2316
2317         sin = kmalloc(sizeof(*sin), M_SONAME, M_WAITOK | M_ZERO);
2318         sin->sin_family = AF_INET;
2319         sin->sin_len = sizeof(*sin);
2320         sin->sin_port = ((const struct sockaddr_in *)faddr)->sin_port;
2321         sin->sin_addr = ((const struct sockaddr_in *)faddr)->sin_addr;
2322
2323         so->so_faddr = (struct sockaddr *)sin;
2324 }
2325
2326 void
2327 in_pcbportinfo_init(struct inpcbportinfo *portinfo, int hashsize,
2328     u_short offset)
2329 {
2330         memset(portinfo, 0, sizeof(*portinfo));
2331
2332         portinfo->offset = offset;
2333         portinfo->lastport = offset;
2334         portinfo->lastlow = offset;
2335         portinfo->lasthi = offset;
2336
2337         portinfo->porthashbase = hashinit(hashsize, M_PCB,
2338             &portinfo->porthashmask);
2339 }
2340
2341 void
2342 in_pcbportrange(u_short *hi0, u_short *lo0, u_short ofs, u_short step)
2343 {
2344         int hi, lo;
2345
2346         if (step == 1)
2347                 return;
2348
2349         hi = *hi0;
2350         lo = *lo0;
2351
2352         hi = rounddown(hi, step);
2353         hi += ofs;
2354         if (hi > (int)*hi0)
2355                 hi -= step;
2356
2357         lo = roundup(lo, step);
2358         lo -= (step - ofs);
2359         if (lo < (int)*lo0)
2360                 lo += step;
2361
2362         *hi0 = hi;
2363         *lo0 = lo;
2364 }
2365
2366 void
2367 in_pcbglobalinit(void)
2368 {
2369         int cpu;
2370
2371         in_pcbmarkers = kmalloc(ncpus * sizeof(struct inpcb), M_PCB,
2372             M_WAITOK | M_ZERO);
2373         in_pcbcontainer_markers = kmalloc(ncpus * sizeof(struct inpcontainer),
2374             M_PCB, M_WAITOK | M_ZERO);
2375
2376         for (cpu = 0; cpu < ncpus; ++cpu) {
2377                 struct inpcontainer *ic = &in_pcbcontainer_markers[cpu];
2378                 struct inpcb *marker = &in_pcbmarkers[cpu];
2379
2380                 marker->inp_flags |= INP_PLACEMARKER;
2381                 ic->ic_inp = marker;
2382         }
2383 }
2384
2385 struct inpcb *
2386 in_pcbmarker(int cpuid)
2387 {
2388         KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid));
2389         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
2390
2391         return &in_pcbmarkers[cpuid];
2392 }
2393
2394 struct inpcontainer *
2395 in_pcbcontainer_marker(int cpuid)
2396 {
2397         KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid));
2398         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
2399
2400         return &in_pcbcontainer_markers[cpuid];
2401 }
2402
2403 void
2404 in_pcbresetroute(struct inpcb *inp)
2405 {
2406         struct route *ro = &inp->inp_route;
2407
2408         if (ro->ro_rt != NULL)
2409                 RTFREE(ro->ro_rt);
2410         bzero(ro, sizeof(*ro));
2411 }
2412
2413 u_short
2414 in_pcblastport_down(volatile u_short *lastport, u_short first, u_short last,
2415     u_short step)
2416 {
2417         u_short lport;
2418
2419         for (;;) {
2420                 u_short olport;
2421
2422                 olport = *lastport;
2423                 lport = olport - step;
2424                 if (__predict_false(lport > first || lport < last))
2425                         lport = first;
2426                 if (atomic_cmpset_short(lastport, olport, lport))
2427                         break;
2428         }
2429         return lport;
2430 }
2431
2432 u_short
2433 in_pcblastport_up(volatile u_short *lastport, u_short first, u_short last,
2434     u_short step)
2435 {
2436         u_short lport;
2437
2438         for (;;) {
2439                 u_short olport;
2440
2441                 olport = *lastport;
2442                 lport = olport + step;
2443                 if (__predict_false(lport < first || lport > last))
2444                         lport = first;
2445                 if (atomic_cmpset_short(lastport, olport, lport))
2446                         break;
2447         }
2448         return lport;
2449 }