sys/netinet/in_pcb.c

   1 /*
   2  * Copyright (c) 2004 Jeffrey M. Hsu.  All rights reserved.
   3  * Copyright (c) 2004 The DragonFly Project.  All rights reserved.
   4  *
   5  * This code is derived from software contributed to The DragonFly Project
   6  * by Jeffrey M. Hsu.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the name of The DragonFly Project nor the names of its
  17  *    contributors may be used to endorse or promote products derived
  18  *    from this software without specific, prior written permission.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  23  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  24  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  25  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  26  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  27  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  28  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  29  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  30  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  */
  33
  34 /*
  35  * Copyright (c) 1982, 1986, 1991, 1993, 1995
  36  *      The Regents of the University of California.  All rights reserved.
  37  *
  38  * Redistribution and use in source and binary forms, with or without
  39  * modification, are permitted provided that the following conditions
  40  * are met:
  41  * 1. Redistributions of source code must retain the above copyright
  42  *    notice, this list of conditions and the following disclaimer.
  43  * 2. Redistributions in binary form must reproduce the above copyright
  44  *    notice, this list of conditions and the following disclaimer in the
  45  *    documentation and/or other materials provided with the distribution.
  46  * 3. Neither the name of the University nor the names of its contributors
  47  *    may be used to endorse or promote products derived from this software
  48  *    without specific prior written permission.
  49  *
  50  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  51  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  52  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  53  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  54  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  55  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  56  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  57  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  58  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  59  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  60  * SUCH DAMAGE.
  61  *
  62  *      @(#)in_pcb.c    8.4 (Berkeley) 5/24/95
  63  * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $
  64  */
  65
  66 #include "opt_ipsec.h"
  67 #include "opt_inet6.h"
  68
  69 #include <sys/param.h>
  70 #include <sys/systm.h>
  71 #include <sys/malloc.h>
  72 #include <sys/mbuf.h>
  73 #include <sys/domain.h>
  74 #include <sys/protosw.h>
  75 #include <sys/socket.h>
  76 #include <sys/socketvar.h>
  77 #include <sys/proc.h>
  78 #include <sys/priv.h>
  79 #include <sys/jail.h>
  80 #include <sys/kernel.h>
  81 #include <sys/sysctl.h>
  82
  83 #include <sys/thread2.h>
  84 #include <sys/socketvar2.h>
  85 #include <sys/msgport2.h>
  86
  87 #include <machine/limits.h>
  88
  89 #include <net/if.h>
  90 #include <net/if_types.h>
  91 #include <net/route.h>
  92 #include <net/netisr2.h>
  93
  94 #include <netinet/in.h>
  95 #include <netinet/in_pcb.h>
  96 #include <netinet/in_var.h>
  97 #include <netinet/ip_var.h>
  98 #ifdef INET6
  99 #include <netinet/ip6.h>
 100 #include <netinet6/ip6_var.h>
 101 #endif /* INET6 */
 102
 103 #ifdef IPSEC
 104 #include <netinet6/ipsec.h>
 105 #include <netproto/key/key.h>
 106 #include <netproto/ipsec/esp_var.h>
 107 #endif
 108
 109 #ifdef FAST_IPSEC
 110 #if defined(IPSEC) || defined(IPSEC_ESP)
 111 #error "Bad idea: don't compile with both IPSEC and FAST_IPSEC!"
 112 #endif
 113
 114 #include <netproto/ipsec/ipsec.h>
 115 #include <netproto/ipsec/key.h>
 116 #define IPSEC
 117 #endif /* FAST_IPSEC */
 118
 119 #define INP_LOCALGROUP_SIZMIN   8
 120 #define INP_LOCALGROUP_SIZMAX   256
 121
 122 struct in_addr zeroin_addr;
 123
 124 /*
 125  * These configure the range of local port addresses assigned to
 126  * "unspecified" outgoing connections/packets/whatever.
 127  */
 128 int ipport_lowfirstauto = IPPORT_RESERVED - 1;  /* 1023 */
 129 int ipport_lowlastauto = IPPORT_RESERVEDSTART;  /* 600 */
 130
 131 int ipport_firstauto = IPPORT_RESERVED;         /* 1024 */
 132 int ipport_lastauto = IPPORT_USERRESERVED;      /* 5000 */
 133
 134 int ipport_hifirstauto = IPPORT_HIFIRSTAUTO;    /* 49152 */
 135 int ipport_hilastauto = IPPORT_HILASTAUTO;      /* 65535 */
 136
 137 #define RANGECHK(var, min, max) \
 138         if ((var) < (min)) { (var) = (min); } \
 139         else if ((var) > (max)) { (var) = (max); }
 140
 141 int udpencap_enable = 1;        /* enabled by default */
 142 int udpencap_port = 4500;       /* triggers decapsulation */
 143
 144 /*
 145  * Per-netisr inpcb markers.
 146  * NOTE: they should only be used in netisrs.
 147  */
 148 static struct inpcb             *in_pcbmarkers;
 149 static struct inpcontainer      *in_pcbcontainer_markers;
 150
 151 static int
 152 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
 153 {
 154         int error;
 155
 156         error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
 157         if (!error) {
 158                 RANGECHK(ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
 159                 RANGECHK(ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
 160
 161                 RANGECHK(ipport_firstauto, IPPORT_RESERVED, USHRT_MAX);
 162                 RANGECHK(ipport_lastauto, IPPORT_RESERVED, USHRT_MAX);
 163
 164                 RANGECHK(ipport_hifirstauto, IPPORT_RESERVED, USHRT_MAX);
 165                 RANGECHK(ipport_hilastauto, IPPORT_RESERVED, USHRT_MAX);
 166         }
 167         return (error);
 168 }
 169
 170 #undef RANGECHK
 171
 172 SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports");
 173
 174 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW,
 175            &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", "");
 176 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, CTLTYPE_INT|CTLFLAG_RW,
 177            &ipport_lowlastauto, 0, &sysctl_net_ipport_check, "I", "");
 178 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, CTLTYPE_INT|CTLFLAG_RW,
 179            &ipport_firstauto, 0, &sysctl_net_ipport_check, "I", "");
 180 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, CTLTYPE_INT|CTLFLAG_RW,
 181            &ipport_lastauto, 0, &sysctl_net_ipport_check, "I", "");
 182 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, CTLTYPE_INT|CTLFLAG_RW,
 183            &ipport_hifirstauto, 0, &sysctl_net_ipport_check, "I", "");
 184 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW,
 185            &ipport_hilastauto, 0, &sysctl_net_ipport_check, "I", "");
 186
 187 /*
 188  * in_pcb.c: manage the Protocol Control Blocks.
 189  *
 190  * NOTE: It is assumed that most of these functions will be called from
 191  * a critical section.  XXX - There are, unfortunately, a few exceptions
 192  * to this rule that should be fixed.
 193  *
 194  * NOTE: The caller should initialize the cpu field to the cpu running the
 195  * protocol stack associated with this inpcbinfo.
 196  */
 197
 198 void
 199 in_pcbinfo_init(struct inpcbinfo *pcbinfo, int cpu, boolean_t shared)
 200 {
 201         KASSERT(cpu >= 0 && cpu < ncpus, ("invalid cpu%d", cpu));
 202         pcbinfo->cpu = cpu;
 203
 204         LIST_INIT(&pcbinfo->pcblisthead);
 205         pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), M_PCB,
 206                                     M_WAITOK | M_ZERO);
 207
 208         if (shared) {
 209                 pcbinfo->infotoken = kmalloc(sizeof(struct lwkt_token),
 210                     M_PCB, M_WAITOK);
 211                 lwkt_token_init(pcbinfo->infotoken, "infotoken");
 212         } else {
 213                 pcbinfo->infotoken = NULL;
 214         }
 215 }
 216
 217 struct baddynamicports baddynamicports;
 218
 219 /*
 220  * Check if the specified port is invalid for dynamic allocation.
 221  */
 222 int
 223 in_baddynamic(u_int16_t port, u_int16_t proto)
 224 {
 225         switch (proto) {
 226         case IPPROTO_TCP:
 227                 return (DP_ISSET(baddynamicports.tcp, port));
 228         case IPPROTO_UDP:
 229 #ifdef IPSEC
 230                 /* Cannot preset this as it is a sysctl */
 231                 if (port == udpencap_port)
 232                         return (1);
 233 #endif
 234                 return (DP_ISSET(baddynamicports.udp, port));
 235         default:
 236                 return (0);
 237         }
 238 }
 239
 240 void
 241 in_pcbonlist(struct inpcb *inp)
 242 {
 243         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 244
 245         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
 246             ("not in the correct netisr"));
 247         KASSERT((inp->inp_flags & INP_ONLIST) == 0, ("already on pcblist"));
 248         inp->inp_flags |= INP_ONLIST;
 249
 250         GET_PCBINFO_TOKEN(pcbinfo);
 251         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list);
 252         pcbinfo->ipi_count++;
 253         REL_PCBINFO_TOKEN(pcbinfo);
 254 }
 255
 256 void
 257 in_pcbofflist(struct inpcb *inp)
 258 {
 259         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 260
 261         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
 262             ("not in the correct netisr"));
 263         KASSERT(inp->inp_flags & INP_ONLIST, ("not on pcblist"));
 264         inp->inp_flags &= ~INP_ONLIST;
 265
 266         GET_PCBINFO_TOKEN(pcbinfo);
 267         LIST_REMOVE(inp, inp_list);
 268         KASSERT(pcbinfo->ipi_count > 0,
 269             ("invalid inpcb count %d", pcbinfo->ipi_count));
 270         pcbinfo->ipi_count--;
 271         REL_PCBINFO_TOKEN(pcbinfo);
 272 }
 273
 274 /*
 275  * Allocate a PCB and associate it with the socket.
 276  */
 277 int
 278 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
 279 {
 280         struct inpcb *inp;
 281 #ifdef IPSEC
 282         int error;
 283 #endif
 284
 285         inp = kmalloc(pcbinfo->ipi_size, M_PCB, M_WAITOK|M_ZERO|M_NULLOK);
 286         if (inp == NULL)
 287                 return (ENOMEM);
 288         inp->inp_lgrpindex = -1;
 289         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
 290         inp->inp_pcbinfo = pcbinfo;
 291         inp->inp_socket = so;
 292 #ifdef IPSEC
 293         error = ipsec_init_policy(so, &inp->inp_sp);
 294         if (error != 0) {
 295                 kfree(inp, M_PCB);
 296                 return (error);
 297         }
 298 #endif
 299 #ifdef INET6
 300         if (INP_SOCKAF(so) == AF_INET6 && ip6_v6only)
 301                 inp->inp_flags |= IN6P_IPV6_V6ONLY;
 302         if (ip6_auto_flowlabel)
 303                 inp->inp_flags |= IN6P_AUTOFLOWLABEL;
 304 #endif
 305         soreference(so);
 306         so->so_pcb = inp;
 307
 308         in_pcbonlist(inp);
 309         return (0);
 310 }
 311
 312 /*
 313  * Unlink a pcb with the intention of moving it to another cpu with a
 314  * different pcbinfo.  While unlinked nothing should attempt to dereference
 315  * inp_pcbinfo, NULL it out so we assert if it does.
 316  */
 317 void
 318 in_pcbunlink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags)
 319 {
 320         KASSERT(inp->inp_pcbinfo == pcbinfo, ("pcbinfo mismatch"));
 321         KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0,
 322             ("already linked"));
 323
 324         in_pcbofflist(inp);
 325         inp->inp_pcbinfo = NULL;
 326 }
 327
 328 void
 329 in_pcbunlink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
 330 {
 331         in_pcbunlink_flags(inp, pcbinfo, INP_WILDCARD);
 332 }
 333
 334 /*
 335  * Relink a pcb into a new pcbinfo.
 336  */
 337 void
 338 in_pcblink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags)
 339 {
 340         KASSERT(inp->inp_pcbinfo == NULL, ("has pcbinfo"));
 341         KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0,
 342             ("already linked"));
 343
 344         inp->inp_pcbinfo = pcbinfo;
 345         in_pcbonlist(inp);
 346 }
 347
 348 void
 349 in_pcblink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
 350 {
 351         return in_pcblink_flags(inp, pcbinfo, INP_WILDCARD);
 352 }
 353
 354 static int
 355 in_pcbsetlport(struct inpcb *inp, int wild, struct ucred *cred)
 356 {
 357         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 358         struct inpcbportinfo *portinfo;
 359         u_short first, last, lport, step;
 360         u_short *lastport;
 361         int count, error;
 362         int portinfo_first, portinfo_idx;
 363
 364         inp->inp_flags |= INP_ANONPORT;
 365
 366         step = pcbinfo->portinfo_mask + 1;
 367         portinfo_first = mycpuid & pcbinfo->portinfo_mask;
 368         portinfo_idx = portinfo_first;
 369 loop:
 370         portinfo = &pcbinfo->portinfo[portinfo_idx];
 371
 372         if (inp->inp_flags & INP_HIGHPORT) {
 373                 first = ipport_hifirstauto;     /* sysctl */
 374                 last  = ipport_hilastauto;
 375                 lastport = &portinfo->lasthi;
 376         } else if (inp->inp_flags & INP_LOWPORT) {
 377                 if (cred &&
 378                     (error =
 379                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 380                         inp->inp_laddr.s_addr = INADDR_ANY;
 381                         return error;
 382                 }
 383                 first = ipport_lowfirstauto;    /* 1023 */
 384                 last  = ipport_lowlastauto;     /* 600 */
 385                 lastport = &portinfo->lastlow;
 386         } else {
 387                 first = ipport_firstauto;       /* sysctl */
 388                 last  = ipport_lastauto;
 389                 lastport = &portinfo->lastport;
 390         }
 391
 392         /*
 393          * This has to be atomic.  If the porthash is shared across multiple
 394          * protocol threads (aka tcp) then the token must be held.
 395          */
 396         GET_PORT_TOKEN(portinfo);
 397
 398         /*
 399          * Simple check to ensure all ports are not used up causing
 400          * a deadlock here.
 401          *
 402          * We split the two cases (up and down) so that the direction
 403          * is not being tested on each round of the loop.
 404          */
 405         if (first > last) {
 406                 /*
 407                  * counting down
 408                  */
 409                 in_pcbportrange(&first, &last, portinfo->offset, step);
 410                 count = (first - last) / step;
 411
 412                 do {
 413                         if (count-- < 0) {      /* completely used? */
 414                                 error = EADDRNOTAVAIL;
 415                                 goto done;
 416                         }
 417                         *lastport -= step;
 418                         if (*lastport > first || *lastport < last)
 419                                 *lastport = first;
 420                         KKASSERT((*lastport & pcbinfo->portinfo_mask) ==
 421                             portinfo->offset);
 422                         lport = htons(*lastport);
 423                 } while (in_pcblookup_local(portinfo, inp->inp_laddr, lport,
 424                     wild, cred));
 425         } else {
 426                 /*
 427                  * counting up
 428                  */
 429                 in_pcbportrange(&last, &first, portinfo->offset, step);
 430                 count = (last - first) / step;
 431
 432                 do {
 433                         if (count-- < 0) {      /* completely used? */
 434                                 error = EADDRNOTAVAIL;
 435                                 goto done;
 436                         }
 437                         *lastport += step;
 438                         if (*lastport < first || *lastport > last)
 439                                 *lastport = first;
 440                         KKASSERT((*lastport & pcbinfo->portinfo_mask) ==
 441                             portinfo->offset);
 442                         lport = htons(*lastport);
 443                 } while (in_pcblookup_local(portinfo, inp->inp_laddr, lport,
 444                     wild, cred));
 445         }
 446         inp->inp_lport = lport;
 447         in_pcbinsporthash(portinfo, inp);
 448         error = 0;
 449 done:
 450         REL_PORT_TOKEN(portinfo);
 451
 452         if (error) {
 453                 /* Try next portinfo */
 454                 portinfo_idx++;
 455                 portinfo_idx &= pcbinfo->portinfo_mask;
 456                 if (portinfo_idx != portinfo_first)
 457                         goto loop;
 458                 inp->inp_laddr.s_addr = INADDR_ANY;
 459         }
 460         return error;
 461 }
 462
 463 int
 464 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
 465 {
 466         struct socket *so = inp->inp_socket;
 467         struct sockaddr_in jsin;
 468         struct ucred *cred = NULL;
 469         int wild = 0;
 470
 471         if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */
 472                 return (EADDRNOTAVAIL);
 473         if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
 474                 return (EINVAL);        /* already bound */
 475
 476         if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT)))
 477                 wild = 1;    /* neither SO_REUSEADDR nor SO_REUSEPORT is set */
 478         if (td->td_proc)
 479                 cred = td->td_proc->p_ucred;
 480
 481         if (nam != NULL) {
 482                 struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 483                 struct inpcbinfo *pcbinfo;
 484                 struct inpcbportinfo *portinfo;
 485                 struct inpcb *t;
 486                 u_short lport, lport_ho;
 487                 int reuseport = (so->so_options & SO_REUSEPORT);
 488                 int error;
 489
 490                 if (nam->sa_len != sizeof *sin)
 491                         return (EINVAL);
 492 #ifdef notdef
 493                 /*
 494                  * We should check the family, but old programs
 495                  * incorrectly fail to initialize it.
 496                  */
 497                 if (sin->sin_family != AF_INET)
 498                         return (EAFNOSUPPORT);
 499 #endif
 500                 if (!prison_replace_wildcards(td, nam))
 501                         return (EINVAL);
 502
 503                 lport = sin->sin_port;
 504                 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 505                         /*
 506                          * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
 507                          * allow complete duplication of binding if
 508                          * SO_REUSEPORT is set, or if SO_REUSEADDR is set
 509                          * and a multicast address is bound on both
 510                          * new and duplicated sockets.
 511                          */
 512                         if (so->so_options & SO_REUSEADDR)
 513                                 reuseport = SO_REUSEADDR | SO_REUSEPORT;
 514                 } else if (sin->sin_addr.s_addr != INADDR_ANY) {
 515                         sin->sin_port = 0;              /* yech... */
 516                         bzero(&sin->sin_zero, sizeof sin->sin_zero);
 517                         if (ifa_ifwithaddr((struct sockaddr *)sin) == NULL)
 518                                 return (EADDRNOTAVAIL);
 519                 }
 520
 521                 inp->inp_laddr = sin->sin_addr;
 522
 523                 jsin.sin_family = AF_INET;
 524                 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 525                 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 526                         inp->inp_laddr.s_addr = INADDR_ANY;
 527                         return (EINVAL);
 528                 }
 529                 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 530
 531                 if (lport == 0) {
 532                         /* Auto-select local port */
 533                         return in_pcbsetlport(inp, wild, cred);
 534                 }
 535                 lport_ho = ntohs(lport);
 536
 537                 /* GROSS */
 538                 if (lport_ho < IPPORT_RESERVED && cred &&
 539                     (error =
 540                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 541                         inp->inp_laddr.s_addr = INADDR_ANY;
 542                         return (error);
 543                 }
 544
 545                 /*
 546                  * Locate the proper portinfo based on lport
 547                  */
 548                 pcbinfo = inp->inp_pcbinfo;
 549                 portinfo =
 550                     &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask];
 551                 KKASSERT((lport_ho & pcbinfo->portinfo_mask) ==
 552                     portinfo->offset);
 553
 554                 /*
 555                  * This has to be atomic.  If the porthash is shared across
 556                  * multiple protocol threads (aka tcp) then the token must
 557                  * be held.
 558                  */
 559                 GET_PORT_TOKEN(portinfo);
 560
 561                 if (so->so_cred->cr_uid != 0 &&
 562                     !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 563                         t = in_pcblookup_local(portinfo, sin->sin_addr, lport,
 564                             INPLOOKUP_WILDCARD, cred);
 565                         if (t &&
 566                             (!in_nullhost(sin->sin_addr) ||
 567                              !in_nullhost(t->inp_laddr) ||
 568                              (t->inp_socket->so_options & SO_REUSEPORT) == 0) &&
 569                             (so->so_cred->cr_uid !=
 570                              t->inp_socket->so_cred->cr_uid)) {
 571 #ifdef INET6
 572                                 if (!in_nullhost(sin->sin_addr) ||
 573                                     !in_nullhost(t->inp_laddr) ||
 574                                     INP_SOCKAF(so) == INP_SOCKAF(t->inp_socket))
 575 #endif
 576                                 {
 577                                         inp->inp_laddr.s_addr = INADDR_ANY;
 578                                         error = EADDRINUSE;
 579                                         goto done;
 580                                 }
 581                         }
 582                 }
 583                 if (cred && !prison_replace_wildcards(td, nam)) {
 584                         inp->inp_laddr.s_addr = INADDR_ANY;
 585                         error = EADDRNOTAVAIL;
 586                         goto done;
 587                 }
 588                 t = in_pcblookup_local(portinfo, sin->sin_addr, lport,
 589                     wild, cred);
 590                 if (t && !(reuseport & t->inp_socket->so_options)) {
 591 #ifdef INET6
 592                         if (!in_nullhost(sin->sin_addr) ||
 593                             !in_nullhost(t->inp_laddr) ||
 594                             INP_SOCKAF(so) == INP_SOCKAF(t->inp_socket))
 595 #endif
 596                         {
 597                                 inp->inp_laddr.s_addr = INADDR_ANY;
 598                                 error = EADDRINUSE;
 599                                 goto done;
 600                         }
 601                 }
 602                 inp->inp_lport = lport;
 603                 in_pcbinsporthash(portinfo, inp);
 604                 error = 0;
 605 done:
 606                 REL_PORT_TOKEN(portinfo);
 607                 return (error);
 608         } else {
 609                 jsin.sin_family = AF_INET;
 610                 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 611                 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 612                         inp->inp_laddr.s_addr = INADDR_ANY;
 613                         return (EINVAL);
 614                 }
 615                 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 616
 617                 return in_pcbsetlport(inp, wild, cred);
 618         }
 619 }
 620
 621 static struct inpcb *
 622 in_pcblookup_localremote(struct inpcbportinfo *portinfo, struct in_addr laddr,
 623     u_short lport, struct in_addr faddr, u_short fport, struct ucred *cred)
 624 {
 625         struct inpcb *inp;
 626         struct inpcbporthead *porthash;
 627         struct inpcbport *phd;
 628         struct inpcb *match = NULL;
 629
 630         /*
 631          * If the porthashbase is shared across several cpus, it must
 632          * have been locked.
 633          */
 634         ASSERT_PORT_TOKEN_HELD(portinfo);
 635
 636         /*
 637          * Best fit PCB lookup.
 638          *
 639          * First see if this local port is in use by looking on the
 640          * port hash list.
 641          */
 642         porthash = &portinfo->porthashbase[
 643                         INP_PCBPORTHASH(lport, portinfo->porthashmask)];
 644         LIST_FOREACH(phd, porthash, phd_hash) {
 645                 if (phd->phd_port == lport)
 646                         break;
 647         }
 648         if (phd != NULL) {
 649                 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
 650 #ifdef INET6
 651                         if ((inp->inp_vflag & INP_IPV4) == 0)
 652                                 continue;
 653 #endif
 654                         if (inp->inp_laddr.s_addr != INADDR_ANY &&
 655                             inp->inp_laddr.s_addr != laddr.s_addr)
 656                                 continue;
 657
 658                         if (inp->inp_faddr.s_addr != INADDR_ANY &&
 659                             inp->inp_faddr.s_addr != faddr.s_addr)
 660                                 continue;
 661
 662                         if (inp->inp_fport != 0 && inp->inp_fport != fport)
 663                                 continue;
 664
 665                         if (cred == NULL ||
 666                             cred->cr_prison ==
 667                             inp->inp_socket->so_cred->cr_prison) {
 668                                 match = inp;
 669                                 break;
 670                         }
 671                 }
 672         }
 673         return (match);
 674 }
 675
 676 int
 677 in_pcbbind_remote(struct inpcb *inp, const struct sockaddr *remote,
 678     struct thread *td)
 679 {
 680         struct proc *p = td->td_proc;
 681         unsigned short *lastport;
 682         const struct sockaddr_in *sin = (const struct sockaddr_in *)remote;
 683         struct sockaddr_in jsin;
 684         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 685         struct inpcbportinfo *portinfo;
 686         struct ucred *cred = NULL;
 687         u_short first, last, lport, step;
 688         int count, error, dup;
 689         int portinfo_first, portinfo_idx;
 690
 691         if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */
 692                 return (EADDRNOTAVAIL);
 693
 694         KKASSERT(inp->inp_laddr.s_addr != INADDR_ANY);
 695         if (inp->inp_lport != 0)
 696                 return (EINVAL);        /* already bound */
 697
 698         KKASSERT(p);
 699         cred = p->p_ucred;
 700
 701         jsin.sin_family = AF_INET;
 702         jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 703         if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 704                 inp->inp_laddr.s_addr = INADDR_ANY;
 705                 return (EINVAL);
 706         }
 707         inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 708
 709         inp->inp_flags |= INP_ANONPORT;
 710
 711         step = pcbinfo->portinfo_mask + 1;
 712         portinfo_first = mycpuid & pcbinfo->portinfo_mask;
 713         portinfo_idx = portinfo_first;
 714 loop:
 715         portinfo = &pcbinfo->portinfo[portinfo_idx];
 716         dup = 0;
 717
 718         if (inp->inp_flags & INP_HIGHPORT) {
 719                 first = ipport_hifirstauto;     /* sysctl */
 720                 last  = ipport_hilastauto;
 721                 lastport = &portinfo->lasthi;
 722         } else if (inp->inp_flags & INP_LOWPORT) {
 723                 if (cred &&
 724                     (error =
 725                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 726                         inp->inp_laddr.s_addr = INADDR_ANY;
 727                         return (error);
 728                 }
 729                 first = ipport_lowfirstauto;    /* 1023 */
 730                 last  = ipport_lowlastauto;     /* 600 */
 731                 lastport = &portinfo->lastlow;
 732         } else {
 733                 first = ipport_firstauto;       /* sysctl */
 734                 last  = ipport_lastauto;
 735                 lastport = &portinfo->lastport;
 736         }
 737
 738         /*
 739          * This has to be atomic.  If the porthash is shared across multiple
 740          * protocol threads (aka tcp) then the token must be held.
 741          */
 742         GET_PORT_TOKEN(portinfo);
 743
 744 again:
 745         /*
 746          * Simple check to ensure all ports are not used up causing
 747          * a deadlock here.
 748          *
 749          * We split the two cases (up and down) so that the direction
 750          * is not being tested on each round of the loop.
 751          */
 752         if (first > last) {
 753                 /*
 754                  * counting down
 755                  */
 756                 in_pcbportrange(&first, &last, portinfo->offset, step);
 757                 count = (first - last) / step;
 758
 759                 do {
 760                         if (count-- < 0) {      /* completely used? */
 761                                 error = EADDRNOTAVAIL;
 762                                 goto done;
 763                         }
 764                         *lastport -= step;
 765                         if (*lastport > first || *lastport < last)
 766                                 *lastport = first;
 767                         KKASSERT((*lastport & pcbinfo->portinfo_mask) ==
 768                             portinfo->offset);
 769                         lport = htons(*lastport);
 770                 } while (in_pcblookup_localremote(portinfo, inp->inp_laddr,
 771                     lport, sin->sin_addr, sin->sin_port, cred));
 772         } else {
 773                 /*
 774                  * counting up
 775                  */
 776                 in_pcbportrange(&last, &first, portinfo->offset, step);
 777                 count = (last - first) / step;
 778
 779                 do {
 780                         if (count-- < 0) {      /* completely used? */
 781                                 error = EADDRNOTAVAIL;
 782                                 goto done;
 783                         }
 784                         *lastport += step;
 785                         if (*lastport < first || *lastport > last)
 786                                 *lastport = first;
 787                         KKASSERT((*lastport & pcbinfo->portinfo_mask) ==
 788                             portinfo->offset);
 789                         lport = htons(*lastport);
 790                 } while (in_pcblookup_localremote(portinfo, inp->inp_laddr,
 791                     lport, sin->sin_addr, sin->sin_port, cred));
 792         }
 793
 794         /* This could happen on loopback interface */
 795         if (sin->sin_port == lport &&
 796             sin->sin_addr.s_addr == inp->inp_laddr.s_addr) {
 797                 if (dup) {
 798                         /*
 799                          * Duplicate again; give up
 800                          */
 801                         error = EADDRNOTAVAIL;
 802                         goto done;
 803                 }
 804                 dup = 1;
 805                 goto again;
 806         }
 807         inp->inp_lport = lport;
 808         in_pcbinsporthash(portinfo, inp);
 809         error = 0;
 810 done:
 811         REL_PORT_TOKEN(portinfo);
 812
 813         if (error) {
 814                 /* Try next portinfo */
 815                 portinfo_idx++;
 816                 portinfo_idx &= pcbinfo->portinfo_mask;
 817                 if (portinfo_idx != portinfo_first)
 818                         goto loop;
 819                 inp->inp_laddr.s_addr = INADDR_ANY;
 820         }
 821         return error;
 822 }
 823
 824 /*
 825  *   Transform old in_pcbconnect() into an inner subroutine for new
 826  *   in_pcbconnect(): Do some validity-checking on the remote
 827  *   address (in mbuf 'nam') and then determine local host address
 828  *   (i.e., which interface) to use to access that remote host.
 829  *
 830  *   This preserves definition of in_pcbconnect(), while supporting a
 831  *   slightly different version for T/TCP.  (This is more than
 832  *   a bit of a kludge, but cleaning up the internal interfaces would
 833  *   have forced minor changes in every protocol).
 834  */
 835 int
 836 in_pcbladdr_find(struct inpcb *inp, struct sockaddr *nam,
 837     struct sockaddr_in **plocal_sin, struct thread *td, int find)
 838 {
 839         struct in_ifaddr *ia;
 840         struct ucred *cred = NULL;
 841         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 842         struct sockaddr *jsin;
 843         int jailed = 0, alloc_route = 0;
 844
 845         if (nam->sa_len != sizeof *sin)
 846                 return (EINVAL);
 847         if (sin->sin_family != AF_INET)
 848                 return (EAFNOSUPPORT);
 849         if (sin->sin_port == 0)
 850                 return (EADDRNOTAVAIL);
 851         if (td && td->td_proc && td->td_proc->p_ucred)
 852                 cred = td->td_proc->p_ucred;
 853         if (cred && cred->cr_prison)
 854                 jailed = 1;
 855         if (!TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) {
 856                 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia;
 857                 /*
 858                  * If the destination address is INADDR_ANY,
 859                  * use the primary local address.
 860                  * If the supplied address is INADDR_BROADCAST,
 861                  * and the primary interface supports broadcast,
 862                  * choose the broadcast address for that interface.
 863                  */
 864                 if (sin->sin_addr.s_addr == INADDR_ANY)
 865                         sin->sin_addr = IA_SIN(ia)->sin_addr;
 866                 else if (sin->sin_addr.s_addr == (u_long)INADDR_BROADCAST &&
 867                     (ia->ia_ifp->if_flags & IFF_BROADCAST))
 868                         sin->sin_addr = satosin(&ia->ia_broadaddr)->sin_addr;
 869         }
 870         if (find) {
 871                 struct route *ro;
 872
 873                 ia = NULL;
 874                 /*
 875                  * If route is known or can be allocated now,
 876                  * our src addr is taken from the i/f, else punt.
 877                  * Note that we should check the address family of the cached
 878                  * destination, in case of sharing the cache with IPv6.
 879                  */
 880                 ro = &inp->inp_route;
 881                 if (ro->ro_rt &&
 882                     (!(ro->ro_rt->rt_flags & RTF_UP) ||
 883                      ro->ro_dst.sa_family != AF_INET ||
 884                      satosin(&ro->ro_dst)->sin_addr.s_addr !=
 885                                       sin->sin_addr.s_addr ||
 886                      inp->inp_socket->so_options & SO_DONTROUTE)) {
 887                         RTFREE(ro->ro_rt);
 888                         ro->ro_rt = NULL;
 889                 }
 890                 if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/
 891                     (ro->ro_rt == NULL ||
 892                     ro->ro_rt->rt_ifp == NULL)) {
 893                         /* No route yet, so try to acquire one */
 894                         bzero(&ro->ro_dst, sizeof(struct sockaddr_in));
 895                         ro->ro_dst.sa_family = AF_INET;
 896                         ro->ro_dst.sa_len = sizeof(struct sockaddr_in);
 897                         ((struct sockaddr_in *) &ro->ro_dst)->sin_addr =
 898                                 sin->sin_addr;
 899                         rtalloc(ro);
 900                         alloc_route = 1;
 901                 }
 902                 /*
 903                  * If we found a route, use the address
 904                  * corresponding to the outgoing interface
 905                  * unless it is the loopback (in case a route
 906                  * to our address on another net goes to loopback).
 907                  */
 908                 if (ro->ro_rt &&
 909                     !(ro->ro_rt->rt_ifp->if_flags & IFF_LOOPBACK)) {
 910                         if (jailed) {
 911                                 if (jailed_ip(cred->cr_prison,
 912                                     ro->ro_rt->rt_ifa->ifa_addr)) {
 913                                         ia = ifatoia(ro->ro_rt->rt_ifa);
 914                                 }
 915                         } else {
 916                                 ia = ifatoia(ro->ro_rt->rt_ifa);
 917                         }
 918                 }
 919                 if (ia == NULL) {
 920                         u_short fport = sin->sin_port;
 921
 922                         sin->sin_port = 0;
 923                         ia = ifatoia(ifa_ifwithdstaddr(sintosa(sin)));
 924                         if (ia && jailed && !jailed_ip(cred->cr_prison,
 925                             sintosa(&ia->ia_addr)))
 926                                 ia = NULL;
 927                         if (ia == NULL)
 928                                 ia = ifatoia(ifa_ifwithnet(sintosa(sin)));
 929                         if (ia && jailed && !jailed_ip(cred->cr_prison,
 930                             sintosa(&ia->ia_addr)))
 931                                 ia = NULL;
 932                         sin->sin_port = fport;
 933                         if (ia == NULL &&
 934                             !TAILQ_EMPTY(&in_ifaddrheads[mycpuid]))
 935                                 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia;
 936                         if (ia && jailed && !jailed_ip(cred->cr_prison,
 937                             sintosa(&ia->ia_addr)))
 938                                 ia = NULL;
 939
 940                         if (!jailed && ia == NULL)
 941                                 goto fail;
 942                 }
 943                 /*
 944                  * If the destination address is multicast and an outgoing
 945                  * interface has been set as a multicast option, use the
 946                  * address of that interface as our source address.
 947                  */
 948                 if (!jailed && IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
 949                     inp->inp_moptions != NULL) {
 950                         struct ip_moptions *imo;
 951                         struct ifnet *ifp;
 952
 953                         imo = inp->inp_moptions;
 954                         if (imo->imo_multicast_ifp != NULL) {
 955                                 struct in_ifaddr_container *iac;
 956
 957                                 ifp = imo->imo_multicast_ifp;
 958                                 ia = NULL;
 959                                 TAILQ_FOREACH(iac,
 960                                 &in_ifaddrheads[mycpuid], ia_link) {
 961                                         if (iac->ia->ia_ifp == ifp) {
 962                                                 ia = iac->ia;
 963                                                 break;
 964                                         }
 965                                 }
 966                                 if (ia == NULL)
 967                                         goto fail;
 968                         }
 969                 }
 970                 /*
 971                  * Don't do pcblookup call here; return interface in plocal_sin
 972                  * and exit to caller, that will do the lookup.
 973                  */
 974                 if (ia == NULL && jailed) {
 975                         if ((jsin = prison_get_nonlocal(
 976                                 cred->cr_prison, AF_INET, NULL)) != NULL ||
 977                             (jsin = prison_get_local(
 978                                 cred->cr_prison, AF_INET, NULL)) != NULL) {
 979                                 *plocal_sin = satosin(jsin);
 980                         } else {
 981                                 /* IPv6 only Jail */
 982                                 goto fail;
 983                         }
 984                 } else {
 985                         *plocal_sin = &ia->ia_addr;
 986                 }
 987         }
 988         return (0);
 989 fail:
 990         if (alloc_route) {
 991                 struct route *ro = &inp->inp_route;
 992
 993                 if (ro->ro_rt != NULL)
 994                         RTFREE(ro->ro_rt);
 995                 bzero(ro, sizeof(*ro));
 996         }
 997         return (EADDRNOTAVAIL);
 998 }
 999
1000 int
1001 in_pcbladdr(struct inpcb *inp, struct sockaddr *nam,
1002     struct sockaddr_in **plocal_sin, struct thread *td)
1003 {
1004         return in_pcbladdr_find(inp, nam, plocal_sin, td,
1005             (inp->inp_laddr.s_addr == INADDR_ANY));
1006 }
1007
1008 /*
1009  * Outer subroutine:
1010  * Connect from a socket to a specified address.
1011  * Both address and port must be specified in argument sin.
1012  * If don't have a local address for this socket yet,
1013  * then pick one.
1014  */
1015 int
1016 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
1017 {
1018         struct sockaddr_in *if_sin;
1019         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
1020         int error;
1021
1022         /* Call inner routine to assign local interface address. */
1023         if ((error = in_pcbladdr(inp, nam, &if_sin, td)) != 0)
1024                 return (error);
1025
1026         if (in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port,
1027                               inp->inp_laddr.s_addr ?
1028                                 inp->inp_laddr : if_sin->sin_addr,
1029                               inp->inp_lport, FALSE, NULL) != NULL) {
1030                 return (EADDRINUSE);
1031         }
1032         if (inp->inp_laddr.s_addr == INADDR_ANY) {
1033                 if (inp->inp_lport == 0) {
1034                         error = in_pcbbind(inp, NULL, td);
1035                         if (error)
1036                                 return (error);
1037                 }
1038                 inp->inp_laddr = if_sin->sin_addr;
1039         }
1040         inp->inp_faddr = sin->sin_addr;
1041         inp->inp_fport = sin->sin_port;
1042         in_pcbinsconnhash(inp);
1043         return (0);
1044 }
1045
1046 void
1047 in_pcbdisconnect(struct inpcb *inp)
1048 {
1049
1050         in_pcbremconnhash(inp);
1051         inp->inp_faddr.s_addr = INADDR_ANY;
1052         inp->inp_fport = 0;
1053 }
1054
1055 void
1056 in_pcbdetach(struct inpcb *inp)
1057 {
1058         struct socket *so = inp->inp_socket;
1059         struct inpcbinfo *ipi = inp->inp_pcbinfo;
1060
1061 #ifdef IPSEC
1062         ipsec4_delete_pcbpolicy(inp);
1063 #endif /*IPSEC*/
1064         inp->inp_gencnt = ++ipi->ipi_gencnt;
1065         KKASSERT((so->so_state & SS_ASSERTINPROG) == 0);
1066         in_pcbremlists(inp);
1067         so->so_pcb = NULL;
1068         sofree(so);                     /* remove pcb ref */
1069         if (inp->inp_options)
1070                 m_free(inp->inp_options);
1071         if (inp->inp_route.ro_rt)
1072                 rtfree(inp->inp_route.ro_rt);
1073         ip_freemoptions(inp->inp_moptions);
1074         inp->inp_vflag = 0;
1075         kfree(inp, M_PCB);
1076 }
1077
1078 /*
1079  * The calling convention of in_setsockaddr() and in_setpeeraddr() was
1080  * modified to match the pru_sockaddr() and pru_peeraddr() entry points
1081  * in struct pr_usrreqs, so that protocols can just reference then directly
1082  * without the need for a wrapper function.  The socket must have a valid
1083  * (i.e., non-nil) PCB, but it should be impossible to get an invalid one
1084  * except through a kernel programming error, so it is acceptable to panic
1085  * (or in this case trap) if the PCB is invalid.  (Actually, we don't trap
1086  * because there actually /is/ a programming error somewhere... XXX)
1087  */
1088 int
1089 in_setsockaddr(struct socket *so, struct sockaddr **nam)
1090 {
1091         struct inpcb *inp;
1092         struct sockaddr_in *sin;
1093
1094         /*
1095          * Do the malloc first in case it blocks.
1096          */
1097         sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1098         sin->sin_family = AF_INET;
1099         sin->sin_len = sizeof *sin;
1100
1101         crit_enter();
1102         inp = so->so_pcb;
1103         if (!inp) {
1104                 crit_exit();
1105                 kfree(sin, M_SONAME);
1106                 return (ECONNRESET);
1107         }
1108         sin->sin_port = inp->inp_lport;
1109         sin->sin_addr = inp->inp_laddr;
1110         crit_exit();
1111
1112         *nam = (struct sockaddr *)sin;
1113         return (0);
1114 }
1115
1116 void
1117 in_setsockaddr_dispatch(netmsg_t msg)
1118 {
1119         int error;
1120
1121         error = in_setsockaddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1122         lwkt_replymsg(&msg->lmsg, error);
1123 }
1124
1125 int
1126 in_setpeeraddr(struct socket *so, struct sockaddr **nam)
1127 {
1128         struct inpcb *inp;
1129         struct sockaddr_in *sin;
1130
1131         /*
1132          * Do the malloc first in case it blocks.
1133          */
1134         sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1135         sin->sin_family = AF_INET;
1136         sin->sin_len = sizeof *sin;
1137
1138         crit_enter();
1139         inp = so->so_pcb;
1140         if (!inp) {
1141                 crit_exit();
1142                 kfree(sin, M_SONAME);
1143                 return (ECONNRESET);
1144         }
1145         sin->sin_port = inp->inp_fport;
1146         sin->sin_addr = inp->inp_faddr;
1147         crit_exit();
1148
1149         *nam = (struct sockaddr *)sin;
1150         return (0);
1151 }
1152
1153 void
1154 in_setpeeraddr_dispatch(netmsg_t msg)
1155 {
1156         int error;
1157
1158         error = in_setpeeraddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1159         lwkt_replymsg(&msg->lmsg, error);
1160 }
1161
1162 void
1163 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int err,
1164                 void (*notify)(struct inpcb *, int))
1165 {
1166         struct inpcb *inp, *marker;
1167
1168         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1169             ("not in the correct netisr"));
1170         marker = &in_pcbmarkers[mycpuid];
1171
1172         /*
1173          * NOTE:
1174          * - If INP_PLACEMARKER is set we must ignore the rest of the
1175          *   structure and skip it.
1176          * - It is safe to nuke inpcbs here, since we are in their own
1177          *   netisr.
1178          */
1179         GET_PCBINFO_TOKEN(pcbinfo);
1180
1181         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1182         while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1183                 LIST_REMOVE(marker, inp_list);
1184                 LIST_INSERT_AFTER(inp, marker, inp_list);
1185
1186                 if (inp->inp_flags & INP_PLACEMARKER)
1187                         continue;
1188 #ifdef INET6
1189                 if (!(inp->inp_vflag & INP_IPV4))
1190                         continue;
1191 #endif
1192                 if (inp->inp_faddr.s_addr != faddr.s_addr ||
1193                     inp->inp_socket == NULL)
1194                         continue;
1195                 (*notify)(inp, err);            /* can remove inp from list! */
1196         }
1197         LIST_REMOVE(marker, inp_list);
1198
1199         REL_PCBINFO_TOKEN(pcbinfo);
1200 }
1201
1202 void
1203 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1204 {
1205         struct inpcb *inp, *marker;
1206
1207         /*
1208          * We only need to make sure that we are in netisr0, where all
1209          * multicast operation happen.  We could check inpcbinfo which
1210          * does not belong to netisr0 by holding the inpcbinfo's token.
1211          * In this case, the pcbinfo must be able to be shared, i.e.
1212          * pcbinfo->infotoken is not NULL.
1213          */
1214         KASSERT(&curthread->td_msgport == netisr_cpuport(0),
1215             ("not in netisr0"));
1216         KASSERT(pcbinfo->cpu == 0 || pcbinfo->infotoken != NULL,
1217             ("pcbinfo could not be shared"));
1218
1219         /*
1220          * Get a marker for the current netisr (netisr0).
1221          *
1222          * It is possible that the multicast address deletion blocks,
1223          * which could cause temporary token releasing.  So we use
1224          * inpcb marker here to get a coherent view of the inpcb list.
1225          *
1226          * While, on the other hand, moptions are only added and deleted
1227          * in netisr0, so we would not see staled moption or miss moption
1228          * even if the token was released due to the blocking multicast
1229          * address deletion.
1230          */
1231         marker = &in_pcbmarkers[mycpuid];
1232
1233         GET_PCBINFO_TOKEN(pcbinfo);
1234
1235         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1236         while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1237                 struct ip_moptions *imo;
1238
1239                 LIST_REMOVE(marker, inp_list);
1240                 LIST_INSERT_AFTER(inp, marker, inp_list);
1241
1242                 if (inp->inp_flags & INP_PLACEMARKER)
1243                         continue;
1244                 imo = inp->inp_moptions;
1245                 if ((inp->inp_vflag & INP_IPV4) && imo != NULL) {
1246                         int i, gap;
1247
1248                         /*
1249                          * Unselect the outgoing interface if it is being
1250                          * detached.
1251                          */
1252                         if (imo->imo_multicast_ifp == ifp)
1253                                 imo->imo_multicast_ifp = NULL;
1254
1255                         /*
1256                          * Drop multicast group membership if we joined
1257                          * through the interface being detached.
1258                          */
1259                         for (i = 0, gap = 0; i < imo->imo_num_memberships;
1260                             i++) {
1261                                 if (imo->imo_membership[i]->inm_ifp == ifp) {
1262                                         /*
1263                                          * NOTE:
1264                                          * This could block and the pcbinfo
1265                                          * token could be passively released.
1266                                          */
1267                                         in_delmulti(imo->imo_membership[i]);
1268                                         gap++;
1269                                 } else if (gap != 0)
1270                                         imo->imo_membership[i - gap] =
1271                                             imo->imo_membership[i];
1272                         }
1273                         imo->imo_num_memberships -= gap;
1274                 }
1275         }
1276         LIST_REMOVE(marker, inp_list);
1277
1278         REL_PCBINFO_TOKEN(pcbinfo);
1279 }
1280
1281 /*
1282  * Check for alternatives when higher level complains
1283  * about service problems.  For now, invalidate cached
1284  * routing information.  If the route was created dynamically
1285  * (by a redirect), time to try a default gateway again.
1286  */
1287 void
1288 in_losing(struct inpcb *inp)
1289 {
1290         struct rtentry *rt;
1291         struct rt_addrinfo rtinfo;
1292
1293         if ((rt = inp->inp_route.ro_rt)) {
1294                 bzero(&rtinfo, sizeof(struct rt_addrinfo));
1295                 rtinfo.rti_info[RTAX_DST] = rt_key(rt);
1296                 rtinfo.rti_info[RTAX_GATEWAY] = rt->rt_gateway;
1297                 rtinfo.rti_info[RTAX_NETMASK] = rt_mask(rt);
1298                 rtinfo.rti_flags = rt->rt_flags;
1299                 rt_missmsg(RTM_LOSING, &rtinfo, rt->rt_flags, 0);
1300                 if (rt->rt_flags & RTF_DYNAMIC) {
1301                         rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway,
1302                             rt_mask(rt), rt->rt_flags, NULL);
1303                 }
1304                 inp->inp_route.ro_rt = NULL;
1305                 rtfree(rt);
1306                 /*
1307                  * A new route can be allocated
1308                  * the next time output is attempted.
1309                  */
1310         }
1311 }
1312
1313 /*
1314  * After a routing change, flush old routing
1315  * and allocate a (hopefully) better one.
1316  */
1317 void
1318 in_rtchange(struct inpcb *inp, int err)
1319 {
1320         if (inp->inp_route.ro_rt) {
1321                 rtfree(inp->inp_route.ro_rt);
1322                 inp->inp_route.ro_rt = NULL;
1323                 /*
1324                  * A new route can be allocated the next time
1325                  * output is attempted.
1326                  */
1327         }
1328 }
1329
1330 /*
1331  * Lookup a PCB based on the local address and port.
1332  */
1333 struct inpcb *
1334 in_pcblookup_local(struct inpcbportinfo *portinfo, struct in_addr laddr,
1335                    u_int lport_arg, int wild_okay, struct ucred *cred)
1336 {
1337         struct inpcb *inp;
1338         int matchwild = 3, wildcard;
1339         u_short lport = lport_arg;
1340         struct inpcbporthead *porthash;
1341         struct inpcbport *phd;
1342         struct inpcb *match = NULL;
1343
1344         /*
1345          * If the porthashbase is shared across several cpus, it must
1346          * have been locked.
1347          */
1348         ASSERT_PORT_TOKEN_HELD(portinfo);
1349
1350         /*
1351          * Best fit PCB lookup.
1352          *
1353          * First see if this local port is in use by looking on the
1354          * port hash list.
1355          */
1356         porthash = &portinfo->porthashbase[
1357                         INP_PCBPORTHASH(lport, portinfo->porthashmask)];
1358         LIST_FOREACH(phd, porthash, phd_hash) {
1359                 if (phd->phd_port == lport)
1360                         break;
1361         }
1362         if (phd != NULL) {
1363                 /*
1364                  * Port is in use by one or more PCBs. Look for best
1365                  * fit.
1366                  */
1367                 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
1368                         wildcard = 0;
1369 #ifdef INET6
1370                         if ((inp->inp_vflag & INP_IPV4) == 0)
1371                                 continue;
1372 #endif
1373                         if (inp->inp_faddr.s_addr != INADDR_ANY)
1374                                 wildcard++;
1375                         if (inp->inp_laddr.s_addr != INADDR_ANY) {
1376                                 if (laddr.s_addr == INADDR_ANY)
1377                                         wildcard++;
1378                                 else if (inp->inp_laddr.s_addr != laddr.s_addr)
1379                                         continue;
1380                         } else {
1381                                 if (laddr.s_addr != INADDR_ANY)
1382                                         wildcard++;
1383                         }
1384                         if (wildcard && !wild_okay)
1385                                 continue;
1386                         if (wildcard < matchwild &&
1387                             (cred == NULL ||
1388                              cred->cr_prison ==
1389                                         inp->inp_socket->so_cred->cr_prison)) {
1390                                 match = inp;
1391                                 matchwild = wildcard;
1392                                 if (matchwild == 0) {
1393                                         break;
1394                                 }
1395                         }
1396                 }
1397         }
1398         return (match);
1399 }
1400
1401 struct inpcb *
1402 in_pcblocalgroup_last(const struct inpcbinfo *pcbinfo,
1403     const struct inpcb *inp)
1404 {
1405         const struct inp_localgrphead *hdr;
1406         const struct inp_localgroup *grp;
1407         int i;
1408
1409         if (pcbinfo->localgrphashbase == NULL)
1410                 return NULL;
1411
1412         GET_PCBINFO_TOKEN(pcbinfo);
1413
1414         hdr = &pcbinfo->localgrphashbase[
1415             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
1416
1417         LIST_FOREACH(grp, hdr, il_list) {
1418                 if (grp->il_vflag == inp->inp_vflag &&
1419                     grp->il_lport == inp->inp_lport &&
1420                     memcmp(&grp->il_dependladdr,
1421                         &inp->inp_inc.inc_ie.ie_dependladdr,
1422                         sizeof(grp->il_dependladdr)) == 0) {
1423                         break;
1424                 }
1425         }
1426         if (grp == NULL || grp->il_inpcnt == 1) {
1427                 REL_PCBINFO_TOKEN(pcbinfo);
1428                 return NULL;
1429         }
1430
1431         KASSERT(grp->il_inpcnt >= 2,
1432             ("invalid localgroup inp count %d", grp->il_inpcnt));
1433         for (i = 0; i < grp->il_inpcnt; ++i) {
1434                 if (grp->il_inp[i] == inp) {
1435                         int last = grp->il_inpcnt - 1;
1436
1437                         if (i == last)
1438                                 last = grp->il_inpcnt - 2;
1439                         REL_PCBINFO_TOKEN(pcbinfo);
1440                         return grp->il_inp[last];
1441                 }
1442         }
1443         REL_PCBINFO_TOKEN(pcbinfo);
1444         return NULL;
1445 }
1446
1447 static struct inpcb *
1448 inp_localgroup_lookup(const struct inpcbinfo *pcbinfo,
1449     struct in_addr laddr, uint16_t lport, uint32_t pkt_hash)
1450 {
1451         struct inpcb *local_wild = NULL;
1452         const struct inp_localgrphead *hdr;
1453         const struct inp_localgroup *grp;
1454
1455         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1456
1457         hdr = &pcbinfo->localgrphashbase[
1458             INP_PCBLOCALGRPHASH(lport, pcbinfo->localgrphashmask)];
1459
1460         /*
1461          * Order of socket selection:
1462          * 1. non-wild.
1463          * 2. wild.
1464          *
1465          * NOTE:
1466          * - Local group does not contain jailed sockets
1467          * - Local group does not contain IPv4 mapped INET6 wild sockets
1468          */
1469         LIST_FOREACH(grp, hdr, il_list) {
1470 #ifdef INET6
1471                 if (!(grp->il_vflag & INP_IPV4))
1472                         continue;
1473 #endif
1474                 if (grp->il_lport == lport) {
1475                         int idx;
1476
1477                         /*
1478                          * Modulo-N is used here, which greatly reduces
1479                          * completion queue token contention, thus more
1480                          * cpu time is saved.
1481                          */
1482                         idx = pkt_hash % grp->il_inpcnt;
1483                         if (grp->il_laddr.s_addr == laddr.s_addr)
1484                                 return grp->il_inp[idx];
1485                         else if (grp->il_laddr.s_addr == INADDR_ANY)
1486                                 local_wild = grp->il_inp[idx];
1487                 }
1488         }
1489         if (local_wild != NULL)
1490                 return local_wild;
1491         return NULL;
1492 }
1493
1494 /*
1495  * Lookup PCB in hash list.
1496  */
1497 struct inpcb *
1498 in_pcblookup_pkthash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1499     u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1500     boolean_t wildcard, struct ifnet *ifp, const struct mbuf *m)
1501 {
1502         struct inpcbhead *head;
1503         struct inpcb *inp, *jinp=NULL;
1504         u_short fport = fport_arg, lport = lport_arg;
1505
1506         /*
1507          * First look for an exact match.
1508          */
1509         head = &pcbinfo->hashbase[INP_PCBCONNHASH(faddr.s_addr, fport,
1510             laddr.s_addr, lport, pcbinfo->hashmask)];
1511         LIST_FOREACH(inp, head, inp_hash) {
1512 #ifdef INET6
1513                 if (!(inp->inp_vflag & INP_IPV4))
1514                         continue;
1515 #endif
1516                 if (in_hosteq(inp->inp_faddr, faddr) &&
1517                     in_hosteq(inp->inp_laddr, laddr) &&
1518                     inp->inp_fport == fport && inp->inp_lport == lport) {
1519                         /* found */
1520                         if (inp->inp_socket == NULL ||
1521                             inp->inp_socket->so_cred->cr_prison == NULL) {
1522                                 return (inp);
1523                         } else {
1524                                 if  (jinp == NULL)
1525                                         jinp = inp;
1526                         }
1527                 }
1528         }
1529         if (jinp != NULL)
1530                 return (jinp);
1531
1532         if (wildcard) {
1533                 struct inpcb *local_wild = NULL;
1534                 struct inpcb *jinp_wild = NULL;
1535 #ifdef INET6
1536                 struct inpcb *local_wild_mapped = NULL;
1537 #endif
1538                 struct inpcontainer *ic;
1539                 struct inpcontainerhead *chead;
1540                 struct sockaddr_in jsin;
1541                 struct ucred *cred;
1542
1543                 GET_PCBINFO_TOKEN(pcbinfo);
1544
1545                 /*
1546                  * Check local group first
1547                  */
1548                 if (pcbinfo->localgrphashbase != NULL &&
1549                     m != NULL && (m->m_flags & M_HASH) &&
1550                     !(ifp && ifp->if_type == IFT_FAITH)) {
1551                         inp = inp_localgroup_lookup(pcbinfo,
1552                             laddr, lport, m->m_pkthdr.hash);
1553                         if (inp != NULL) {
1554                                 REL_PCBINFO_TOKEN(pcbinfo);
1555                                 return inp;
1556                         }
1557                 }
1558
1559                 /*
1560                  * Order of socket selection:
1561                  * 1. non-jailed, non-wild.
1562                  * 2. non-jailed, wild.
1563                  * 3. jailed, non-wild.
1564                  * 4. jailed, wild.
1565                  */
1566                 jsin.sin_family = AF_INET;
1567                 chead = &pcbinfo->wildcardhashbase[
1568                     INP_PCBWILDCARDHASH(lport, pcbinfo->wildcardhashmask)];
1569                 LIST_FOREACH(ic, chead, ic_list) {
1570                         inp = ic->ic_inp;
1571                         if (inp->inp_flags & INP_PLACEMARKER)
1572                                 continue;
1573
1574                         jsin.sin_addr.s_addr = laddr.s_addr;
1575 #ifdef INET6
1576                         if (!(inp->inp_vflag & INP_IPV4))
1577                                 continue;
1578 #endif
1579                         if (inp->inp_socket != NULL)
1580                                 cred = inp->inp_socket->so_cred;
1581                         else
1582                                 cred = NULL;
1583                         if (cred != NULL && jailed(cred)) {
1584                                 if (jinp != NULL)
1585                                         continue;
1586                                 else
1587                                         if (!jailed_ip(cred->cr_prison,
1588                                             (struct sockaddr *)&jsin))
1589                                                 continue;
1590                         }
1591                         if (inp->inp_lport == lport) {
1592                                 if (ifp && ifp->if_type == IFT_FAITH &&
1593                                     !(inp->inp_flags & INP_FAITH))
1594                                         continue;
1595                                 if (inp->inp_laddr.s_addr == laddr.s_addr) {
1596                                         if (cred != NULL && jailed(cred)) {
1597                                                 jinp = inp;
1598                                         } else {
1599                                                 REL_PCBINFO_TOKEN(pcbinfo);
1600                                                 return (inp);
1601                                         }
1602                                 }
1603                                 if (inp->inp_laddr.s_addr == INADDR_ANY) {
1604 #ifdef INET6
1605                                         if (INP_CHECK_SOCKAF(inp->inp_socket,
1606                                                              AF_INET6))
1607                                                 local_wild_mapped = inp;
1608                                         else
1609 #endif
1610                                                 if (cred != NULL &&
1611                                                     jailed(cred))
1612                                                         jinp_wild = inp;
1613                                                 else
1614                                                         local_wild = inp;
1615                                 }
1616                         }
1617                 }
1618
1619                 REL_PCBINFO_TOKEN(pcbinfo);
1620
1621                 if (local_wild != NULL)
1622                         return (local_wild);
1623 #ifdef INET6
1624                 if (local_wild_mapped != NULL)
1625                         return (local_wild_mapped);
1626 #endif
1627                 if (jinp != NULL)
1628                         return (jinp);
1629                 return (jinp_wild);
1630         }
1631
1632         /*
1633          * Not found.
1634          */
1635         return (NULL);
1636 }
1637
1638 struct inpcb *
1639 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1640     u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1641     boolean_t wildcard, struct ifnet *ifp)
1642 {
1643         return in_pcblookup_pkthash(pcbinfo, faddr, fport_arg,
1644             laddr, lport_arg, wildcard, ifp, NULL);
1645 }
1646
1647 /*
1648  * Insert PCB into connection hash table.
1649  */
1650 void
1651 in_pcbinsconnhash(struct inpcb *inp)
1652 {
1653         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1654         struct inpcbhead *bucket;
1655         u_int32_t hashkey_faddr, hashkey_laddr;
1656
1657 #ifdef INET6
1658         if (inp->inp_vflag & INP_IPV6) {
1659                 hashkey_faddr = inp->in6p_faddr.s6_addr32[3] /* XXX JH */;
1660                 hashkey_laddr = inp->in6p_laddr.s6_addr32[3] /* XXX JH */;
1661         } else {
1662 #endif
1663                 hashkey_faddr = inp->inp_faddr.s_addr;
1664                 hashkey_laddr = inp->inp_laddr.s_addr;
1665 #ifdef INET6
1666         }
1667 #endif
1668
1669         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1670             ("not in the correct netisr"));
1671         KASSERT(!(inp->inp_flags & INP_WILDCARD), ("already on wildcardhash"));
1672         KASSERT(!(inp->inp_flags & INP_CONNECTED), ("already on connhash"));
1673         inp->inp_flags |= INP_CONNECTED;
1674
1675         /*
1676          * Insert into the connection hash table.
1677          */
1678         bucket = &pcbinfo->hashbase[INP_PCBCONNHASH(hashkey_faddr,
1679             inp->inp_fport, hashkey_laddr, inp->inp_lport, pcbinfo->hashmask)];
1680         LIST_INSERT_HEAD(bucket, inp, inp_hash);
1681 }
1682
1683 /*
1684  * Remove PCB from connection hash table.
1685  */
1686 void
1687 in_pcbremconnhash(struct inpcb *inp)
1688 {
1689         struct inpcbinfo *pcbinfo __debugvar = inp->inp_pcbinfo;
1690
1691         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1692             ("not in the correct netisr"));
1693         KASSERT(inp->inp_flags & INP_CONNECTED, ("inp not connected"));
1694
1695         LIST_REMOVE(inp, inp_hash);
1696         inp->inp_flags &= ~INP_CONNECTED;
1697 }
1698
1699 /*
1700  * Insert PCB into port hash table.
1701  */
1702 void
1703 in_pcbinsporthash(struct inpcbportinfo *portinfo, struct inpcb *inp)
1704 {
1705         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1706         struct inpcbporthead *pcbporthash;
1707         struct inpcbport *phd;
1708
1709         /*
1710          * If the porthashbase is shared across several cpus, it must
1711          * have been locked.
1712          */
1713         ASSERT_PORT_TOKEN_HELD(portinfo);
1714
1715         /*
1716          * Insert into the port hash table.
1717          */
1718         pcbporthash = &portinfo->porthashbase[
1719             INP_PCBPORTHASH(inp->inp_lport, portinfo->porthashmask)];
1720
1721         /* Go through port list and look for a head for this lport. */
1722         LIST_FOREACH(phd, pcbporthash, phd_hash) {
1723                 if (phd->phd_port == inp->inp_lport)
1724                         break;
1725         }
1726
1727         /* If none exists, use saved one and tack it on. */
1728         if (phd == NULL) {
1729                 KKASSERT(pcbinfo->portsave != NULL);
1730                 phd = pcbinfo->portsave;
1731                 pcbinfo->portsave = NULL;
1732                 phd->phd_port = inp->inp_lport;
1733                 LIST_INIT(&phd->phd_pcblist);
1734                 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
1735         }
1736
1737         inp->inp_portinfo = portinfo;
1738         inp->inp_phd = phd;
1739         LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
1740
1741         /*
1742          * Malloc one inpcbport for later use.  It is safe to use
1743          * "wait" malloc here (port token would be released, if
1744          * malloc ever blocked), since all changes to the porthash
1745          * are done.
1746          */
1747         if (pcbinfo->portsave == NULL) {
1748                 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave),
1749                                             M_PCB, M_INTWAIT | M_ZERO);
1750         }
1751 }
1752
1753 void
1754 in_pcbinsporthash_lport(struct inpcb *inp)
1755 {
1756         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1757         struct inpcbportinfo *portinfo;
1758         u_short lport_ho;
1759
1760         /* Locate the proper portinfo based on lport */
1761         lport_ho = ntohs(inp->inp_lport);
1762         portinfo = &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask];
1763         KKASSERT((lport_ho & pcbinfo->portinfo_mask) == portinfo->offset);
1764
1765         GET_PORT_TOKEN(portinfo);
1766         in_pcbinsporthash(portinfo, inp);
1767         REL_PORT_TOKEN(portinfo);
1768 }
1769
1770 static struct inp_localgroup *
1771 inp_localgroup_alloc(u_char vflag,
1772     uint16_t port, const union in_dependaddr *addr, int size)
1773 {
1774         struct inp_localgroup *grp;
1775
1776         grp = kmalloc(__offsetof(struct inp_localgroup, il_inp[size]),
1777             M_TEMP, M_INTWAIT | M_ZERO);
1778         grp->il_vflag = vflag;
1779         grp->il_lport = port;
1780         grp->il_dependladdr = *addr;
1781         grp->il_inpsiz = size;
1782
1783         return grp;
1784 }
1785
1786 static void
1787 inp_localgroup_free(struct inp_localgroup *grp)
1788 {
1789         kfree(grp, M_TEMP);
1790 }
1791
1792 static void
1793 inp_localgroup_destroy(struct inp_localgroup *grp)
1794 {
1795         LIST_REMOVE(grp, il_list);
1796         inp_localgroup_free(grp);
1797 }
1798
1799 static void
1800 inp_localgroup_copy(struct inp_localgroup *grp,
1801     const struct inp_localgroup *old_grp)
1802 {
1803         int i;
1804
1805         KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
1806             ("invalid new local group size %d and old local group count %d",
1807              grp->il_inpsiz, old_grp->il_inpcnt));
1808         for (i = 0; i < old_grp->il_inpcnt; ++i)
1809                 grp->il_inp[i] = old_grp->il_inp[i];
1810         grp->il_inpcnt = old_grp->il_inpcnt;
1811 }
1812
1813 static void
1814 in_pcbinslocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
1815 {
1816         struct inp_localgrphead *hdr;
1817         struct inp_localgroup *grp, *grp_alloc = NULL;
1818         struct ucred *cred;
1819         int i, idx;
1820
1821         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1822
1823         if (pcbinfo->localgrphashbase == NULL)
1824                 return;
1825
1826         /*
1827          * XXX don't allow jailed socket to join local group
1828          */
1829         if (inp->inp_socket != NULL)
1830                 cred = inp->inp_socket->so_cred;
1831         else
1832                 cred = NULL;
1833         if (cred != NULL && jailed(cred))
1834                 return;
1835
1836 #ifdef INET6
1837         /*
1838          * XXX don't allow IPv4 mapped INET6 wild socket
1839          */
1840         if ((inp->inp_vflag & INP_IPV4) &&
1841             inp->inp_laddr.s_addr == INADDR_ANY &&
1842             INP_CHECK_SOCKAF(inp->inp_socket, AF_INET6))
1843                 return;
1844 #endif
1845
1846         hdr = &pcbinfo->localgrphashbase[
1847             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
1848
1849 again:
1850         LIST_FOREACH(grp, hdr, il_list) {
1851                 if (grp->il_vflag == inp->inp_vflag &&
1852                     grp->il_lport == inp->inp_lport &&
1853                     memcmp(&grp->il_dependladdr,
1854                         &inp->inp_inc.inc_ie.ie_dependladdr,
1855                         sizeof(grp->il_dependladdr)) == 0) {
1856                         break;
1857                 }
1858         }
1859         if (grp == NULL) {
1860                 /*
1861                  * Create a new local group
1862                  */
1863                 if (grp_alloc == NULL) {
1864                         grp_alloc = inp_localgroup_alloc(inp->inp_vflag,
1865                             inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
1866                             INP_LOCALGROUP_SIZMIN);
1867                         /*
1868                          * Local group allocation could block and the
1869                          * local group w/ the same property might have
1870                          * been added by others when we were blocked;
1871                          * check again.
1872                          */
1873                         goto again;
1874                 } else {
1875                         /* Local group has been allocated; link it */
1876                         grp = grp_alloc;
1877                         grp_alloc = NULL;
1878                         LIST_INSERT_HEAD(hdr, grp, il_list);
1879                 }
1880         } else if (grp->il_inpcnt == grp->il_inpsiz) {
1881                 if (grp->il_inpsiz >= INP_LOCALGROUP_SIZMAX) {
1882                         static int limit_logged = 0;
1883
1884                         if (!limit_logged) {
1885                                 limit_logged = 1;
1886                                 kprintf("local group port %d, "
1887                                     "limit reached\n", ntohs(grp->il_lport));
1888                         }
1889                         if (grp_alloc != NULL) {
1890                                 /*
1891                                  * This would happen if the local group
1892                                  * w/ the same property was expanded when
1893                                  * our local group allocation blocked.
1894                                  */
1895                                 inp_localgroup_free(grp_alloc);
1896                         }
1897                         return;
1898                 }
1899
1900                 /*
1901                  * Expand this local group
1902                  */
1903                 if (grp_alloc == NULL ||
1904                     grp->il_inpcnt >= grp_alloc->il_inpsiz) {
1905                         if (grp_alloc != NULL)
1906                                 inp_localgroup_free(grp_alloc);
1907                         grp_alloc = inp_localgroup_alloc(grp->il_vflag,
1908                             grp->il_lport, &grp->il_dependladdr,
1909                             grp->il_inpsiz * 2);
1910                         /*
1911                          * Local group allocation could block and the
1912                          * local group w/ the same property might have
1913                          * been expanded by others when we were blocked;
1914                          * check again.
1915                          */
1916                         goto again;
1917                 }
1918
1919                 /*
1920                  * Save the old local group, link the new one, and then
1921                  * destroy the old local group
1922                  */
1923                 inp_localgroup_copy(grp_alloc, grp);
1924                 LIST_INSERT_HEAD(hdr, grp_alloc, il_list);
1925                 inp_localgroup_destroy(grp);
1926
1927                 grp = grp_alloc;
1928                 grp_alloc = NULL;
1929         } else {
1930                 /*
1931                  * Found the local group
1932                  */
1933                 if (grp_alloc != NULL) {
1934                         /*
1935                          * This would happen if the local group w/ the
1936                          * same property was added or expanded when our
1937                          * local group allocation blocked.
1938                          */
1939                         inp_localgroup_free(grp_alloc);
1940                         grp_alloc = NULL;
1941                 }
1942         }
1943
1944         KASSERT(grp->il_inpcnt < grp->il_inpsiz,
1945             ("invalid local group size %d and count %d",
1946              grp->il_inpsiz, grp->il_inpcnt));
1947
1948         /*
1949          * Keep the local group sorted by the inpcb local group index
1950          * in ascending order.
1951          *
1952          * This eases the multi-process userland application which uses
1953          * SO_REUSEPORT sockets and binds process to the owner cpu of
1954          * the SO_REUSEPORT socket:
1955          * If we didn't sort the local group by the inpcb local group
1956          * index and one of the process owning an inpcb in this local
1957          * group restarted, e.g. crashed and restarted by watchdog,
1958          * other processes owning a inpcb in this local group would have
1959          * to detect that event, refetch its socket's owner cpu, and
1960          * re-bind.
1961          */
1962         idx = grp->il_inpcnt;
1963         for (i = 0; i < idx; ++i) {
1964                 struct inpcb *oinp = grp->il_inp[i];
1965
1966                 if (oinp->inp_lgrpindex > i) {
1967                         if (inp->inp_lgrpindex < 0) {
1968                                 inp->inp_lgrpindex = i;
1969                         } else if (inp->inp_lgrpindex != i) {
1970                                 if (bootverbose) {
1971                                         kprintf("inp %p: grpidx %d, "
1972                                             "assigned to %d, cpu%d\n",
1973                                             inp, inp->inp_lgrpindex, i,
1974                                             mycpuid);
1975                                 }
1976                         }
1977                         grp->il_inp[i] = inp;
1978
1979                         /* Pull down inpcbs */
1980                         for (; i < grp->il_inpcnt; ++i) {
1981                                 struct inpcb *oinp1 = grp->il_inp[i + 1];
1982
1983                                 grp->il_inp[i + 1] = oinp;
1984                                 oinp = oinp1;
1985                         }
1986                         grp->il_inpcnt++;
1987                         return;
1988                 }
1989         }
1990
1991         if (inp->inp_lgrpindex < 0) {
1992                 inp->inp_lgrpindex = idx;
1993         } else if (inp->inp_lgrpindex != idx) {
1994                 if (bootverbose) {
1995                         kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n",
1996                             inp, inp->inp_lgrpindex, idx, mycpuid);
1997                 }
1998         }
1999         grp->il_inp[idx] = inp;
2000         grp->il_inpcnt++;
2001 }
2002
2003 void
2004 in_pcbinswildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2005 {
2006         struct inpcontainer *ic;
2007         struct inpcontainerhead *bucket;
2008
2009         GET_PCBINFO_TOKEN(pcbinfo);
2010
2011         in_pcbinslocalgrphash_oncpu(inp, pcbinfo);
2012
2013         bucket = &pcbinfo->wildcardhashbase[
2014             INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
2015
2016         ic = kmalloc(sizeof(struct inpcontainer), M_TEMP, M_INTWAIT);
2017         ic->ic_inp = inp;
2018         LIST_INSERT_HEAD(bucket, ic, ic_list);
2019
2020         REL_PCBINFO_TOKEN(pcbinfo);
2021 }
2022
2023 /*
2024  * Insert PCB into wildcard hash table.
2025  */
2026 void
2027 in_pcbinswildcardhash(struct inpcb *inp)
2028 {
2029         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2030
2031         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
2032             ("not in correct netisr"));
2033         KASSERT(!(inp->inp_flags & INP_CONNECTED),
2034                 ("already on connhash"));
2035         KASSERT(!(inp->inp_flags & INP_WILDCARD),
2036                 ("already on wildcardhash"));
2037         inp->inp_flags |= INP_WILDCARD;
2038
2039         in_pcbinswildcardhash_oncpu(inp, pcbinfo);
2040 }
2041
2042 static void
2043 in_pcbremlocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2044 {
2045         struct inp_localgrphead *hdr;
2046         struct inp_localgroup *grp;
2047
2048         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
2049
2050         if (pcbinfo->localgrphashbase == NULL)
2051                 return;
2052
2053         hdr = &pcbinfo->localgrphashbase[
2054             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
2055
2056         LIST_FOREACH(grp, hdr, il_list) {
2057                 int i;
2058
2059                 for (i = 0; i < grp->il_inpcnt; ++i) {
2060                         if (grp->il_inp[i] != inp)
2061                                 continue;
2062
2063                         if (grp->il_inpcnt == 1) {
2064                                 /* Destroy this local group */
2065                                 inp_localgroup_destroy(grp);
2066                         } else {
2067                                 /* Pull up inpcbs */
2068                                 for (; i + 1 < grp->il_inpcnt; ++i)
2069                                         grp->il_inp[i] = grp->il_inp[i + 1];
2070                                 grp->il_inpcnt--;
2071                         }
2072                         return;
2073                 }
2074         }
2075 }
2076
2077 void
2078 in_pcbremwildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2079 {
2080         struct inpcontainer *ic;
2081         struct inpcontainerhead *head;
2082
2083         GET_PCBINFO_TOKEN(pcbinfo);
2084
2085         in_pcbremlocalgrphash_oncpu(inp, pcbinfo);
2086
2087         /* find bucket */
2088         head = &pcbinfo->wildcardhashbase[
2089             INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
2090
2091         LIST_FOREACH(ic, head, ic_list) {
2092                 if (ic->ic_inp == inp)
2093                         goto found;
2094         }
2095         REL_PCBINFO_TOKEN(pcbinfo);
2096         return;                 /* not found! */
2097
2098 found:
2099         LIST_REMOVE(ic, ic_list);       /* remove container from bucket chain */
2100         REL_PCBINFO_TOKEN(pcbinfo);
2101         kfree(ic, M_TEMP);              /* deallocate container */
2102 }
2103
2104 /*
2105  * Remove PCB from wildcard hash table.
2106  */
2107 void
2108 in_pcbremwildcardhash(struct inpcb *inp)
2109 {
2110         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2111
2112         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
2113             ("not in correct netisr"));
2114         KASSERT(inp->inp_flags & INP_WILDCARD, ("inp not wildcard"));
2115
2116         in_pcbremwildcardhash_oncpu(inp, pcbinfo);
2117         inp->inp_lgrpindex = -1;
2118         inp->inp_flags &= ~INP_WILDCARD;
2119 }
2120
2121 /*
2122  * Remove PCB from various lists.
2123  */
2124 void
2125 in_pcbremlists(struct inpcb *inp)
2126 {
2127         if (inp->inp_lport) {
2128                 struct inpcbportinfo *portinfo;
2129                 struct inpcbport *phd;
2130
2131                 /*
2132                  * NOTE:
2133                  * inp->inp_portinfo is _not_ necessary same as
2134                  * inp->inp_pcbinfo->portinfo.
2135                  */
2136                 portinfo = inp->inp_portinfo;
2137                 GET_PORT_TOKEN(portinfo);
2138
2139                 phd = inp->inp_phd;
2140                 LIST_REMOVE(inp, inp_portlist);
2141                 if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
2142                         LIST_REMOVE(phd, phd_hash);
2143                         kfree(phd, M_PCB);
2144                 }
2145
2146                 REL_PORT_TOKEN(portinfo);
2147         }
2148         if (inp->inp_flags & INP_WILDCARD) {
2149                 in_pcbremwildcardhash(inp);
2150         } else if (inp->inp_flags & INP_CONNECTED) {
2151                 in_pcbremconnhash(inp);
2152         }
2153
2154         if (inp->inp_flags & INP_ONLIST)
2155                 in_pcbofflist(inp);
2156 }
2157
2158 int
2159 prison_xinpcb(struct thread *td, struct inpcb *inp)
2160 {
2161         struct ucred *cr;
2162
2163         if (td->td_proc == NULL)
2164                 return (0);
2165         cr = td->td_proc->p_ucred;
2166         if (cr->cr_prison == NULL)
2167                 return (0);
2168         if (inp->inp_socket && inp->inp_socket->so_cred &&
2169             inp->inp_socket->so_cred->cr_prison &&
2170             cr->cr_prison == inp->inp_socket->so_cred->cr_prison)
2171                 return (0);
2172         return (1);
2173 }
2174
2175 int
2176 in_pcblist_global(SYSCTL_HANDLER_ARGS)
2177 {
2178         struct inpcbinfo *pcbinfo_arr = arg1;
2179         int pcbinfo_arrlen = arg2;
2180         struct inpcb *marker;
2181         int cpu, origcpu;
2182         int error, n;
2183
2184         KASSERT(pcbinfo_arrlen <= ncpus && pcbinfo_arrlen >= 1,
2185             ("invalid pcbinfo count %d", pcbinfo_arrlen));
2186
2187         /*
2188          * The process of preparing the TCB list is too time-consuming and
2189          * resource-intensive to repeat twice on every request.
2190          */
2191         n = 0;
2192         if (req->oldptr == NULL) {
2193                 for (cpu = 0; cpu < pcbinfo_arrlen; ++cpu)
2194                         n += pcbinfo_arr[cpu].ipi_count;
2195                 req->oldidx = (n + n/8 + 10) * sizeof(struct xinpcb);
2196                 return 0;
2197         }
2198
2199         if (req->newptr != NULL)
2200                 return EPERM;
2201
2202         marker = kmalloc(sizeof(struct inpcb), M_TEMP, M_WAITOK|M_ZERO);
2203         marker->inp_flags |= INP_PLACEMARKER;
2204
2205         /*
2206          * OK, now we're committed to doing something.  Re-fetch ipi_count
2207          * after obtaining the generation count.
2208          */
2209         error = 0;
2210         origcpu = mycpuid;
2211         for (cpu = 0; cpu < pcbinfo_arrlen && error == 0; ++cpu) {
2212                 struct inpcbinfo *pcbinfo = &pcbinfo_arr[cpu];
2213                 struct inpcb *inp;
2214                 struct xinpcb xi;
2215                 int i;
2216
2217                 lwkt_migratecpu(cpu);
2218
2219                 GET_PCBINFO_TOKEN(pcbinfo);
2220
2221                 n = pcbinfo->ipi_count;
2222
2223                 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
2224                 i = 0;
2225                 while ((inp = LIST_NEXT(marker, inp_list)) != NULL && i < n) {
2226                         LIST_REMOVE(marker, inp_list);
2227                         LIST_INSERT_AFTER(inp, marker, inp_list);
2228
2229                         if (inp->inp_flags & INP_PLACEMARKER)
2230                                 continue;
2231                         if (prison_xinpcb(req->td, inp))
2232                                 continue;
2233
2234                         bzero(&xi, sizeof xi);
2235                         xi.xi_len = sizeof xi;
2236                         bcopy(inp, &xi.xi_inp, sizeof *inp);
2237                         if (inp->inp_socket)
2238                                 sotoxsocket(inp->inp_socket, &xi.xi_socket);
2239                         if ((error = SYSCTL_OUT(req, &xi, sizeof xi)) != 0)
2240                                 break;
2241                         ++i;
2242                 }
2243                 LIST_REMOVE(marker, inp_list);
2244
2245                 REL_PCBINFO_TOKEN(pcbinfo);
2246
2247                 if (error == 0 && i < n) {
2248                         bzero(&xi, sizeof xi);
2249                         xi.xi_len = sizeof xi;
2250                         while (i < n) {
2251                                 error = SYSCTL_OUT(req, &xi, sizeof xi);
2252                                 if (error)
2253                                         break;
2254                                 ++i;
2255                         }
2256                 }
2257         }
2258
2259         lwkt_migratecpu(origcpu);
2260         kfree(marker, M_TEMP);
2261         return error;
2262 }
2263
2264 int
2265 in_pcblist_global_ncpus2(SYSCTL_HANDLER_ARGS)
2266 {
2267         return in_pcblist_global(oidp, arg1, ncpus2, req);
2268 }
2269
2270 void
2271 in_savefaddr(struct socket *so, const struct sockaddr *faddr)
2272 {
2273         struct sockaddr_in *sin;
2274
2275         KASSERT(faddr->sa_family == AF_INET,
2276             ("not AF_INET faddr %d", faddr->sa_family));
2277
2278         sin = kmalloc(sizeof(*sin), M_SONAME, M_WAITOK | M_ZERO);
2279         sin->sin_family = AF_INET;
2280         sin->sin_len = sizeof(*sin);
2281         sin->sin_port = ((const struct sockaddr_in *)faddr)->sin_port;
2282         sin->sin_addr = ((const struct sockaddr_in *)faddr)->sin_addr;
2283
2284         so->so_faddr = (struct sockaddr *)sin;
2285 }
2286
2287 void
2288 in_pcbportinfo_init(struct inpcbportinfo *portinfo, int hashsize,
2289     boolean_t shared, u_short offset)
2290 {
2291         memset(portinfo, 0, sizeof(*portinfo));
2292
2293         portinfo->offset = offset;
2294         portinfo->lastport = offset;
2295         portinfo->lastlow = offset;
2296         portinfo->lasthi = offset;
2297
2298         portinfo->porthashbase = hashinit(hashsize, M_PCB,
2299             &portinfo->porthashmask);
2300
2301         if (shared) {
2302                 portinfo->porttoken = kmalloc(sizeof(struct lwkt_token),
2303                     M_PCB, M_WAITOK);
2304                 lwkt_token_init(portinfo->porttoken, "porttoken");
2305         }
2306 }
2307
2308 void
2309 in_pcbportrange(u_short *hi0, u_short *lo0, u_short ofs, u_short step)
2310 {
2311         int hi, lo;
2312
2313         if (step == 1)
2314                 return;
2315
2316         hi = *hi0;
2317         lo = *lo0;
2318
2319         hi = rounddown2(hi, step);
2320         hi += ofs;
2321         if (hi > (int)*hi0)
2322                 hi -= step;
2323
2324         lo = roundup2(lo, step);
2325         lo -= (step - ofs);
2326         if (lo < (int)*lo0)
2327                 lo += step;
2328
2329         *hi0 = hi;
2330         *lo0 = lo;
2331 }
2332
2333 void
2334 in_pcbglobalinit(void)
2335 {
2336         int cpu;
2337
2338         in_pcbmarkers = kmalloc(ncpus * sizeof(struct inpcb), M_PCB,
2339             M_WAITOK | M_ZERO);
2340         in_pcbcontainer_markers = kmalloc(ncpus * sizeof(struct inpcontainer),
2341             M_PCB, M_WAITOK | M_ZERO);
2342
2343         for (cpu = 0; cpu < ncpus; ++cpu) {
2344                 struct inpcontainer *ic = &in_pcbcontainer_markers[cpu];
2345                 struct inpcb *marker = &in_pcbmarkers[cpu];
2346
2347                 marker->inp_flags |= INP_PLACEMARKER;
2348                 ic->ic_inp = marker;
2349         }
2350 }
2351
2352 struct inpcb *
2353 in_pcbmarker(int cpuid)
2354 {
2355         KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid));
2356         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
2357
2358         return &in_pcbmarkers[cpuid];
2359 }
2360
2361 struct inpcontainer *
2362 in_pcbcontainer_marker(int cpuid)
2363 {
2364         KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid));
2365         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
2366
2367         return &in_pcbcontainer_markers[cpuid];
2368 }