sys/netinet/in_pcb.c

   1 /*
   2  * Copyright (c) 2004 Jeffrey M. Hsu.  All rights reserved.
   3  * Copyright (c) 2004 The DragonFly Project.  All rights reserved.
   4  *
   5  * This code is derived from software contributed to The DragonFly Project
   6  * by Jeffrey M. Hsu.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the name of The DragonFly Project nor the names of its
  17  *    contributors may be used to endorse or promote products derived
  18  *    from this software without specific, prior written permission.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  23  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  24  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  25  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  26  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  27  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  28  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  29  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  30  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  */
  33
  34 /*
  35  * Copyright (c) 1982, 1986, 1991, 1993, 1995
  36  *      The Regents of the University of California.  All rights reserved.
  37  *
  38  * Redistribution and use in source and binary forms, with or without
  39  * modification, are permitted provided that the following conditions
  40  * are met:
  41  * 1. Redistributions of source code must retain the above copyright
  42  *    notice, this list of conditions and the following disclaimer.
  43  * 2. Redistributions in binary form must reproduce the above copyright
  44  *    notice, this list of conditions and the following disclaimer in the
  45  *    documentation and/or other materials provided with the distribution.
  46  * 3. Neither the name of the University nor the names of its contributors
  47  *    may be used to endorse or promote products derived from this software
  48  *    without specific prior written permission.
  49  *
  50  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  51  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  52  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  53  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  54  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  55  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  56  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  57  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  58  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  59  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  60  * SUCH DAMAGE.
  61  *
  62  *      @(#)in_pcb.c    8.4 (Berkeley) 5/24/95
  63  * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $
  64  */
  65
  66 #include "opt_ipsec.h"
  67 #include "opt_inet6.h"
  68
  69 #include <sys/param.h>
  70 #include <sys/systm.h>
  71 #include <sys/malloc.h>
  72 #include <sys/mbuf.h>
  73 #include <sys/domain.h>
  74 #include <sys/protosw.h>
  75 #include <sys/socket.h>
  76 #include <sys/socketvar.h>
  77 #include <sys/proc.h>
  78 #include <sys/priv.h>
  79 #include <sys/jail.h>
  80 #include <sys/kernel.h>
  81 #include <sys/sysctl.h>
  82
  83 #include <sys/thread2.h>
  84 #include <sys/socketvar2.h>
  85 #include <sys/msgport2.h>
  86
  87 #include <machine/limits.h>
  88
  89 #include <net/if.h>
  90 #include <net/if_types.h>
  91 #include <net/route.h>
  92 #include <net/netisr2.h>
  93
  94 #include <netinet/in.h>
  95 #include <netinet/in_pcb.h>
  96 #include <netinet/in_var.h>
  97 #include <netinet/ip_var.h>
  98 #ifdef INET6
  99 #include <netinet/ip6.h>
 100 #include <netinet6/ip6_var.h>
 101 #endif /* INET6 */
 102
 103 #ifdef IPSEC
 104 #include <netinet6/ipsec.h>
 105 #include <netproto/key/key.h>
 106 #include <netproto/ipsec/esp_var.h>
 107 #endif
 108
 109 #ifdef FAST_IPSEC
 110 #if defined(IPSEC) || defined(IPSEC_ESP)
 111 #error "Bad idea: don't compile with both IPSEC and FAST_IPSEC!"
 112 #endif
 113
 114 #include <netproto/ipsec/ipsec.h>
 115 #include <netproto/ipsec/key.h>
 116 #define IPSEC
 117 #endif /* FAST_IPSEC */
 118
 119 #define INP_LOCALGROUP_SIZMIN   8
 120 #define INP_LOCALGROUP_SIZMAX   256
 121
 122 struct in_addr zeroin_addr;
 123
 124 /*
 125  * These configure the range of local port addresses assigned to
 126  * "unspecified" outgoing connections/packets/whatever.
 127  */
 128 int ipport_lowfirstauto = IPPORT_RESERVED - 1;  /* 1023 */
 129 int ipport_lowlastauto = IPPORT_RESERVEDSTART;  /* 600 */
 130
 131 int ipport_firstauto = IPPORT_RESERVED;         /* 1024 */
 132 int ipport_lastauto = IPPORT_USERRESERVED;      /* 5000 */
 133
 134 int ipport_hifirstauto = IPPORT_HIFIRSTAUTO;    /* 49152 */
 135 int ipport_hilastauto = IPPORT_HILASTAUTO;      /* 65535 */
 136
 137 #define RANGECHK(var, min, max) \
 138         if ((var) < (min)) { (var) = (min); } \
 139         else if ((var) > (max)) { (var) = (max); }
 140
 141 int udpencap_enable = 1;        /* enabled by default */
 142 int udpencap_port = 4500;       /* triggers decapsulation */
 143
 144 /*
 145  * Per-netisr inpcb markers.
 146  * NOTE: they should only be used in netisrs.
 147  */
 148 static struct inpcb             *in_pcbmarkers;
 149 static struct inpcontainer      *in_pcbcontainer_markers;
 150
 151 static int
 152 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
 153 {
 154         int error;
 155
 156         error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
 157         if (!error) {
 158                 RANGECHK(ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
 159                 RANGECHK(ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
 160
 161                 RANGECHK(ipport_firstauto, IPPORT_RESERVED, USHRT_MAX);
 162                 RANGECHK(ipport_lastauto, IPPORT_RESERVED, USHRT_MAX);
 163
 164                 RANGECHK(ipport_hifirstauto, IPPORT_RESERVED, USHRT_MAX);
 165                 RANGECHK(ipport_hilastauto, IPPORT_RESERVED, USHRT_MAX);
 166         }
 167         return (error);
 168 }
 169
 170 #undef RANGECHK
 171
 172 SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports");
 173
 174 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW,
 175            &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", "");
 176 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, CTLTYPE_INT|CTLFLAG_RW,
 177            &ipport_lowlastauto, 0, &sysctl_net_ipport_check, "I", "");
 178 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, CTLTYPE_INT|CTLFLAG_RW,
 179            &ipport_firstauto, 0, &sysctl_net_ipport_check, "I", "");
 180 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, CTLTYPE_INT|CTLFLAG_RW,
 181            &ipport_lastauto, 0, &sysctl_net_ipport_check, "I", "");
 182 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, CTLTYPE_INT|CTLFLAG_RW,
 183            &ipport_hifirstauto, 0, &sysctl_net_ipport_check, "I", "");
 184 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW,
 185            &ipport_hilastauto, 0, &sysctl_net_ipport_check, "I", "");
 186
 187 /*
 188  * in_pcb.c: manage the Protocol Control Blocks.
 189  *
 190  * NOTE: It is assumed that most of these functions will be called from
 191  * a critical section.  XXX - There are, unfortunately, a few exceptions
 192  * to this rule that should be fixed.
 193  *
 194  * NOTE: The caller should initialize the cpu field to the cpu running the
 195  * protocol stack associated with this inpcbinfo.
 196  */
 197
 198 void
 199 in_pcbinfo_init(struct inpcbinfo *pcbinfo, int cpu, boolean_t shared)
 200 {
 201         KASSERT(cpu >= 0 && cpu < ncpus, ("invalid cpu%d", cpu));
 202         pcbinfo->cpu = cpu;
 203
 204         LIST_INIT(&pcbinfo->pcblisthead);
 205         pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), M_PCB,
 206                                     M_WAITOK | M_ZERO);
 207
 208         if (shared) {
 209                 pcbinfo->infotoken = kmalloc(sizeof(struct lwkt_token),
 210                     M_PCB, M_WAITOK);
 211                 lwkt_token_init(pcbinfo->infotoken, "infotoken");
 212         } else {
 213                 pcbinfo->infotoken = NULL;
 214         }
 215 }
 216
 217 struct baddynamicports baddynamicports;
 218
 219 /*
 220  * Check if the specified port is invalid for dynamic allocation.
 221  */
 222 int
 223 in_baddynamic(u_int16_t port, u_int16_t proto)
 224 {
 225         switch (proto) {
 226         case IPPROTO_TCP:
 227                 return (DP_ISSET(baddynamicports.tcp, port));
 228         case IPPROTO_UDP:
 229 #ifdef IPSEC
 230                 /* Cannot preset this as it is a sysctl */
 231                 if (port == udpencap_port)
 232                         return (1);
 233 #endif
 234                 return (DP_ISSET(baddynamicports.udp, port));
 235         default:
 236                 return (0);
 237         }
 238 }
 239
 240 void
 241 in_pcbonlist(struct inpcb *inp)
 242 {
 243         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 244
 245         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
 246             ("not in the correct netisr"));
 247         KASSERT((inp->inp_flags & INP_ONLIST) == 0, ("already on pcblist"));
 248         inp->inp_flags |= INP_ONLIST;
 249
 250         GET_PCBINFO_TOKEN(pcbinfo);
 251         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list);
 252         pcbinfo->ipi_count++;
 253         REL_PCBINFO_TOKEN(pcbinfo);
 254 }
 255
 256 void
 257 in_pcbofflist(struct inpcb *inp)
 258 {
 259         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 260
 261         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
 262             ("not in the correct netisr"));
 263         KASSERT(inp->inp_flags & INP_ONLIST, ("not on pcblist"));
 264         inp->inp_flags &= ~INP_ONLIST;
 265
 266         GET_PCBINFO_TOKEN(pcbinfo);
 267         LIST_REMOVE(inp, inp_list);
 268         KASSERT(pcbinfo->ipi_count > 0,
 269             ("invalid inpcb count %d", pcbinfo->ipi_count));
 270         pcbinfo->ipi_count--;
 271         REL_PCBINFO_TOKEN(pcbinfo);
 272 }
 273
 274 /*
 275  * Allocate a PCB and associate it with the socket.
 276  */
 277 int
 278 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
 279 {
 280         struct inpcb *inp;
 281 #ifdef IPSEC
 282         int error;
 283 #endif
 284
 285         inp = kmalloc(pcbinfo->ipi_size, M_PCB, M_WAITOK|M_ZERO|M_NULLOK);
 286         if (inp == NULL)
 287                 return (ENOMEM);
 288         inp->inp_lgrpindex = -1;
 289         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
 290         inp->inp_pcbinfo = pcbinfo;
 291         inp->inp_socket = so;
 292 #ifdef IPSEC
 293         error = ipsec_init_policy(so, &inp->inp_sp);
 294         if (error != 0) {
 295                 kfree(inp, M_PCB);
 296                 return (error);
 297         }
 298 #endif
 299 #ifdef INET6
 300         if (INP_CHECK_SOCKAF(so, AF_INET6)) {
 301                 if (ip6_auto_flowlabel)
 302                         inp->inp_flags |= IN6P_AUTOFLOWLABEL;
 303                 inp->inp_af = AF_INET6;
 304         } else
 305 #endif
 306         inp->inp_af = AF_INET;
 307         soreference(so);
 308         so->so_pcb = inp;
 309
 310         in_pcbonlist(inp);
 311         return (0);
 312 }
 313
 314 /*
 315  * Unlink a pcb with the intention of moving it to another cpu with a
 316  * different pcbinfo.  While unlinked nothing should attempt to dereference
 317  * inp_pcbinfo, NULL it out so we assert if it does.
 318  */
 319 void
 320 in_pcbunlink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags)
 321 {
 322         KASSERT(inp->inp_pcbinfo == pcbinfo, ("pcbinfo mismatch"));
 323         KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0,
 324             ("already linked"));
 325
 326         in_pcbofflist(inp);
 327         inp->inp_pcbinfo = NULL;
 328 }
 329
 330 void
 331 in_pcbunlink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
 332 {
 333         in_pcbunlink_flags(inp, pcbinfo, INP_WILDCARD);
 334 }
 335
 336 /*
 337  * Relink a pcb into a new pcbinfo.
 338  */
 339 void
 340 in_pcblink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags)
 341 {
 342         KASSERT(inp->inp_pcbinfo == NULL, ("has pcbinfo"));
 343         KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0,
 344             ("already linked"));
 345
 346         inp->inp_pcbinfo = pcbinfo;
 347         in_pcbonlist(inp);
 348 }
 349
 350 void
 351 in_pcblink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
 352 {
 353         return in_pcblink_flags(inp, pcbinfo, INP_WILDCARD);
 354 }
 355
 356 static int
 357 in_pcbsetlport(struct inpcb *inp, int wild, struct ucred *cred)
 358 {
 359         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 360         struct inpcbportinfo *portinfo;
 361         u_short first, last, lport, step;
 362         u_short *lastport;
 363         int count, error;
 364         int portinfo_first, portinfo_idx;
 365
 366         inp->inp_flags |= INP_ANONPORT;
 367
 368         step = pcbinfo->portinfo_mask + 1;
 369         portinfo_first = mycpuid & pcbinfo->portinfo_mask;
 370         portinfo_idx = portinfo_first;
 371 loop:
 372         portinfo = &pcbinfo->portinfo[portinfo_idx];
 373
 374         if (inp->inp_flags & INP_HIGHPORT) {
 375                 first = ipport_hifirstauto;     /* sysctl */
 376                 last  = ipport_hilastauto;
 377                 lastport = &portinfo->lasthi;
 378         } else if (inp->inp_flags & INP_LOWPORT) {
 379                 if (cred &&
 380                     (error =
 381                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 382                         inp->inp_laddr.s_addr = INADDR_ANY;
 383                         return error;
 384                 }
 385                 first = ipport_lowfirstauto;    /* 1023 */
 386                 last  = ipport_lowlastauto;     /* 600 */
 387                 lastport = &portinfo->lastlow;
 388         } else {
 389                 first = ipport_firstauto;       /* sysctl */
 390                 last  = ipport_lastauto;
 391                 lastport = &portinfo->lastport;
 392         }
 393
 394         /*
 395          * This has to be atomic.  If the porthash is shared across multiple
 396          * protocol threads (aka tcp) then the token must be held.
 397          */
 398         GET_PORT_TOKEN(portinfo);
 399
 400         /*
 401          * Simple check to ensure all ports are not used up causing
 402          * a deadlock here.
 403          *
 404          * We split the two cases (up and down) so that the direction
 405          * is not being tested on each round of the loop.
 406          */
 407         if (first > last) {
 408                 /*
 409                  * counting down
 410                  */
 411                 in_pcbportrange(&first, &last, portinfo->offset, step);
 412                 count = (first - last) / step;
 413
 414                 do {
 415                         if (count-- < 0) {      /* completely used? */
 416                                 error = EADDRNOTAVAIL;
 417                                 goto done;
 418                         }
 419                         *lastport -= step;
 420                         if (*lastport > first || *lastport < last)
 421                                 *lastport = first;
 422                         KKASSERT((*lastport & pcbinfo->portinfo_mask) ==
 423                             portinfo->offset);
 424                         lport = htons(*lastport);
 425                 } while (in_pcblookup_local(portinfo, inp->inp_laddr, lport,
 426                     wild, cred));
 427         } else {
 428                 /*
 429                  * counting up
 430                  */
 431                 in_pcbportrange(&last, &first, portinfo->offset, step);
 432                 count = (last - first) / step;
 433
 434                 do {
 435                         if (count-- < 0) {      /* completely used? */
 436                                 error = EADDRNOTAVAIL;
 437                                 goto done;
 438                         }
 439                         *lastport += step;
 440                         if (*lastport < first || *lastport > last)
 441                                 *lastport = first;
 442                         KKASSERT((*lastport & pcbinfo->portinfo_mask) ==
 443                             portinfo->offset);
 444                         lport = htons(*lastport);
 445                 } while (in_pcblookup_local(portinfo, inp->inp_laddr, lport,
 446                     wild, cred));
 447         }
 448         inp->inp_lport = lport;
 449         in_pcbinsporthash(portinfo, inp);
 450         error = 0;
 451 done:
 452         REL_PORT_TOKEN(portinfo);
 453
 454         if (error) {
 455                 /* Try next portinfo */
 456                 portinfo_idx++;
 457                 portinfo_idx &= pcbinfo->portinfo_mask;
 458                 if (portinfo_idx != portinfo_first)
 459                         goto loop;
 460                 inp->inp_laddr.s_addr = INADDR_ANY;
 461         }
 462         return error;
 463 }
 464
 465 int
 466 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
 467 {
 468         struct socket *so = inp->inp_socket;
 469         struct sockaddr_in jsin;
 470         struct ucred *cred = NULL;
 471         int wild = 0;
 472
 473         if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */
 474                 return (EADDRNOTAVAIL);
 475         if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
 476                 return (EINVAL);        /* already bound */
 477
 478         if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT)))
 479                 wild = 1;    /* neither SO_REUSEADDR nor SO_REUSEPORT is set */
 480         if (td->td_proc)
 481                 cred = td->td_proc->p_ucred;
 482
 483         if (nam != NULL) {
 484                 struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 485                 struct inpcbinfo *pcbinfo;
 486                 struct inpcbportinfo *portinfo;
 487                 struct inpcb *t;
 488                 u_short lport, lport_ho;
 489                 int reuseport = (so->so_options & SO_REUSEPORT);
 490                 int error;
 491
 492                 if (nam->sa_len != sizeof *sin)
 493                         return (EINVAL);
 494 #ifdef notdef
 495                 /*
 496                  * We should check the family, but old programs
 497                  * incorrectly fail to initialize it.
 498                  */
 499                 if (sin->sin_family != AF_INET)
 500                         return (EAFNOSUPPORT);
 501 #endif
 502                 if (!prison_replace_wildcards(td, nam))
 503                         return (EINVAL);
 504
 505                 lport = sin->sin_port;
 506                 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 507                         /*
 508                          * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
 509                          * allow complete duplication of binding if
 510                          * SO_REUSEPORT is set, or if SO_REUSEADDR is set
 511                          * and a multicast address is bound on both
 512                          * new and duplicated sockets.
 513                          */
 514                         if (so->so_options & SO_REUSEADDR)
 515                                 reuseport = SO_REUSEADDR | SO_REUSEPORT;
 516                 } else if (sin->sin_addr.s_addr != INADDR_ANY) {
 517                         sin->sin_port = 0;              /* yech... */
 518                         bzero(&sin->sin_zero, sizeof sin->sin_zero);
 519                         if (ifa_ifwithaddr((struct sockaddr *)sin) == NULL)
 520                                 return (EADDRNOTAVAIL);
 521                 }
 522
 523                 inp->inp_laddr = sin->sin_addr;
 524
 525                 jsin.sin_family = AF_INET;
 526                 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 527                 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 528                         inp->inp_laddr.s_addr = INADDR_ANY;
 529                         return (EINVAL);
 530                 }
 531                 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 532
 533                 if (lport == 0) {
 534                         /* Auto-select local port */
 535                         return in_pcbsetlport(inp, wild, cred);
 536                 }
 537                 lport_ho = ntohs(lport);
 538
 539                 /* GROSS */
 540                 if (lport_ho < IPPORT_RESERVED && cred &&
 541                     (error =
 542                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 543                         inp->inp_laddr.s_addr = INADDR_ANY;
 544                         return (error);
 545                 }
 546
 547                 /*
 548                  * Locate the proper portinfo based on lport
 549                  */
 550                 pcbinfo = inp->inp_pcbinfo;
 551                 portinfo =
 552                     &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask];
 553                 KKASSERT((lport_ho & pcbinfo->portinfo_mask) ==
 554                     portinfo->offset);
 555
 556                 /*
 557                  * This has to be atomic.  If the porthash is shared across
 558                  * multiple protocol threads (aka tcp) then the token must
 559                  * be held.
 560                  */
 561                 GET_PORT_TOKEN(portinfo);
 562
 563                 if (so->so_cred->cr_uid != 0 &&
 564                     !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 565                         t = in_pcblookup_local(portinfo, sin->sin_addr, lport,
 566                             INPLOOKUP_WILDCARD, cred);
 567                         if (t &&
 568                             (so->so_cred->cr_uid !=
 569                              t->inp_socket->so_cred->cr_uid)) {
 570                                 inp->inp_laddr.s_addr = INADDR_ANY;
 571                                 error = EADDRINUSE;
 572                                 goto done;
 573                         }
 574                 }
 575                 if (cred && !prison_replace_wildcards(td, nam)) {
 576                         inp->inp_laddr.s_addr = INADDR_ANY;
 577                         error = EADDRNOTAVAIL;
 578                         goto done;
 579                 }
 580                 t = in_pcblookup_local(portinfo, sin->sin_addr, lport,
 581                     wild, cred);
 582                 if (t && !(reuseport & t->inp_socket->so_options)) {
 583                         inp->inp_laddr.s_addr = INADDR_ANY;
 584                         error = EADDRINUSE;
 585                         goto done;
 586                 }
 587                 inp->inp_lport = lport;
 588                 in_pcbinsporthash(portinfo, inp);
 589                 error = 0;
 590 done:
 591                 REL_PORT_TOKEN(portinfo);
 592                 return (error);
 593         } else {
 594                 jsin.sin_family = AF_INET;
 595                 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 596                 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 597                         inp->inp_laddr.s_addr = INADDR_ANY;
 598                         return (EINVAL);
 599                 }
 600                 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 601
 602                 return in_pcbsetlport(inp, wild, cred);
 603         }
 604 }
 605
 606 static struct inpcb *
 607 in_pcblookup_localremote(struct inpcbportinfo *portinfo, struct in_addr laddr,
 608     u_short lport, struct in_addr faddr, u_short fport, struct ucred *cred)
 609 {
 610         struct inpcb *inp;
 611         struct inpcbporthead *porthash;
 612         struct inpcbport *phd;
 613         struct inpcb *match = NULL;
 614
 615         /*
 616          * If the porthashbase is shared across several cpus, it must
 617          * have been locked.
 618          */
 619         ASSERT_PORT_TOKEN_HELD(portinfo);
 620
 621         /*
 622          * Best fit PCB lookup.
 623          *
 624          * First see if this local port is in use by looking on the
 625          * port hash list.
 626          */
 627         porthash = &portinfo->porthashbase[
 628                         INP_PCBPORTHASH(lport, portinfo->porthashmask)];
 629         LIST_FOREACH(phd, porthash, phd_hash) {
 630                 if (phd->phd_port == lport)
 631                         break;
 632         }
 633         if (phd != NULL) {
 634                 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
 635 #ifdef INET6
 636                         if (!INP_ISIPV4(inp))
 637                                 continue;
 638 #endif
 639                         if (inp->inp_laddr.s_addr != INADDR_ANY &&
 640                             inp->inp_laddr.s_addr != laddr.s_addr)
 641                                 continue;
 642
 643                         if (inp->inp_faddr.s_addr != INADDR_ANY &&
 644                             inp->inp_faddr.s_addr != faddr.s_addr)
 645                                 continue;
 646
 647                         if (inp->inp_fport != 0 && inp->inp_fport != fport)
 648                                 continue;
 649
 650                         if (cred == NULL ||
 651                             cred->cr_prison ==
 652                             inp->inp_socket->so_cred->cr_prison) {
 653                                 match = inp;
 654                                 break;
 655                         }
 656                 }
 657         }
 658         return (match);
 659 }
 660
 661 int
 662 in_pcbbind_remote(struct inpcb *inp, const struct sockaddr *remote,
 663     struct thread *td)
 664 {
 665         struct proc *p = td->td_proc;
 666         unsigned short *lastport;
 667         const struct sockaddr_in *sin = (const struct sockaddr_in *)remote;
 668         struct sockaddr_in jsin;
 669         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 670         struct inpcbportinfo *portinfo;
 671         struct ucred *cred = NULL;
 672         u_short first, last, lport, step;
 673         int count, error, dup;
 674         int portinfo_first, portinfo_idx;
 675
 676         if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */
 677                 return (EADDRNOTAVAIL);
 678
 679         KKASSERT(inp->inp_laddr.s_addr != INADDR_ANY);
 680         if (inp->inp_lport != 0)
 681                 return (EINVAL);        /* already bound */
 682
 683         KKASSERT(p);
 684         cred = p->p_ucred;
 685
 686         jsin.sin_family = AF_INET;
 687         jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 688         if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 689                 inp->inp_laddr.s_addr = INADDR_ANY;
 690                 return (EINVAL);
 691         }
 692         inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 693
 694         inp->inp_flags |= INP_ANONPORT;
 695
 696         step = pcbinfo->portinfo_mask + 1;
 697         portinfo_first = mycpuid & pcbinfo->portinfo_mask;
 698         portinfo_idx = portinfo_first;
 699 loop:
 700         portinfo = &pcbinfo->portinfo[portinfo_idx];
 701         dup = 0;
 702
 703         if (inp->inp_flags & INP_HIGHPORT) {
 704                 first = ipport_hifirstauto;     /* sysctl */
 705                 last  = ipport_hilastauto;
 706                 lastport = &portinfo->lasthi;
 707         } else if (inp->inp_flags & INP_LOWPORT) {
 708                 if (cred &&
 709                     (error =
 710                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 711                         inp->inp_laddr.s_addr = INADDR_ANY;
 712                         return (error);
 713                 }
 714                 first = ipport_lowfirstauto;    /* 1023 */
 715                 last  = ipport_lowlastauto;     /* 600 */
 716                 lastport = &portinfo->lastlow;
 717         } else {
 718                 first = ipport_firstauto;       /* sysctl */
 719                 last  = ipport_lastauto;
 720                 lastport = &portinfo->lastport;
 721         }
 722
 723         /*
 724          * This has to be atomic.  If the porthash is shared across multiple
 725          * protocol threads (aka tcp) then the token must be held.
 726          */
 727         GET_PORT_TOKEN(portinfo);
 728
 729 again:
 730         /*
 731          * Simple check to ensure all ports are not used up causing
 732          * a deadlock here.
 733          *
 734          * We split the two cases (up and down) so that the direction
 735          * is not being tested on each round of the loop.
 736          */
 737         if (first > last) {
 738                 /*
 739                  * counting down
 740                  */
 741                 in_pcbportrange(&first, &last, portinfo->offset, step);
 742                 count = (first - last) / step;
 743
 744                 do {
 745                         if (count-- < 0) {      /* completely used? */
 746                                 error = EADDRNOTAVAIL;
 747                                 goto done;
 748                         }
 749                         *lastport -= step;
 750                         if (*lastport > first || *lastport < last)
 751                                 *lastport = first;
 752                         KKASSERT((*lastport & pcbinfo->portinfo_mask) ==
 753                             portinfo->offset);
 754                         lport = htons(*lastport);
 755                 } while (in_pcblookup_localremote(portinfo, inp->inp_laddr,
 756                     lport, sin->sin_addr, sin->sin_port, cred));
 757         } else {
 758                 /*
 759                  * counting up
 760                  */
 761                 in_pcbportrange(&last, &first, portinfo->offset, step);
 762                 count = (last - first) / step;
 763
 764                 do {
 765                         if (count-- < 0) {      /* completely used? */
 766                                 error = EADDRNOTAVAIL;
 767                                 goto done;
 768                         }
 769                         *lastport += step;
 770                         if (*lastport < first || *lastport > last)
 771                                 *lastport = first;
 772                         KKASSERT((*lastport & pcbinfo->portinfo_mask) ==
 773                             portinfo->offset);
 774                         lport = htons(*lastport);
 775                 } while (in_pcblookup_localremote(portinfo, inp->inp_laddr,
 776                     lport, sin->sin_addr, sin->sin_port, cred));
 777         }
 778
 779         /* This could happen on loopback interface */
 780         if (sin->sin_port == lport &&
 781             sin->sin_addr.s_addr == inp->inp_laddr.s_addr) {
 782                 if (dup) {
 783                         /*
 784                          * Duplicate again; give up
 785                          */
 786                         error = EADDRNOTAVAIL;
 787                         goto done;
 788                 }
 789                 dup = 1;
 790                 goto again;
 791         }
 792         inp->inp_lport = lport;
 793         in_pcbinsporthash(portinfo, inp);
 794         error = 0;
 795 done:
 796         REL_PORT_TOKEN(portinfo);
 797
 798         if (error) {
 799                 /* Try next portinfo */
 800                 portinfo_idx++;
 801                 portinfo_idx &= pcbinfo->portinfo_mask;
 802                 if (portinfo_idx != portinfo_first)
 803                         goto loop;
 804                 inp->inp_laddr.s_addr = INADDR_ANY;
 805         }
 806         return error;
 807 }
 808
 809 /*
 810  *   Transform old in_pcbconnect() into an inner subroutine for new
 811  *   in_pcbconnect(): Do some validity-checking on the remote
 812  *   address (in mbuf 'nam') and then determine local host address
 813  *   (i.e., which interface) to use to access that remote host.
 814  *
 815  *   This preserves definition of in_pcbconnect(), while supporting a
 816  *   slightly different version for T/TCP.  (This is more than
 817  *   a bit of a kludge, but cleaning up the internal interfaces would
 818  *   have forced minor changes in every protocol).
 819  */
 820 int
 821 in_pcbladdr_find(struct inpcb *inp, struct sockaddr *nam,
 822     struct sockaddr_in **plocal_sin, struct thread *td, int find)
 823 {
 824         struct in_ifaddr *ia;
 825         struct ucred *cred = NULL;
 826         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 827         struct sockaddr *jsin;
 828         int jailed = 0, alloc_route = 0;
 829
 830         if (nam->sa_len != sizeof *sin)
 831                 return (EINVAL);
 832         if (sin->sin_family != AF_INET)
 833                 return (EAFNOSUPPORT);
 834         if (sin->sin_port == 0)
 835                 return (EADDRNOTAVAIL);
 836         if (td && td->td_proc && td->td_proc->p_ucred)
 837                 cred = td->td_proc->p_ucred;
 838         if (cred && cred->cr_prison)
 839                 jailed = 1;
 840         if (!TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) {
 841                 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia;
 842                 /*
 843                  * If the destination address is INADDR_ANY,
 844                  * use the primary local address.
 845                  * If the supplied address is INADDR_BROADCAST,
 846                  * and the primary interface supports broadcast,
 847                  * choose the broadcast address for that interface.
 848                  */
 849                 if (sin->sin_addr.s_addr == INADDR_ANY)
 850                         sin->sin_addr = IA_SIN(ia)->sin_addr;
 851                 else if (sin->sin_addr.s_addr == (u_long)INADDR_BROADCAST &&
 852                     (ia->ia_ifp->if_flags & IFF_BROADCAST))
 853                         sin->sin_addr = satosin(&ia->ia_broadaddr)->sin_addr;
 854         }
 855         if (find) {
 856                 struct route *ro;
 857
 858                 ia = NULL;
 859                 /*
 860                  * If route is known or can be allocated now,
 861                  * our src addr is taken from the i/f, else punt.
 862                  * Note that we should check the address family of the cached
 863                  * destination, in case of sharing the cache with IPv6.
 864                  */
 865                 ro = &inp->inp_route;
 866                 if (ro->ro_rt &&
 867                     (!(ro->ro_rt->rt_flags & RTF_UP) ||
 868                      ro->ro_dst.sa_family != AF_INET ||
 869                      satosin(&ro->ro_dst)->sin_addr.s_addr !=
 870                                       sin->sin_addr.s_addr ||
 871                      inp->inp_socket->so_options & SO_DONTROUTE)) {
 872                         RTFREE(ro->ro_rt);
 873                         ro->ro_rt = NULL;
 874                 }
 875                 if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/
 876                     (ro->ro_rt == NULL ||
 877                     ro->ro_rt->rt_ifp == NULL)) {
 878                         /* No route yet, so try to acquire one */
 879                         bzero(&ro->ro_dst, sizeof(struct sockaddr_in));
 880                         ro->ro_dst.sa_family = AF_INET;
 881                         ro->ro_dst.sa_len = sizeof(struct sockaddr_in);
 882                         ((struct sockaddr_in *) &ro->ro_dst)->sin_addr =
 883                                 sin->sin_addr;
 884                         rtalloc(ro);
 885                         alloc_route = 1;
 886                 }
 887                 /*
 888                  * If we found a route, use the address
 889                  * corresponding to the outgoing interface
 890                  * unless it is the loopback (in case a route
 891                  * to our address on another net goes to loopback).
 892                  */
 893                 if (ro->ro_rt &&
 894                     !(ro->ro_rt->rt_ifp->if_flags & IFF_LOOPBACK)) {
 895                         if (jailed) {
 896                                 if (jailed_ip(cred->cr_prison,
 897                                     ro->ro_rt->rt_ifa->ifa_addr)) {
 898                                         ia = ifatoia(ro->ro_rt->rt_ifa);
 899                                 }
 900                         } else {
 901                                 ia = ifatoia(ro->ro_rt->rt_ifa);
 902                         }
 903                 }
 904                 if (ia == NULL) {
 905                         u_short fport = sin->sin_port;
 906
 907                         sin->sin_port = 0;
 908                         ia = ifatoia(ifa_ifwithdstaddr(sintosa(sin)));
 909                         if (ia && jailed && !jailed_ip(cred->cr_prison,
 910                             sintosa(&ia->ia_addr)))
 911                                 ia = NULL;
 912                         if (ia == NULL)
 913                                 ia = ifatoia(ifa_ifwithnet(sintosa(sin)));
 914                         if (ia && jailed && !jailed_ip(cred->cr_prison,
 915                             sintosa(&ia->ia_addr)))
 916                                 ia = NULL;
 917                         sin->sin_port = fport;
 918                         if (ia == NULL &&
 919                             !TAILQ_EMPTY(&in_ifaddrheads[mycpuid]))
 920                                 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia;
 921                         if (ia && jailed && !jailed_ip(cred->cr_prison,
 922                             sintosa(&ia->ia_addr)))
 923                                 ia = NULL;
 924
 925                         if (!jailed && ia == NULL)
 926                                 goto fail;
 927                 }
 928                 /*
 929                  * If the destination address is multicast and an outgoing
 930                  * interface has been set as a multicast option, use the
 931                  * address of that interface as our source address.
 932                  */
 933                 if (!jailed && IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
 934                     inp->inp_moptions != NULL) {
 935                         struct ip_moptions *imo;
 936                         struct ifnet *ifp;
 937
 938                         imo = inp->inp_moptions;
 939                         if (imo->imo_multicast_ifp != NULL) {
 940                                 struct in_ifaddr_container *iac;
 941
 942                                 ifp = imo->imo_multicast_ifp;
 943                                 ia = NULL;
 944                                 TAILQ_FOREACH(iac,
 945                                 &in_ifaddrheads[mycpuid], ia_link) {
 946                                         if (iac->ia->ia_ifp == ifp) {
 947                                                 ia = iac->ia;
 948                                                 break;
 949                                         }
 950                                 }
 951                                 if (ia == NULL)
 952                                         goto fail;
 953                         }
 954                 }
 955                 /*
 956                  * Don't do pcblookup call here; return interface in plocal_sin
 957                  * and exit to caller, that will do the lookup.
 958                  */
 959                 if (ia == NULL && jailed) {
 960                         if ((jsin = prison_get_nonlocal(
 961                                 cred->cr_prison, AF_INET, NULL)) != NULL ||
 962                             (jsin = prison_get_local(
 963                                 cred->cr_prison, AF_INET, NULL)) != NULL) {
 964                                 *plocal_sin = satosin(jsin);
 965                         } else {
 966                                 /* IPv6 only Jail */
 967                                 goto fail;
 968                         }
 969                 } else {
 970                         *plocal_sin = &ia->ia_addr;
 971                 }
 972         }
 973         return (0);
 974 fail:
 975         if (alloc_route)
 976                 in_pcbresetroute(inp);
 977         return (EADDRNOTAVAIL);
 978 }
 979
 980 int
 981 in_pcbladdr(struct inpcb *inp, struct sockaddr *nam,
 982     struct sockaddr_in **plocal_sin, struct thread *td)
 983 {
 984         return in_pcbladdr_find(inp, nam, plocal_sin, td,
 985             (inp->inp_laddr.s_addr == INADDR_ANY));
 986 }
 987
 988 /*
 989  * Outer subroutine:
 990  * Connect from a socket to a specified address.
 991  * Both address and port must be specified in argument sin.
 992  * If don't have a local address for this socket yet,
 993  * then pick one.
 994  */
 995 int
 996 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
 997 {
 998         struct sockaddr_in *if_sin;
 999         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
1000         int error;
1001
1002         /* Call inner routine to assign local interface address. */
1003         if ((error = in_pcbladdr(inp, nam, &if_sin, td)) != 0)
1004                 return (error);
1005
1006         if (in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port,
1007                               inp->inp_laddr.s_addr ?
1008                                 inp->inp_laddr : if_sin->sin_addr,
1009                               inp->inp_lport, FALSE, NULL) != NULL) {
1010                 return (EADDRINUSE);
1011         }
1012         if (inp->inp_laddr.s_addr == INADDR_ANY) {
1013                 if (inp->inp_lport == 0) {
1014                         error = in_pcbbind(inp, NULL, td);
1015                         if (error)
1016                                 return (error);
1017                 }
1018                 inp->inp_laddr = if_sin->sin_addr;
1019         }
1020         inp->inp_faddr = sin->sin_addr;
1021         inp->inp_fport = sin->sin_port;
1022         in_pcbinsconnhash(inp);
1023         return (0);
1024 }
1025
1026 void
1027 in_pcbdisconnect(struct inpcb *inp)
1028 {
1029
1030         in_pcbremconnhash(inp);
1031         inp->inp_faddr.s_addr = INADDR_ANY;
1032         inp->inp_fport = 0;
1033 }
1034
1035 void
1036 in_pcbdetach(struct inpcb *inp)
1037 {
1038         struct socket *so = inp->inp_socket;
1039         struct inpcbinfo *ipi = inp->inp_pcbinfo;
1040
1041 #ifdef IPSEC
1042         ipsec4_delete_pcbpolicy(inp);
1043 #endif /*IPSEC*/
1044         inp->inp_gencnt = ++ipi->ipi_gencnt;
1045         KKASSERT((so->so_state & SS_ASSERTINPROG) == 0);
1046         in_pcbremlists(inp);
1047         so->so_pcb = NULL;
1048         sofree(so);                     /* remove pcb ref */
1049         if (inp->inp_options)
1050                 m_free(inp->inp_options);
1051         if (inp->inp_route.ro_rt)
1052                 rtfree(inp->inp_route.ro_rt);
1053         ip_freemoptions(inp->inp_moptions);
1054         kfree(inp, M_PCB);
1055 }
1056
1057 /*
1058  * The calling convention of in_setsockaddr() and in_setpeeraddr() was
1059  * modified to match the pru_sockaddr() and pru_peeraddr() entry points
1060  * in struct pr_usrreqs, so that protocols can just reference then directly
1061  * without the need for a wrapper function.  The socket must have a valid
1062  * (i.e., non-nil) PCB, but it should be impossible to get an invalid one
1063  * except through a kernel programming error, so it is acceptable to panic
1064  * (or in this case trap) if the PCB is invalid.  (Actually, we don't trap
1065  * because there actually /is/ a programming error somewhere... XXX)
1066  */
1067 int
1068 in_setsockaddr(struct socket *so, struct sockaddr **nam)
1069 {
1070         struct inpcb *inp;
1071         struct sockaddr_in *sin;
1072
1073         /*
1074          * Do the malloc first in case it blocks.
1075          */
1076         sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1077         sin->sin_family = AF_INET;
1078         sin->sin_len = sizeof *sin;
1079
1080         crit_enter();
1081         inp = so->so_pcb;
1082         if (!inp) {
1083                 crit_exit();
1084                 kfree(sin, M_SONAME);
1085                 return (ECONNRESET);
1086         }
1087         sin->sin_port = inp->inp_lport;
1088         sin->sin_addr = inp->inp_laddr;
1089         crit_exit();
1090
1091         *nam = (struct sockaddr *)sin;
1092         return (0);
1093 }
1094
1095 void
1096 in_setsockaddr_dispatch(netmsg_t msg)
1097 {
1098         int error;
1099
1100         error = in_setsockaddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1101         lwkt_replymsg(&msg->lmsg, error);
1102 }
1103
1104 int
1105 in_setpeeraddr(struct socket *so, struct sockaddr **nam)
1106 {
1107         struct inpcb *inp;
1108         struct sockaddr_in *sin;
1109
1110         /*
1111          * Do the malloc first in case it blocks.
1112          */
1113         sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1114         sin->sin_family = AF_INET;
1115         sin->sin_len = sizeof *sin;
1116
1117         crit_enter();
1118         inp = so->so_pcb;
1119         if (!inp) {
1120                 crit_exit();
1121                 kfree(sin, M_SONAME);
1122                 return (ECONNRESET);
1123         }
1124         sin->sin_port = inp->inp_fport;
1125         sin->sin_addr = inp->inp_faddr;
1126         crit_exit();
1127
1128         *nam = (struct sockaddr *)sin;
1129         return (0);
1130 }
1131
1132 void
1133 in_setpeeraddr_dispatch(netmsg_t msg)
1134 {
1135         int error;
1136
1137         error = in_setpeeraddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1138         lwkt_replymsg(&msg->lmsg, error);
1139 }
1140
1141 void
1142 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int err,
1143     inp_notify_t notify)
1144 {
1145         struct inpcb *inp, *marker;
1146
1147         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1148             ("not in the correct netisr"));
1149         marker = &in_pcbmarkers[mycpuid];
1150
1151         /*
1152          * NOTE:
1153          * - If INP_PLACEMARKER is set we must ignore the rest of the
1154          *   structure and skip it.
1155          * - It is safe to nuke inpcbs here, since we are in their own
1156          *   netisr.
1157          */
1158         GET_PCBINFO_TOKEN(pcbinfo);
1159
1160         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1161         while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1162                 LIST_REMOVE(marker, inp_list);
1163                 LIST_INSERT_AFTER(inp, marker, inp_list);
1164
1165                 if (inp->inp_flags & INP_PLACEMARKER)
1166                         continue;
1167 #ifdef INET6
1168                 if (!INP_ISIPV4(inp))
1169                         continue;
1170 #endif
1171                 if (inp->inp_faddr.s_addr != faddr.s_addr ||
1172                     inp->inp_socket == NULL)
1173                         continue;
1174                 (*notify)(inp, err);            /* can remove inp from list! */
1175         }
1176         LIST_REMOVE(marker, inp_list);
1177
1178         REL_PCBINFO_TOKEN(pcbinfo);
1179 }
1180
1181 void
1182 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1183 {
1184         struct inpcb *inp, *marker;
1185
1186         /*
1187          * We only need to make sure that we are in netisr0, where all
1188          * multicast operation happen.  We could check inpcbinfo which
1189          * does not belong to netisr0 by holding the inpcbinfo's token.
1190          * In this case, the pcbinfo must be able to be shared, i.e.
1191          * pcbinfo->infotoken is not NULL.
1192          */
1193         KASSERT(&curthread->td_msgport == netisr_cpuport(0),
1194             ("not in netisr0"));
1195         KASSERT(pcbinfo->cpu == 0 || pcbinfo->infotoken != NULL,
1196             ("pcbinfo could not be shared"));
1197
1198         /*
1199          * Get a marker for the current netisr (netisr0).
1200          *
1201          * It is possible that the multicast address deletion blocks,
1202          * which could cause temporary token releasing.  So we use
1203          * inpcb marker here to get a coherent view of the inpcb list.
1204          *
1205          * While, on the other hand, moptions are only added and deleted
1206          * in netisr0, so we would not see staled moption or miss moption
1207          * even if the token was released due to the blocking multicast
1208          * address deletion.
1209          */
1210         marker = &in_pcbmarkers[mycpuid];
1211
1212         GET_PCBINFO_TOKEN(pcbinfo);
1213
1214         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1215         while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1216                 struct ip_moptions *imo;
1217
1218                 LIST_REMOVE(marker, inp_list);
1219                 LIST_INSERT_AFTER(inp, marker, inp_list);
1220
1221                 if (inp->inp_flags & INP_PLACEMARKER)
1222                         continue;
1223                 imo = inp->inp_moptions;
1224                 if (INP_ISIPV4(inp) && imo != NULL) {
1225                         int i, gap;
1226
1227                         /*
1228                          * Unselect the outgoing interface if it is being
1229                          * detached.
1230                          */
1231                         if (imo->imo_multicast_ifp == ifp)
1232                                 imo->imo_multicast_ifp = NULL;
1233
1234                         /*
1235                          * Drop multicast group membership if we joined
1236                          * through the interface being detached.
1237                          */
1238                         for (i = 0, gap = 0; i < imo->imo_num_memberships;
1239                             i++) {
1240                                 if (imo->imo_membership[i]->inm_ifp == ifp) {
1241                                         /*
1242                                          * NOTE:
1243                                          * This could block and the pcbinfo
1244                                          * token could be passively released.
1245                                          */
1246                                         in_delmulti(imo->imo_membership[i]);
1247                                         gap++;
1248                                 } else if (gap != 0)
1249                                         imo->imo_membership[i - gap] =
1250                                             imo->imo_membership[i];
1251                         }
1252                         imo->imo_num_memberships -= gap;
1253                 }
1254         }
1255         LIST_REMOVE(marker, inp_list);
1256
1257         REL_PCBINFO_TOKEN(pcbinfo);
1258 }
1259
1260 /*
1261  * Check for alternatives when higher level complains
1262  * about service problems.  For now, invalidate cached
1263  * routing information.  If the route was created dynamically
1264  * (by a redirect), time to try a default gateway again.
1265  */
1266 void
1267 in_losing(struct inpcb *inp)
1268 {
1269         struct rtentry *rt;
1270         struct rt_addrinfo rtinfo;
1271
1272         if ((rt = inp->inp_route.ro_rt)) {
1273                 bzero(&rtinfo, sizeof(struct rt_addrinfo));
1274                 rtinfo.rti_info[RTAX_DST] = rt_key(rt);
1275                 rtinfo.rti_info[RTAX_GATEWAY] = rt->rt_gateway;
1276                 rtinfo.rti_info[RTAX_NETMASK] = rt_mask(rt);
1277                 rtinfo.rti_flags = rt->rt_flags;
1278                 rt_missmsg(RTM_LOSING, &rtinfo, rt->rt_flags, 0);
1279                 if (rt->rt_flags & RTF_DYNAMIC) {
1280                         rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway,
1281                             rt_mask(rt), rt->rt_flags, NULL);
1282                 }
1283                 inp->inp_route.ro_rt = NULL;
1284                 rtfree(rt);
1285                 /*
1286                  * A new route can be allocated
1287                  * the next time output is attempted.
1288                  */
1289         }
1290 }
1291
1292 /*
1293  * After a routing change, flush old routing
1294  * and allocate a (hopefully) better one.
1295  */
1296 void
1297 in_rtchange(struct inpcb *inp, int err)
1298 {
1299         if (inp->inp_route.ro_rt) {
1300                 rtfree(inp->inp_route.ro_rt);
1301                 inp->inp_route.ro_rt = NULL;
1302                 /*
1303                  * A new route can be allocated the next time
1304                  * output is attempted.
1305                  */
1306         }
1307 }
1308
1309 /*
1310  * Lookup a PCB based on the local address and port.
1311  */
1312 struct inpcb *
1313 in_pcblookup_local(struct inpcbportinfo *portinfo, struct in_addr laddr,
1314                    u_int lport_arg, int wild_okay, struct ucred *cred)
1315 {
1316         struct inpcb *inp;
1317         int matchwild = 3, wildcard;
1318         u_short lport = lport_arg;
1319         struct inpcbporthead *porthash;
1320         struct inpcbport *phd;
1321         struct inpcb *match = NULL;
1322
1323         /*
1324          * If the porthashbase is shared across several cpus, it must
1325          * have been locked.
1326          */
1327         ASSERT_PORT_TOKEN_HELD(portinfo);
1328
1329         /*
1330          * Best fit PCB lookup.
1331          *
1332          * First see if this local port is in use by looking on the
1333          * port hash list.
1334          */
1335         porthash = &portinfo->porthashbase[
1336                         INP_PCBPORTHASH(lport, portinfo->porthashmask)];
1337         LIST_FOREACH(phd, porthash, phd_hash) {
1338                 if (phd->phd_port == lport)
1339                         break;
1340         }
1341         if (phd != NULL) {
1342                 /*
1343                  * Port is in use by one or more PCBs. Look for best
1344                  * fit.
1345                  */
1346                 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
1347                         wildcard = 0;
1348 #ifdef INET6
1349                         if (!INP_ISIPV4(inp))
1350                                 continue;
1351 #endif
1352                         if (inp->inp_faddr.s_addr != INADDR_ANY)
1353                                 wildcard++;
1354                         if (inp->inp_laddr.s_addr != INADDR_ANY) {
1355                                 if (laddr.s_addr == INADDR_ANY)
1356                                         wildcard++;
1357                                 else if (inp->inp_laddr.s_addr != laddr.s_addr)
1358                                         continue;
1359                         } else {
1360                                 if (laddr.s_addr != INADDR_ANY)
1361                                         wildcard++;
1362                         }
1363                         if (wildcard && !wild_okay)
1364                                 continue;
1365                         if (wildcard < matchwild &&
1366                             (cred == NULL ||
1367                              cred->cr_prison ==
1368                                         inp->inp_socket->so_cred->cr_prison)) {
1369                                 match = inp;
1370                                 matchwild = wildcard;
1371                                 if (matchwild == 0) {
1372                                         break;
1373                                 }
1374                         }
1375                 }
1376         }
1377         return (match);
1378 }
1379
1380 struct inpcb *
1381 in_pcblocalgroup_last(const struct inpcbinfo *pcbinfo,
1382     const struct inpcb *inp)
1383 {
1384         const struct inp_localgrphead *hdr;
1385         const struct inp_localgroup *grp;
1386         int i;
1387
1388         if (pcbinfo->localgrphashbase == NULL)
1389                 return NULL;
1390
1391         GET_PCBINFO_TOKEN(pcbinfo);
1392
1393         hdr = &pcbinfo->localgrphashbase[
1394             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
1395
1396         LIST_FOREACH(grp, hdr, il_list) {
1397                 if (grp->il_af == inp->inp_af &&
1398                     grp->il_lport == inp->inp_lport &&
1399                     memcmp(&grp->il_dependladdr,
1400                         &inp->inp_inc.inc_ie.ie_dependladdr,
1401                         sizeof(grp->il_dependladdr)) == 0) {
1402                         break;
1403                 }
1404         }
1405         if (grp == NULL || grp->il_inpcnt == 1) {
1406                 REL_PCBINFO_TOKEN(pcbinfo);
1407                 return NULL;
1408         }
1409
1410         KASSERT(grp->il_inpcnt >= 2,
1411             ("invalid localgroup inp count %d", grp->il_inpcnt));
1412         for (i = 0; i < grp->il_inpcnt; ++i) {
1413                 if (grp->il_inp[i] == inp) {
1414                         int last = grp->il_inpcnt - 1;
1415
1416                         if (i == last)
1417                                 last = grp->il_inpcnt - 2;
1418                         REL_PCBINFO_TOKEN(pcbinfo);
1419                         return grp->il_inp[last];
1420                 }
1421         }
1422         REL_PCBINFO_TOKEN(pcbinfo);
1423         return NULL;
1424 }
1425
1426 static struct inpcb *
1427 inp_localgroup_lookup(const struct inpcbinfo *pcbinfo,
1428     struct in_addr laddr, uint16_t lport, uint32_t pkt_hash)
1429 {
1430         struct inpcb *local_wild = NULL;
1431         const struct inp_localgrphead *hdr;
1432         const struct inp_localgroup *grp;
1433
1434         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1435
1436         hdr = &pcbinfo->localgrphashbase[
1437             INP_PCBLOCALGRPHASH(lport, pcbinfo->localgrphashmask)];
1438
1439         /*
1440          * Order of socket selection:
1441          * 1. non-wild.
1442          * 2. wild.
1443          *
1444          * NOTE: Local group does not contain jailed sockets
1445          */
1446         LIST_FOREACH(grp, hdr, il_list) {
1447 #ifdef INET6
1448                 if (grp->il_af != AF_INET)
1449                         continue;
1450 #endif
1451                 if (grp->il_lport == lport) {
1452                         int idx;
1453
1454                         /*
1455                          * Modulo-N is used here, which greatly reduces
1456                          * completion queue token contention, thus more
1457                          * cpu time is saved.
1458                          */
1459                         idx = pkt_hash % grp->il_inpcnt;
1460                         if (grp->il_laddr.s_addr == laddr.s_addr)
1461                                 return grp->il_inp[idx];
1462                         else if (grp->il_laddr.s_addr == INADDR_ANY)
1463                                 local_wild = grp->il_inp[idx];
1464                 }
1465         }
1466         if (local_wild != NULL)
1467                 return local_wild;
1468         return NULL;
1469 }
1470
1471 /*
1472  * Lookup PCB in hash list.
1473  */
1474 struct inpcb *
1475 in_pcblookup_pkthash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1476     u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1477     boolean_t wildcard, struct ifnet *ifp, const struct mbuf *m)
1478 {
1479         struct inpcbhead *head;
1480         struct inpcb *inp, *jinp=NULL;
1481         u_short fport = fport_arg, lport = lport_arg;
1482
1483         /*
1484          * First look for an exact match.
1485          */
1486         head = &pcbinfo->hashbase[INP_PCBCONNHASH(faddr.s_addr, fport,
1487             laddr.s_addr, lport, pcbinfo->hashmask)];
1488         LIST_FOREACH(inp, head, inp_hash) {
1489 #ifdef INET6
1490                 if (!INP_ISIPV4(inp))
1491                         continue;
1492 #endif
1493                 if (in_hosteq(inp->inp_faddr, faddr) &&
1494                     in_hosteq(inp->inp_laddr, laddr) &&
1495                     inp->inp_fport == fport && inp->inp_lport == lport) {
1496                         /* found */
1497                         if (inp->inp_socket == NULL ||
1498                             inp->inp_socket->so_cred->cr_prison == NULL) {
1499                                 return (inp);
1500                         } else {
1501                                 if  (jinp == NULL)
1502                                         jinp = inp;
1503                         }
1504                 }
1505         }
1506         if (jinp != NULL)
1507                 return (jinp);
1508
1509         if (wildcard) {
1510                 struct inpcb *local_wild = NULL;
1511                 struct inpcb *jinp_wild = NULL;
1512                 struct inpcontainer *ic;
1513                 struct inpcontainerhead *chead;
1514                 struct sockaddr_in jsin;
1515                 struct ucred *cred;
1516
1517                 GET_PCBINFO_TOKEN(pcbinfo);
1518
1519                 /*
1520                  * Check local group first
1521                  */
1522                 if (pcbinfo->localgrphashbase != NULL &&
1523                     m != NULL && (m->m_flags & M_HASH) &&
1524                     !(ifp && ifp->if_type == IFT_FAITH)) {
1525                         inp = inp_localgroup_lookup(pcbinfo,
1526                             laddr, lport, m->m_pkthdr.hash);
1527                         if (inp != NULL) {
1528                                 REL_PCBINFO_TOKEN(pcbinfo);
1529                                 return inp;
1530                         }
1531                 }
1532
1533                 /*
1534                  * Order of socket selection:
1535                  * 1. non-jailed, non-wild.
1536                  * 2. non-jailed, wild.
1537                  * 3. jailed, non-wild.
1538                  * 4. jailed, wild.
1539                  */
1540                 jsin.sin_family = AF_INET;
1541                 chead = &pcbinfo->wildcardhashbase[
1542                     INP_PCBWILDCARDHASH(lport, pcbinfo->wildcardhashmask)];
1543                 LIST_FOREACH(ic, chead, ic_list) {
1544                         inp = ic->ic_inp;
1545                         if (inp->inp_flags & INP_PLACEMARKER)
1546                                 continue;
1547
1548                         jsin.sin_addr.s_addr = laddr.s_addr;
1549 #ifdef INET6
1550                         if (!INP_ISIPV4(inp))
1551                                 continue;
1552 #endif
1553                         if (inp->inp_socket != NULL)
1554                                 cred = inp->inp_socket->so_cred;
1555                         else
1556                                 cred = NULL;
1557                         if (cred != NULL && jailed(cred)) {
1558                                 if (jinp != NULL)
1559                                         continue;
1560                                 else
1561                                         if (!jailed_ip(cred->cr_prison,
1562                                             (struct sockaddr *)&jsin))
1563                                                 continue;
1564                         }
1565                         if (inp->inp_lport == lport) {
1566                                 if (ifp && ifp->if_type == IFT_FAITH &&
1567                                     !(inp->inp_flags & INP_FAITH))
1568                                         continue;
1569                                 if (inp->inp_laddr.s_addr == laddr.s_addr) {
1570                                         if (cred != NULL && jailed(cred)) {
1571                                                 jinp = inp;
1572                                         } else {
1573                                                 REL_PCBINFO_TOKEN(pcbinfo);
1574                                                 return (inp);
1575                                         }
1576                                 }
1577                                 if (inp->inp_laddr.s_addr == INADDR_ANY) {
1578                                         if (cred != NULL && jailed(cred))
1579                                                 jinp_wild = inp;
1580                                         else
1581                                                 local_wild = inp;
1582                                 }
1583                         }
1584                 }
1585
1586                 REL_PCBINFO_TOKEN(pcbinfo);
1587
1588                 if (local_wild != NULL)
1589                         return (local_wild);
1590                 if (jinp != NULL)
1591                         return (jinp);
1592                 return (jinp_wild);
1593         }
1594
1595         /*
1596          * Not found.
1597          */
1598         return (NULL);
1599 }
1600
1601 struct inpcb *
1602 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1603     u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1604     boolean_t wildcard, struct ifnet *ifp)
1605 {
1606         return in_pcblookup_pkthash(pcbinfo, faddr, fport_arg,
1607             laddr, lport_arg, wildcard, ifp, NULL);
1608 }
1609
1610 /*
1611  * Insert PCB into connection hash table.
1612  */
1613 void
1614 in_pcbinsconnhash(struct inpcb *inp)
1615 {
1616         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1617         struct inpcbhead *bucket;
1618         u_int32_t hashkey_faddr, hashkey_laddr;
1619
1620 #ifdef INET6
1621         if (INP_ISIPV6(inp)) {
1622                 hashkey_faddr = inp->in6p_faddr.s6_addr32[3] /* XXX JH */;
1623                 hashkey_laddr = inp->in6p_laddr.s6_addr32[3] /* XXX JH */;
1624         } else {
1625 #endif
1626                 hashkey_faddr = inp->inp_faddr.s_addr;
1627                 hashkey_laddr = inp->inp_laddr.s_addr;
1628 #ifdef INET6
1629         }
1630 #endif
1631
1632         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1633             ("not in the correct netisr"));
1634         ASSERT_INP_NOTINHASH(inp);
1635         inp->inp_flags |= INP_CONNECTED;
1636
1637         /*
1638          * Insert into the connection hash table.
1639          */
1640         bucket = &pcbinfo->hashbase[INP_PCBCONNHASH(hashkey_faddr,
1641             inp->inp_fport, hashkey_laddr, inp->inp_lport, pcbinfo->hashmask)];
1642         LIST_INSERT_HEAD(bucket, inp, inp_hash);
1643 }
1644
1645 /*
1646  * Remove PCB from connection hash table.
1647  */
1648 void
1649 in_pcbremconnhash(struct inpcb *inp)
1650 {
1651         struct inpcbinfo *pcbinfo __debugvar = inp->inp_pcbinfo;
1652
1653         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1654             ("not in the correct netisr"));
1655         KASSERT(inp->inp_flags & INP_CONNECTED, ("inp not connected"));
1656
1657         LIST_REMOVE(inp, inp_hash);
1658         inp->inp_flags &= ~INP_CONNECTED;
1659 }
1660
1661 /*
1662  * Insert PCB into port hash table.
1663  */
1664 void
1665 in_pcbinsporthash(struct inpcbportinfo *portinfo, struct inpcb *inp)
1666 {
1667         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1668         struct inpcbporthead *pcbporthash;
1669         struct inpcbport *phd;
1670
1671         /*
1672          * If the porthashbase is shared across several cpus, it must
1673          * have been locked.
1674          */
1675         ASSERT_PORT_TOKEN_HELD(portinfo);
1676
1677         /*
1678          * Insert into the port hash table.
1679          */
1680         pcbporthash = &portinfo->porthashbase[
1681             INP_PCBPORTHASH(inp->inp_lport, portinfo->porthashmask)];
1682
1683         /* Go through port list and look for a head for this lport. */
1684         LIST_FOREACH(phd, pcbporthash, phd_hash) {
1685                 if (phd->phd_port == inp->inp_lport)
1686                         break;
1687         }
1688
1689         /* If none exists, use saved one and tack it on. */
1690         if (phd == NULL) {
1691                 KKASSERT(pcbinfo->portsave != NULL);
1692                 phd = pcbinfo->portsave;
1693                 pcbinfo->portsave = NULL;
1694                 phd->phd_port = inp->inp_lport;
1695                 LIST_INIT(&phd->phd_pcblist);
1696                 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
1697         }
1698
1699         inp->inp_portinfo = portinfo;
1700         inp->inp_phd = phd;
1701         LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
1702
1703         /*
1704          * Malloc one inpcbport for later use.  It is safe to use
1705          * "wait" malloc here (port token would be released, if
1706          * malloc ever blocked), since all changes to the porthash
1707          * are done.
1708          */
1709         if (pcbinfo->portsave == NULL) {
1710                 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave),
1711                                             M_PCB, M_INTWAIT | M_ZERO);
1712         }
1713 }
1714
1715 void
1716 in_pcbinsporthash_lport(struct inpcb *inp)
1717 {
1718         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1719         struct inpcbportinfo *portinfo;
1720         u_short lport_ho;
1721
1722         /* Locate the proper portinfo based on lport */
1723         lport_ho = ntohs(inp->inp_lport);
1724         portinfo = &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask];
1725         KKASSERT((lport_ho & pcbinfo->portinfo_mask) == portinfo->offset);
1726
1727         GET_PORT_TOKEN(portinfo);
1728         in_pcbinsporthash(portinfo, inp);
1729         REL_PORT_TOKEN(portinfo);
1730 }
1731
1732 static struct inp_localgroup *
1733 inp_localgroup_alloc(u_char af, uint16_t port,
1734     const union in_dependaddr *addr, int size)
1735 {
1736         struct inp_localgroup *grp;
1737
1738         grp = kmalloc(__offsetof(struct inp_localgroup, il_inp[size]),
1739             M_TEMP, M_INTWAIT | M_ZERO);
1740         grp->il_af = af;
1741         grp->il_lport = port;
1742         grp->il_dependladdr = *addr;
1743         grp->il_inpsiz = size;
1744
1745         return grp;
1746 }
1747
1748 static void
1749 inp_localgroup_free(struct inp_localgroup *grp)
1750 {
1751         kfree(grp, M_TEMP);
1752 }
1753
1754 static void
1755 inp_localgroup_destroy(struct inp_localgroup *grp)
1756 {
1757         LIST_REMOVE(grp, il_list);
1758         inp_localgroup_free(grp);
1759 }
1760
1761 static void
1762 inp_localgroup_copy(struct inp_localgroup *grp,
1763     const struct inp_localgroup *old_grp)
1764 {
1765         int i;
1766
1767         KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
1768             ("invalid new local group size %d and old local group count %d",
1769              grp->il_inpsiz, old_grp->il_inpcnt));
1770         for (i = 0; i < old_grp->il_inpcnt; ++i)
1771                 grp->il_inp[i] = old_grp->il_inp[i];
1772         grp->il_inpcnt = old_grp->il_inpcnt;
1773 }
1774
1775 static void
1776 in_pcbinslocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
1777 {
1778         struct inp_localgrphead *hdr;
1779         struct inp_localgroup *grp, *grp_alloc = NULL;
1780         struct ucred *cred;
1781         int i, idx;
1782
1783         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1784
1785         if (pcbinfo->localgrphashbase == NULL)
1786                 return;
1787
1788         /*
1789          * XXX don't allow jailed socket to join local group
1790          */
1791         if (inp->inp_socket != NULL)
1792                 cred = inp->inp_socket->so_cred;
1793         else
1794                 cred = NULL;
1795         if (cred != NULL && jailed(cred))
1796                 return;
1797
1798         hdr = &pcbinfo->localgrphashbase[
1799             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
1800
1801 again:
1802         LIST_FOREACH(grp, hdr, il_list) {
1803                 if (grp->il_af == inp->inp_af &&
1804                     grp->il_lport == inp->inp_lport &&
1805                     memcmp(&grp->il_dependladdr,
1806                         &inp->inp_inc.inc_ie.ie_dependladdr,
1807                         sizeof(grp->il_dependladdr)) == 0) {
1808                         break;
1809                 }
1810         }
1811         if (grp == NULL) {
1812                 /*
1813                  * Create a new local group
1814                  */
1815                 if (grp_alloc == NULL) {
1816                         grp_alloc = inp_localgroup_alloc(inp->inp_af,
1817                             inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
1818                             INP_LOCALGROUP_SIZMIN);
1819                         /*
1820                          * Local group allocation could block and the
1821                          * local group w/ the same property might have
1822                          * been added by others when we were blocked;
1823                          * check again.
1824                          */
1825                         goto again;
1826                 } else {
1827                         /* Local group has been allocated; link it */
1828                         grp = grp_alloc;
1829                         grp_alloc = NULL;
1830                         LIST_INSERT_HEAD(hdr, grp, il_list);
1831                 }
1832         } else if (grp->il_inpcnt == grp->il_inpsiz) {
1833                 if (grp->il_inpsiz >= INP_LOCALGROUP_SIZMAX) {
1834                         static int limit_logged = 0;
1835
1836                         if (!limit_logged) {
1837                                 limit_logged = 1;
1838                                 kprintf("local group port %d, "
1839                                     "limit reached\n", ntohs(grp->il_lport));
1840                         }
1841                         if (grp_alloc != NULL) {
1842                                 /*
1843                                  * This would happen if the local group
1844                                  * w/ the same property was expanded when
1845                                  * our local group allocation blocked.
1846                                  */
1847                                 inp_localgroup_free(grp_alloc);
1848                         }
1849                         return;
1850                 }
1851
1852                 /*
1853                  * Expand this local group
1854                  */
1855                 if (grp_alloc == NULL ||
1856                     grp->il_inpcnt >= grp_alloc->il_inpsiz) {
1857                         if (grp_alloc != NULL)
1858                                 inp_localgroup_free(grp_alloc);
1859                         grp_alloc = inp_localgroup_alloc(grp->il_af,
1860                             grp->il_lport, &grp->il_dependladdr,
1861                             grp->il_inpsiz * 2);
1862                         /*
1863                          * Local group allocation could block and the
1864                          * local group w/ the same property might have
1865                          * been expanded by others when we were blocked;
1866                          * check again.
1867                          */
1868                         goto again;
1869                 }
1870
1871                 /*
1872                  * Save the old local group, link the new one, and then
1873                  * destroy the old local group
1874                  */
1875                 inp_localgroup_copy(grp_alloc, grp);
1876                 LIST_INSERT_HEAD(hdr, grp_alloc, il_list);
1877                 inp_localgroup_destroy(grp);
1878
1879                 grp = grp_alloc;
1880                 grp_alloc = NULL;
1881         } else {
1882                 /*
1883                  * Found the local group
1884                  */
1885                 if (grp_alloc != NULL) {
1886                         /*
1887                          * This would happen if the local group w/ the
1888                          * same property was added or expanded when our
1889                          * local group allocation blocked.
1890                          */
1891                         inp_localgroup_free(grp_alloc);
1892                         grp_alloc = NULL;
1893                 }
1894         }
1895
1896         KASSERT(grp->il_inpcnt < grp->il_inpsiz,
1897             ("invalid local group size %d and count %d",
1898              grp->il_inpsiz, grp->il_inpcnt));
1899
1900         /*
1901          * Keep the local group sorted by the inpcb local group index
1902          * in ascending order.
1903          *
1904          * This eases the multi-process userland application which uses
1905          * SO_REUSEPORT sockets and binds process to the owner cpu of
1906          * the SO_REUSEPORT socket:
1907          * If we didn't sort the local group by the inpcb local group
1908          * index and one of the process owning an inpcb in this local
1909          * group restarted, e.g. crashed and restarted by watchdog,
1910          * other processes owning a inpcb in this local group would have
1911          * to detect that event, refetch its socket's owner cpu, and
1912          * re-bind.
1913          */
1914         idx = grp->il_inpcnt;
1915         for (i = 0; i < idx; ++i) {
1916                 struct inpcb *oinp = grp->il_inp[i];
1917
1918                 if (oinp->inp_lgrpindex > i) {
1919                         if (inp->inp_lgrpindex < 0) {
1920                                 inp->inp_lgrpindex = i;
1921                         } else if (inp->inp_lgrpindex != i) {
1922                                 if (bootverbose) {
1923                                         kprintf("inp %p: grpidx %d, "
1924                                             "assigned to %d, cpu%d\n",
1925                                             inp, inp->inp_lgrpindex, i,
1926                                             mycpuid);
1927                                 }
1928                         }
1929                         grp->il_inp[i] = inp;
1930
1931                         /* Pull down inpcbs */
1932                         for (; i < grp->il_inpcnt; ++i) {
1933                                 struct inpcb *oinp1 = grp->il_inp[i + 1];
1934
1935                                 grp->il_inp[i + 1] = oinp;
1936                                 oinp = oinp1;
1937                         }
1938                         grp->il_inpcnt++;
1939                         return;
1940                 }
1941         }
1942
1943         if (inp->inp_lgrpindex < 0) {
1944                 inp->inp_lgrpindex = idx;
1945         } else if (inp->inp_lgrpindex != idx) {
1946                 if (bootverbose) {
1947                         kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n",
1948                             inp, inp->inp_lgrpindex, idx, mycpuid);
1949                 }
1950         }
1951         grp->il_inp[idx] = inp;
1952         grp->il_inpcnt++;
1953 }
1954
1955 void
1956 in_pcbinswildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
1957 {
1958         struct inpcontainer *ic;
1959         struct inpcontainerhead *bucket;
1960
1961         GET_PCBINFO_TOKEN(pcbinfo);
1962
1963         in_pcbinslocalgrphash_oncpu(inp, pcbinfo);
1964
1965         bucket = &pcbinfo->wildcardhashbase[
1966             INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
1967
1968         ic = kmalloc(sizeof(struct inpcontainer), M_TEMP, M_INTWAIT);
1969         ic->ic_inp = inp;
1970         LIST_INSERT_HEAD(bucket, ic, ic_list);
1971
1972         REL_PCBINFO_TOKEN(pcbinfo);
1973 }
1974
1975 /*
1976  * Insert PCB into wildcard hash table.
1977  */
1978 void
1979 in_pcbinswildcardhash(struct inpcb *inp)
1980 {
1981         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1982
1983         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1984             ("not in correct netisr"));
1985         ASSERT_INP_NOTINHASH(inp);
1986         inp->inp_flags |= INP_WILDCARD;
1987
1988         in_pcbinswildcardhash_oncpu(inp, pcbinfo);
1989 }
1990
1991 static void
1992 in_pcbremlocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
1993 {
1994         struct inp_localgrphead *hdr;
1995         struct inp_localgroup *grp;
1996
1997         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1998
1999         if (pcbinfo->localgrphashbase == NULL)
2000                 return;
2001
2002         hdr = &pcbinfo->localgrphashbase[
2003             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
2004
2005         LIST_FOREACH(grp, hdr, il_list) {
2006                 int i;
2007
2008                 for (i = 0; i < grp->il_inpcnt; ++i) {
2009                         if (grp->il_inp[i] != inp)
2010                                 continue;
2011
2012                         if (grp->il_inpcnt == 1) {
2013                                 /* Destroy this local group */
2014                                 inp_localgroup_destroy(grp);
2015                         } else {
2016                                 /* Pull up inpcbs */
2017                                 for (; i + 1 < grp->il_inpcnt; ++i)
2018                                         grp->il_inp[i] = grp->il_inp[i + 1];
2019                                 grp->il_inpcnt--;
2020                         }
2021                         return;
2022                 }
2023         }
2024 }
2025
2026 void
2027 in_pcbremwildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2028 {
2029         struct inpcontainer *ic;
2030         struct inpcontainerhead *head;
2031
2032         GET_PCBINFO_TOKEN(pcbinfo);
2033
2034         in_pcbremlocalgrphash_oncpu(inp, pcbinfo);
2035
2036         /* find bucket */
2037         head = &pcbinfo->wildcardhashbase[
2038             INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
2039
2040         LIST_FOREACH(ic, head, ic_list) {
2041                 if (ic->ic_inp == inp)
2042                         goto found;
2043         }
2044         REL_PCBINFO_TOKEN(pcbinfo);
2045         return;                 /* not found! */
2046
2047 found:
2048         LIST_REMOVE(ic, ic_list);       /* remove container from bucket chain */
2049         REL_PCBINFO_TOKEN(pcbinfo);
2050         kfree(ic, M_TEMP);              /* deallocate container */
2051 }
2052
2053 /*
2054  * Remove PCB from wildcard hash table.
2055  */
2056 void
2057 in_pcbremwildcardhash(struct inpcb *inp)
2058 {
2059         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2060
2061         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
2062             ("not in correct netisr"));
2063         KASSERT(inp->inp_flags & INP_WILDCARD, ("inp not wildcard"));
2064
2065         in_pcbremwildcardhash_oncpu(inp, pcbinfo);
2066         inp->inp_lgrpindex = -1;
2067         inp->inp_flags &= ~INP_WILDCARD;
2068 }
2069
2070 /*
2071  * Remove PCB from various lists.
2072  */
2073 void
2074 in_pcbremlists(struct inpcb *inp)
2075 {
2076         if (inp->inp_lport) {
2077                 struct inpcbportinfo *portinfo;
2078                 struct inpcbport *phd;
2079
2080                 /*
2081                  * NOTE:
2082                  * inp->inp_portinfo is _not_ necessary same as
2083                  * inp->inp_pcbinfo->portinfo.
2084                  */
2085                 portinfo = inp->inp_portinfo;
2086                 GET_PORT_TOKEN(portinfo);
2087
2088                 phd = inp->inp_phd;
2089                 LIST_REMOVE(inp, inp_portlist);
2090                 if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
2091                         LIST_REMOVE(phd, phd_hash);
2092                         kfree(phd, M_PCB);
2093                 }
2094
2095                 REL_PORT_TOKEN(portinfo);
2096         }
2097         if (inp->inp_flags & INP_WILDCARD) {
2098                 in_pcbremwildcardhash(inp);
2099         } else if (inp->inp_flags & INP_CONNECTED) {
2100                 in_pcbremconnhash(inp);
2101         }
2102
2103         if (inp->inp_flags & INP_ONLIST)
2104                 in_pcbofflist(inp);
2105 }
2106
2107 int
2108 prison_xinpcb(struct thread *td, struct inpcb *inp)
2109 {
2110         struct ucred *cr;
2111
2112         if (td->td_proc == NULL)
2113                 return (0);
2114         cr = td->td_proc->p_ucred;
2115         if (cr->cr_prison == NULL)
2116                 return (0);
2117         if (inp->inp_socket && inp->inp_socket->so_cred &&
2118             inp->inp_socket->so_cred->cr_prison &&
2119             cr->cr_prison == inp->inp_socket->so_cred->cr_prison)
2120                 return (0);
2121         return (1);
2122 }
2123
2124 int
2125 in_pcblist_global(SYSCTL_HANDLER_ARGS)
2126 {
2127         struct inpcbinfo *pcbinfo_arr = arg1;
2128         int pcbinfo_arrlen = arg2;
2129         struct inpcb *marker;
2130         int cpu, origcpu;
2131         int error, n;
2132
2133         KASSERT(pcbinfo_arrlen <= ncpus && pcbinfo_arrlen >= 1,
2134             ("invalid pcbinfo count %d", pcbinfo_arrlen));
2135
2136         /*
2137          * The process of preparing the TCB list is too time-consuming and
2138          * resource-intensive to repeat twice on every request.
2139          */
2140         n = 0;
2141         if (req->oldptr == NULL) {
2142                 for (cpu = 0; cpu < pcbinfo_arrlen; ++cpu)
2143                         n += pcbinfo_arr[cpu].ipi_count;
2144                 req->oldidx = (n + n/8 + 10) * sizeof(struct xinpcb);
2145                 return 0;
2146         }
2147
2148         if (req->newptr != NULL)
2149                 return EPERM;
2150
2151         marker = kmalloc(sizeof(struct inpcb), M_TEMP, M_WAITOK|M_ZERO);
2152         marker->inp_flags |= INP_PLACEMARKER;
2153
2154         /*
2155          * OK, now we're committed to doing something.  Re-fetch ipi_count
2156          * after obtaining the generation count.
2157          */
2158         error = 0;
2159         origcpu = mycpuid;
2160         for (cpu = 0; cpu < pcbinfo_arrlen && error == 0; ++cpu) {
2161                 struct inpcbinfo *pcbinfo = &pcbinfo_arr[cpu];
2162                 struct inpcb *inp;
2163                 struct xinpcb xi;
2164                 int i;
2165
2166                 lwkt_migratecpu(cpu);
2167
2168                 GET_PCBINFO_TOKEN(pcbinfo);
2169
2170                 n = pcbinfo->ipi_count;
2171
2172                 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
2173                 i = 0;
2174                 while ((inp = LIST_NEXT(marker, inp_list)) != NULL && i < n) {
2175                         LIST_REMOVE(marker, inp_list);
2176                         LIST_INSERT_AFTER(inp, marker, inp_list);
2177
2178                         if (inp->inp_flags & INP_PLACEMARKER)
2179                                 continue;
2180                         if (prison_xinpcb(req->td, inp))
2181                                 continue;
2182
2183                         bzero(&xi, sizeof xi);
2184                         xi.xi_len = sizeof xi;
2185                         bcopy(inp, &xi.xi_inp, sizeof *inp);
2186                         if (inp->inp_socket)
2187                                 sotoxsocket(inp->inp_socket, &xi.xi_socket);
2188                         if ((error = SYSCTL_OUT(req, &xi, sizeof xi)) != 0)
2189                                 break;
2190                         ++i;
2191                 }
2192                 LIST_REMOVE(marker, inp_list);
2193
2194                 REL_PCBINFO_TOKEN(pcbinfo);
2195
2196                 if (error == 0 && i < n) {
2197                         bzero(&xi, sizeof xi);
2198                         xi.xi_len = sizeof xi;
2199                         while (i < n) {
2200                                 error = SYSCTL_OUT(req, &xi, sizeof xi);
2201                                 if (error)
2202                                         break;
2203                                 ++i;
2204                         }
2205                 }
2206         }
2207
2208         lwkt_migratecpu(origcpu);
2209         kfree(marker, M_TEMP);
2210         return error;
2211 }
2212
2213 int
2214 in_pcblist_global_ncpus2(SYSCTL_HANDLER_ARGS)
2215 {
2216         return in_pcblist_global(oidp, arg1, ncpus2, req);
2217 }
2218
2219 void
2220 in_savefaddr(struct socket *so, const struct sockaddr *faddr)
2221 {
2222         struct sockaddr_in *sin;
2223
2224         KASSERT(faddr->sa_family == AF_INET,
2225             ("not AF_INET faddr %d", faddr->sa_family));
2226
2227         sin = kmalloc(sizeof(*sin), M_SONAME, M_WAITOK | M_ZERO);
2228         sin->sin_family = AF_INET;
2229         sin->sin_len = sizeof(*sin);
2230         sin->sin_port = ((const struct sockaddr_in *)faddr)->sin_port;
2231         sin->sin_addr = ((const struct sockaddr_in *)faddr)->sin_addr;
2232
2233         so->so_faddr = (struct sockaddr *)sin;
2234 }
2235
2236 void
2237 in_pcbportinfo_init(struct inpcbportinfo *portinfo, int hashsize,
2238     boolean_t shared, u_short offset)
2239 {
2240         memset(portinfo, 0, sizeof(*portinfo));
2241
2242         portinfo->offset = offset;
2243         portinfo->lastport = offset;
2244         portinfo->lastlow = offset;
2245         portinfo->lasthi = offset;
2246
2247         portinfo->porthashbase = hashinit(hashsize, M_PCB,
2248             &portinfo->porthashmask);
2249
2250         if (shared) {
2251                 portinfo->porttoken = kmalloc(sizeof(struct lwkt_token),
2252                     M_PCB, M_WAITOK);
2253                 lwkt_token_init(portinfo->porttoken, "porttoken");
2254         }
2255 }
2256
2257 void
2258 in_pcbportrange(u_short *hi0, u_short *lo0, u_short ofs, u_short step)
2259 {
2260         int hi, lo;
2261
2262         if (step == 1)
2263                 return;
2264
2265         hi = *hi0;
2266         lo = *lo0;
2267
2268         hi = rounddown2(hi, step);
2269         hi += ofs;
2270         if (hi > (int)*hi0)
2271                 hi -= step;
2272
2273         lo = roundup2(lo, step);
2274         lo -= (step - ofs);
2275         if (lo < (int)*lo0)
2276                 lo += step;
2277
2278         *hi0 = hi;
2279         *lo0 = lo;
2280 }
2281
2282 void
2283 in_pcbglobalinit(void)
2284 {
2285         int cpu;
2286
2287         in_pcbmarkers = kmalloc(ncpus * sizeof(struct inpcb), M_PCB,
2288             M_WAITOK | M_ZERO);
2289         in_pcbcontainer_markers = kmalloc(ncpus * sizeof(struct inpcontainer),
2290             M_PCB, M_WAITOK | M_ZERO);
2291
2292         for (cpu = 0; cpu < ncpus; ++cpu) {
2293                 struct inpcontainer *ic = &in_pcbcontainer_markers[cpu];
2294                 struct inpcb *marker = &in_pcbmarkers[cpu];
2295
2296                 marker->inp_flags |= INP_PLACEMARKER;
2297                 ic->ic_inp = marker;
2298         }
2299 }
2300
2301 struct inpcb *
2302 in_pcbmarker(int cpuid)
2303 {
2304         KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid));
2305         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
2306
2307         return &in_pcbmarkers[cpuid];
2308 }
2309
2310 struct inpcontainer *
2311 in_pcbcontainer_marker(int cpuid)
2312 {
2313         KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid));
2314         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
2315
2316         return &in_pcbcontainer_markers[cpuid];
2317 }
2318
2319 void
2320 in_pcbresetroute(struct inpcb *inp)
2321 {
2322         struct route *ro = &inp->inp_route;
2323
2324         if (ro->ro_rt != NULL)
2325                 RTFREE(ro->ro_rt);
2326         bzero(ro, sizeof(*ro));
2327 }