sys/netinet/in_pcb.c

   1 /*
   2  * Copyright (c) 2004 Jeffrey M. Hsu.  All rights reserved.
   3  * Copyright (c) 2004 The DragonFly Project.  All rights reserved.
   4  *
   5  * This code is derived from software contributed to The DragonFly Project
   6  * by Jeffrey M. Hsu.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the name of The DragonFly Project nor the names of its
  17  *    contributors may be used to endorse or promote products derived
  18  *    from this software without specific, prior written permission.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  23  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  24  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  25  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  26  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  27  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  28  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  29  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  30  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  */
  33
  34 /*
  35  * Copyright (c) 1982, 1986, 1991, 1993, 1995
  36  *      The Regents of the University of California.  All rights reserved.
  37  *
  38  * Redistribution and use in source and binary forms, with or without
  39  * modification, are permitted provided that the following conditions
  40  * are met:
  41  * 1. Redistributions of source code must retain the above copyright
  42  *    notice, this list of conditions and the following disclaimer.
  43  * 2. Redistributions in binary form must reproduce the above copyright
  44  *    notice, this list of conditions and the following disclaimer in the
  45  *    documentation and/or other materials provided with the distribution.
  46  * 3. Neither the name of the University nor the names of its contributors
  47  *    may be used to endorse or promote products derived from this software
  48  *    without specific prior written permission.
  49  *
  50  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  51  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  52  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  53  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  54  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  55  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  56  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  57  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  58  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  59  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  60  * SUCH DAMAGE.
  61  *
  62  *      @(#)in_pcb.c    8.4 (Berkeley) 5/24/95
  63  * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $
  64  */
  65
  66 #include "opt_ipsec.h"
  67 #include "opt_inet6.h"
  68
  69 #include <sys/param.h>
  70 #include <sys/systm.h>
  71 #include <sys/malloc.h>
  72 #include <sys/mbuf.h>
  73 #include <sys/domain.h>
  74 #include <sys/protosw.h>
  75 #include <sys/socket.h>
  76 #include <sys/socketvar.h>
  77 #include <sys/proc.h>
  78 #include <sys/priv.h>
  79 #include <sys/jail.h>
  80 #include <sys/kernel.h>
  81 #include <sys/sysctl.h>
  82
  83 #include <sys/thread2.h>
  84 #include <sys/socketvar2.h>
  85 #include <sys/msgport2.h>
  86
  87 #include <machine/limits.h>
  88
  89 #include <net/if.h>
  90 #include <net/if_types.h>
  91 #include <net/route.h>
  92 #include <net/netisr2.h>
  93
  94 #include <netinet/in.h>
  95 #include <netinet/in_pcb.h>
  96 #include <netinet/in_var.h>
  97 #include <netinet/ip_var.h>
  98 #ifdef INET6
  99 #include <netinet/ip6.h>
 100 #include <netinet6/ip6_var.h>
 101 #endif /* INET6 */
 102
 103 #ifdef IPSEC
 104 #include <netinet6/ipsec.h>
 105 #include <netproto/key/key.h>
 106 #include <netproto/ipsec/esp_var.h>
 107 #endif
 108
 109 #ifdef FAST_IPSEC
 110 #if defined(IPSEC) || defined(IPSEC_ESP)
 111 #error "Bad idea: don't compile with both IPSEC and FAST_IPSEC!"
 112 #endif
 113
 114 #include <netproto/ipsec/ipsec.h>
 115 #include <netproto/ipsec/key.h>
 116 #define IPSEC
 117 #endif /* FAST_IPSEC */
 118
 119 #define INP_LOCALGROUP_SIZMIN   8
 120 #define INP_LOCALGROUP_SIZMAX   256
 121
 122 struct in_addr zeroin_addr;
 123
 124 /*
 125  * These configure the range of local port addresses assigned to
 126  * "unspecified" outgoing connections/packets/whatever.
 127  */
 128 int ipport_lowfirstauto = IPPORT_RESERVED - 1;  /* 1023 */
 129 int ipport_lowlastauto = IPPORT_RESERVEDSTART;  /* 600 */
 130
 131 int ipport_firstauto = IPPORT_RESERVED;         /* 1024 */
 132 int ipport_lastauto = IPPORT_USERRESERVED;      /* 5000 */
 133
 134 int ipport_hifirstauto = IPPORT_HIFIRSTAUTO;    /* 49152 */
 135 int ipport_hilastauto = IPPORT_HILASTAUTO;      /* 65535 */
 136
 137 #define RANGECHK(var, min, max) \
 138         if ((var) < (min)) { (var) = (min); } \
 139         else if ((var) > (max)) { (var) = (max); }
 140
 141 int udpencap_enable = 1;        /* enabled by default */
 142 int udpencap_port = 4500;       /* triggers decapsulation */
 143
 144 /*
 145  * Per-netisr inpcb markers.
 146  * NOTE: they should only be used in netisrs.
 147  */
 148 static struct inpcb             *in_pcbmarkers;
 149 static struct inpcontainer      *in_pcbcontainer_markers;
 150
 151 static int
 152 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
 153 {
 154         int error;
 155
 156         error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
 157         if (!error) {
 158                 RANGECHK(ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
 159                 RANGECHK(ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
 160
 161                 RANGECHK(ipport_firstauto, IPPORT_RESERVED, USHRT_MAX);
 162                 RANGECHK(ipport_lastauto, IPPORT_RESERVED, USHRT_MAX);
 163
 164                 RANGECHK(ipport_hifirstauto, IPPORT_RESERVED, USHRT_MAX);
 165                 RANGECHK(ipport_hilastauto, IPPORT_RESERVED, USHRT_MAX);
 166         }
 167         return (error);
 168 }
 169
 170 #undef RANGECHK
 171
 172 SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports");
 173
 174 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW,
 175            &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", "");
 176 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, CTLTYPE_INT|CTLFLAG_RW,
 177            &ipport_lowlastauto, 0, &sysctl_net_ipport_check, "I", "");
 178 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, CTLTYPE_INT|CTLFLAG_RW,
 179            &ipport_firstauto, 0, &sysctl_net_ipport_check, "I", "");
 180 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, CTLTYPE_INT|CTLFLAG_RW,
 181            &ipport_lastauto, 0, &sysctl_net_ipport_check, "I", "");
 182 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, CTLTYPE_INT|CTLFLAG_RW,
 183            &ipport_hifirstauto, 0, &sysctl_net_ipport_check, "I", "");
 184 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW,
 185            &ipport_hilastauto, 0, &sysctl_net_ipport_check, "I", "");
 186
 187 /*
 188  * in_pcb.c: manage the Protocol Control Blocks.
 189  *
 190  * NOTE: It is assumed that most of these functions will be called from
 191  * a critical section.  XXX - There are, unfortunately, a few exceptions
 192  * to this rule that should be fixed.
 193  *
 194  * NOTE: The caller should initialize the cpu field to the cpu running the
 195  * protocol stack associated with this inpcbinfo.
 196  */
 197
 198 void
 199 in_pcbinfo_init(struct inpcbinfo *pcbinfo, int cpu, boolean_t shared)
 200 {
 201         KASSERT(cpu >= 0 && cpu < ncpus, ("invalid cpu%d", cpu));
 202         pcbinfo->cpu = cpu;
 203
 204         LIST_INIT(&pcbinfo->pcblisthead);
 205         pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), M_PCB,
 206                                     M_WAITOK | M_ZERO);
 207
 208         if (shared) {
 209                 pcbinfo->infotoken = kmalloc(sizeof(struct lwkt_token),
 210                     M_PCB, M_WAITOK);
 211                 lwkt_token_init(pcbinfo->infotoken, "infotoken");
 212         } else {
 213                 pcbinfo->infotoken = NULL;
 214         }
 215 }
 216
 217 struct baddynamicports baddynamicports;
 218
 219 /*
 220  * Check if the specified port is invalid for dynamic allocation.
 221  */
 222 int
 223 in_baddynamic(u_int16_t port, u_int16_t proto)
 224 {
 225         switch (proto) {
 226         case IPPROTO_TCP:
 227                 return (DP_ISSET(baddynamicports.tcp, port));
 228         case IPPROTO_UDP:
 229 #ifdef IPSEC
 230                 /* Cannot preset this as it is a sysctl */
 231                 if (port == udpencap_port)
 232                         return (1);
 233 #endif
 234                 return (DP_ISSET(baddynamicports.udp, port));
 235         default:
 236                 return (0);
 237         }
 238 }
 239
 240 void
 241 in_pcbonlist(struct inpcb *inp)
 242 {
 243         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 244
 245         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
 246             ("not in the correct netisr"));
 247         KASSERT((inp->inp_flags & INP_ONLIST) == 0, ("already on pcblist"));
 248         inp->inp_flags |= INP_ONLIST;
 249
 250         GET_PCBINFO_TOKEN(pcbinfo);
 251         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list);
 252         pcbinfo->ipi_count++;
 253         REL_PCBINFO_TOKEN(pcbinfo);
 254 }
 255
 256 void
 257 in_pcbofflist(struct inpcb *inp)
 258 {
 259         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 260
 261         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
 262             ("not in the correct netisr"));
 263         KASSERT(inp->inp_flags & INP_ONLIST, ("not on pcblist"));
 264         inp->inp_flags &= ~INP_ONLIST;
 265
 266         GET_PCBINFO_TOKEN(pcbinfo);
 267         LIST_REMOVE(inp, inp_list);
 268         KASSERT(pcbinfo->ipi_count > 0,
 269             ("invalid inpcb count %d", pcbinfo->ipi_count));
 270         pcbinfo->ipi_count--;
 271         REL_PCBINFO_TOKEN(pcbinfo);
 272 }
 273
 274 /*
 275  * Allocate a PCB and associate it with the socket.
 276  */
 277 int
 278 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
 279 {
 280         struct inpcb *inp;
 281 #ifdef IPSEC
 282         int error;
 283 #endif
 284
 285         inp = kmalloc(pcbinfo->ipi_size, M_PCB, M_WAITOK|M_ZERO|M_NULLOK);
 286         if (inp == NULL)
 287                 return (ENOMEM);
 288         inp->inp_lgrpindex = -1;
 289         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
 290         inp->inp_pcbinfo = pcbinfo;
 291         inp->inp_socket = so;
 292 #ifdef IPSEC
 293         error = ipsec_init_policy(so, &inp->inp_sp);
 294         if (error != 0) {
 295                 kfree(inp, M_PCB);
 296                 return (error);
 297         }
 298 #endif
 299 #ifdef INET6
 300         if (INP_SOCKAF(so) == AF_INET6 && ip6_v6only)
 301                 inp->inp_flags |= IN6P_IPV6_V6ONLY;
 302         if (ip6_auto_flowlabel)
 303                 inp->inp_flags |= IN6P_AUTOFLOWLABEL;
 304 #endif
 305         soreference(so);
 306         so->so_pcb = inp;
 307
 308         in_pcbonlist(inp);
 309         return (0);
 310 }
 311
 312 /*
 313  * Unlink a pcb with the intention of moving it to another cpu with a
 314  * different pcbinfo.  While unlinked nothing should attempt to dereference
 315  * inp_pcbinfo, NULL it out so we assert if it does.
 316  */
 317 void
 318 in_pcbunlink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
 319 {
 320         KASSERT(inp->inp_pcbinfo == pcbinfo, ("pcbinfo mismatch"));
 321         KASSERT((inp->inp_flags & (INP_WILDCARD | INP_CONNECTED)) == 0,
 322             ("already linked"));
 323
 324         in_pcbofflist(inp);
 325         inp->inp_pcbinfo = NULL;
 326 }
 327
 328 /*
 329  * Relink a pcb into a new pcbinfo.
 330  */
 331 void
 332 in_pcblink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
 333 {
 334         KASSERT(inp->inp_pcbinfo == NULL, ("has pcbinfo"));
 335         KASSERT((inp->inp_flags & (INP_WILDCARD | INP_CONNECTED)) == 0,
 336             ("already linked"));
 337
 338         inp->inp_pcbinfo = pcbinfo;
 339         in_pcbonlist(inp);
 340 }
 341
 342 static int
 343 in_pcbsetlport(struct inpcb *inp, int wild, struct ucred *cred)
 344 {
 345         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 346         struct inpcbportinfo *portinfo;
 347         u_short first, last, lport, step;
 348         u_short *lastport;
 349         int count, error;
 350         int portinfo_first, portinfo_idx;
 351
 352         inp->inp_flags |= INP_ANONPORT;
 353
 354         step = pcbinfo->portinfo_mask + 1;
 355         portinfo_first = mycpuid & pcbinfo->portinfo_mask;
 356         portinfo_idx = portinfo_first;
 357 loop:
 358         portinfo = &pcbinfo->portinfo[portinfo_idx];
 359
 360         if (inp->inp_flags & INP_HIGHPORT) {
 361                 first = ipport_hifirstauto;     /* sysctl */
 362                 last  = ipport_hilastauto;
 363                 lastport = &portinfo->lasthi;
 364         } else if (inp->inp_flags & INP_LOWPORT) {
 365                 if (cred &&
 366                     (error =
 367                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 368                         inp->inp_laddr.s_addr = INADDR_ANY;
 369                         return error;
 370                 }
 371                 first = ipport_lowfirstauto;    /* 1023 */
 372                 last  = ipport_lowlastauto;     /* 600 */
 373                 lastport = &portinfo->lastlow;
 374         } else {
 375                 first = ipport_firstauto;       /* sysctl */
 376                 last  = ipport_lastauto;
 377                 lastport = &portinfo->lastport;
 378         }
 379
 380         /*
 381          * This has to be atomic.  If the porthash is shared across multiple
 382          * protocol threads (aka tcp) then the token must be held.
 383          */
 384         GET_PORT_TOKEN(portinfo);
 385
 386         /*
 387          * Simple check to ensure all ports are not used up causing
 388          * a deadlock here.
 389          *
 390          * We split the two cases (up and down) so that the direction
 391          * is not being tested on each round of the loop.
 392          */
 393         if (first > last) {
 394                 /*
 395                  * counting down
 396                  */
 397                 in_pcbportrange(&first, &last, portinfo->offset, step);
 398                 count = (first - last) / step;
 399
 400                 do {
 401                         if (count-- < 0) {      /* completely used? */
 402                                 error = EADDRNOTAVAIL;
 403                                 goto done;
 404                         }
 405                         *lastport -= step;
 406                         if (*lastport > first || *lastport < last)
 407                                 *lastport = first;
 408                         KKASSERT((*lastport & pcbinfo->portinfo_mask) ==
 409                             portinfo->offset);
 410                         lport = htons(*lastport);
 411                 } while (in_pcblookup_local(portinfo, inp->inp_laddr, lport,
 412                     wild, cred));
 413         } else {
 414                 /*
 415                  * counting up
 416                  */
 417                 in_pcbportrange(&last, &first, portinfo->offset, step);
 418                 count = (last - first) / step;
 419
 420                 do {
 421                         if (count-- < 0) {      /* completely used? */
 422                                 error = EADDRNOTAVAIL;
 423                                 goto done;
 424                         }
 425                         *lastport += step;
 426                         if (*lastport < first || *lastport > last)
 427                                 *lastport = first;
 428                         KKASSERT((*lastport & pcbinfo->portinfo_mask) ==
 429                             portinfo->offset);
 430                         lport = htons(*lastport);
 431                 } while (in_pcblookup_local(portinfo, inp->inp_laddr, lport,
 432                     wild, cred));
 433         }
 434         inp->inp_lport = lport;
 435         in_pcbinsporthash(portinfo, inp);
 436         error = 0;
 437 done:
 438         REL_PORT_TOKEN(portinfo);
 439
 440         if (error) {
 441                 /* Try next portinfo */
 442                 portinfo_idx++;
 443                 portinfo_idx &= pcbinfo->portinfo_mask;
 444                 if (portinfo_idx != portinfo_first)
 445                         goto loop;
 446                 inp->inp_laddr.s_addr = INADDR_ANY;
 447         }
 448         return error;
 449 }
 450
 451 int
 452 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
 453 {
 454         struct socket *so = inp->inp_socket;
 455         struct sockaddr_in jsin;
 456         struct ucred *cred = NULL;
 457         int wild = 0;
 458
 459         if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */
 460                 return (EADDRNOTAVAIL);
 461         if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
 462                 return (EINVAL);        /* already bound */
 463
 464         if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT)))
 465                 wild = 1;    /* neither SO_REUSEADDR nor SO_REUSEPORT is set */
 466         if (td->td_proc)
 467                 cred = td->td_proc->p_ucred;
 468
 469         if (nam != NULL) {
 470                 struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 471                 struct inpcbinfo *pcbinfo;
 472                 struct inpcbportinfo *portinfo;
 473                 struct inpcb *t;
 474                 u_short lport, lport_ho;
 475                 int reuseport = (so->so_options & SO_REUSEPORT);
 476                 int error;
 477
 478                 if (nam->sa_len != sizeof *sin)
 479                         return (EINVAL);
 480 #ifdef notdef
 481                 /*
 482                  * We should check the family, but old programs
 483                  * incorrectly fail to initialize it.
 484                  */
 485                 if (sin->sin_family != AF_INET)
 486                         return (EAFNOSUPPORT);
 487 #endif
 488                 if (!prison_replace_wildcards(td, nam))
 489                         return (EINVAL);
 490
 491                 lport = sin->sin_port;
 492                 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 493                         /*
 494                          * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
 495                          * allow complete duplication of binding if
 496                          * SO_REUSEPORT is set, or if SO_REUSEADDR is set
 497                          * and a multicast address is bound on both
 498                          * new and duplicated sockets.
 499                          */
 500                         if (so->so_options & SO_REUSEADDR)
 501                                 reuseport = SO_REUSEADDR | SO_REUSEPORT;
 502                 } else if (sin->sin_addr.s_addr != INADDR_ANY) {
 503                         sin->sin_port = 0;              /* yech... */
 504                         bzero(&sin->sin_zero, sizeof sin->sin_zero);
 505                         if (ifa_ifwithaddr((struct sockaddr *)sin) == NULL)
 506                                 return (EADDRNOTAVAIL);
 507                 }
 508
 509                 inp->inp_laddr = sin->sin_addr;
 510
 511                 jsin.sin_family = AF_INET;
 512                 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 513                 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 514                         inp->inp_laddr.s_addr = INADDR_ANY;
 515                         return (EINVAL);
 516                 }
 517                 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 518
 519                 if (lport == 0) {
 520                         /* Auto-select local port */
 521                         return in_pcbsetlport(inp, wild, cred);
 522                 }
 523                 lport_ho = ntohs(lport);
 524
 525                 /* GROSS */
 526                 if (lport_ho < IPPORT_RESERVED && cred &&
 527                     (error =
 528                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 529                         inp->inp_laddr.s_addr = INADDR_ANY;
 530                         return (error);
 531                 }
 532
 533                 /*
 534                  * Locate the proper portinfo based on lport
 535                  */
 536                 pcbinfo = inp->inp_pcbinfo;
 537                 portinfo =
 538                     &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask];
 539                 KKASSERT((lport_ho & pcbinfo->portinfo_mask) ==
 540                     portinfo->offset);
 541
 542                 /*
 543                  * This has to be atomic.  If the porthash is shared across
 544                  * multiple protocol threads (aka tcp) then the token must
 545                  * be held.
 546                  */
 547                 GET_PORT_TOKEN(portinfo);
 548
 549                 if (so->so_cred->cr_uid != 0 &&
 550                     !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 551                         t = in_pcblookup_local(portinfo, sin->sin_addr, lport,
 552                             INPLOOKUP_WILDCARD, cred);
 553                         if (t &&
 554                             (!in_nullhost(sin->sin_addr) ||
 555                              !in_nullhost(t->inp_laddr) ||
 556                              (t->inp_socket->so_options & SO_REUSEPORT) == 0) &&
 557                             (so->so_cred->cr_uid !=
 558                              t->inp_socket->so_cred->cr_uid)) {
 559 #ifdef INET6
 560                                 if (!in_nullhost(sin->sin_addr) ||
 561                                     !in_nullhost(t->inp_laddr) ||
 562                                     INP_SOCKAF(so) == INP_SOCKAF(t->inp_socket))
 563 #endif
 564                                 {
 565                                         inp->inp_laddr.s_addr = INADDR_ANY;
 566                                         error = EADDRINUSE;
 567                                         goto done;
 568                                 }
 569                         }
 570                 }
 571                 if (cred && !prison_replace_wildcards(td, nam)) {
 572                         inp->inp_laddr.s_addr = INADDR_ANY;
 573                         error = EADDRNOTAVAIL;
 574                         goto done;
 575                 }
 576                 t = in_pcblookup_local(portinfo, sin->sin_addr, lport,
 577                     wild, cred);
 578                 if (t && !(reuseport & t->inp_socket->so_options)) {
 579 #ifdef INET6
 580                         if (!in_nullhost(sin->sin_addr) ||
 581                             !in_nullhost(t->inp_laddr) ||
 582                             INP_SOCKAF(so) == INP_SOCKAF(t->inp_socket))
 583 #endif
 584                         {
 585                                 inp->inp_laddr.s_addr = INADDR_ANY;
 586                                 error = EADDRINUSE;
 587                                 goto done;
 588                         }
 589                 }
 590                 inp->inp_lport = lport;
 591                 in_pcbinsporthash(portinfo, inp);
 592                 error = 0;
 593 done:
 594                 REL_PORT_TOKEN(portinfo);
 595                 return (error);
 596         } else {
 597                 jsin.sin_family = AF_INET;
 598                 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 599                 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 600                         inp->inp_laddr.s_addr = INADDR_ANY;
 601                         return (EINVAL);
 602                 }
 603                 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 604
 605                 return in_pcbsetlport(inp, wild, cred);
 606         }
 607 }
 608
 609 static struct inpcb *
 610 in_pcblookup_localremote(struct inpcbportinfo *portinfo, struct in_addr laddr,
 611     u_short lport, struct in_addr faddr, u_short fport, struct ucred *cred)
 612 {
 613         struct inpcb *inp;
 614         struct inpcbporthead *porthash;
 615         struct inpcbport *phd;
 616         struct inpcb *match = NULL;
 617
 618         /*
 619          * If the porthashbase is shared across several cpus, it must
 620          * have been locked.
 621          */
 622         ASSERT_PORT_TOKEN_HELD(portinfo);
 623
 624         /*
 625          * Best fit PCB lookup.
 626          *
 627          * First see if this local port is in use by looking on the
 628          * port hash list.
 629          */
 630         porthash = &portinfo->porthashbase[
 631                         INP_PCBPORTHASH(lport, portinfo->porthashmask)];
 632         LIST_FOREACH(phd, porthash, phd_hash) {
 633                 if (phd->phd_port == lport)
 634                         break;
 635         }
 636         if (phd != NULL) {
 637                 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
 638 #ifdef INET6
 639                         if ((inp->inp_vflag & INP_IPV4) == 0)
 640                                 continue;
 641 #endif
 642                         if (inp->inp_laddr.s_addr != INADDR_ANY &&
 643                             inp->inp_laddr.s_addr != laddr.s_addr)
 644                                 continue;
 645
 646                         if (inp->inp_faddr.s_addr != INADDR_ANY &&
 647                             inp->inp_faddr.s_addr != faddr.s_addr)
 648                                 continue;
 649
 650                         if (inp->inp_fport != 0 && inp->inp_fport != fport)
 651                                 continue;
 652
 653                         if (cred == NULL ||
 654                             cred->cr_prison ==
 655                             inp->inp_socket->so_cred->cr_prison) {
 656                                 match = inp;
 657                                 break;
 658                         }
 659                 }
 660         }
 661         return (match);
 662 }
 663
 664 int
 665 in_pcbbind_remote(struct inpcb *inp, const struct sockaddr *remote,
 666     struct thread *td)
 667 {
 668         struct proc *p = td->td_proc;
 669         unsigned short *lastport;
 670         const struct sockaddr_in *sin = (const struct sockaddr_in *)remote;
 671         struct sockaddr_in jsin;
 672         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 673         struct inpcbportinfo *portinfo;
 674         struct ucred *cred = NULL;
 675         u_short first, last, lport, step;
 676         int count, error, dup;
 677         int portinfo_first, portinfo_idx;
 678
 679         if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */
 680                 return (EADDRNOTAVAIL);
 681
 682         KKASSERT(inp->inp_laddr.s_addr != INADDR_ANY);
 683         if (inp->inp_lport != 0)
 684                 return (EINVAL);        /* already bound */
 685
 686         KKASSERT(p);
 687         cred = p->p_ucred;
 688
 689         jsin.sin_family = AF_INET;
 690         jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 691         if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 692                 inp->inp_laddr.s_addr = INADDR_ANY;
 693                 return (EINVAL);
 694         }
 695         inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 696
 697         inp->inp_flags |= INP_ANONPORT;
 698
 699         step = pcbinfo->portinfo_mask + 1;
 700         portinfo_first = mycpuid & pcbinfo->portinfo_mask;
 701         portinfo_idx = portinfo_first;
 702 loop:
 703         portinfo = &pcbinfo->portinfo[portinfo_idx];
 704         dup = 0;
 705
 706         if (inp->inp_flags & INP_HIGHPORT) {
 707                 first = ipport_hifirstauto;     /* sysctl */
 708                 last  = ipport_hilastauto;
 709                 lastport = &portinfo->lasthi;
 710         } else if (inp->inp_flags & INP_LOWPORT) {
 711                 if (cred &&
 712                     (error =
 713                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 714                         inp->inp_laddr.s_addr = INADDR_ANY;
 715                         return (error);
 716                 }
 717                 first = ipport_lowfirstauto;    /* 1023 */
 718                 last  = ipport_lowlastauto;     /* 600 */
 719                 lastport = &portinfo->lastlow;
 720         } else {
 721                 first = ipport_firstauto;       /* sysctl */
 722                 last  = ipport_lastauto;
 723                 lastport = &portinfo->lastport;
 724         }
 725
 726         /*
 727          * This has to be atomic.  If the porthash is shared across multiple
 728          * protocol threads (aka tcp) then the token must be held.
 729          */
 730         GET_PORT_TOKEN(portinfo);
 731
 732 again:
 733         /*
 734          * Simple check to ensure all ports are not used up causing
 735          * a deadlock here.
 736          *
 737          * We split the two cases (up and down) so that the direction
 738          * is not being tested on each round of the loop.
 739          */
 740         if (first > last) {
 741                 /*
 742                  * counting down
 743                  */
 744                 in_pcbportrange(&first, &last, portinfo->offset, step);
 745                 count = (first - last) / step;
 746
 747                 do {
 748                         if (count-- < 0) {      /* completely used? */
 749                                 error = EADDRNOTAVAIL;
 750                                 goto done;
 751                         }
 752                         *lastport -= step;
 753                         if (*lastport > first || *lastport < last)
 754                                 *lastport = first;
 755                         KKASSERT((*lastport & pcbinfo->portinfo_mask) ==
 756                             portinfo->offset);
 757                         lport = htons(*lastport);
 758                 } while (in_pcblookup_localremote(portinfo, inp->inp_laddr,
 759                     lport, sin->sin_addr, sin->sin_port, cred));
 760         } else {
 761                 /*
 762                  * counting up
 763                  */
 764                 in_pcbportrange(&last, &first, portinfo->offset, step);
 765                 count = (last - first) / step;
 766
 767                 do {
 768                         if (count-- < 0) {      /* completely used? */
 769                                 error = EADDRNOTAVAIL;
 770                                 goto done;
 771                         }
 772                         *lastport += step;
 773                         if (*lastport < first || *lastport > last)
 774                                 *lastport = first;
 775                         KKASSERT((*lastport & pcbinfo->portinfo_mask) ==
 776                             portinfo->offset);
 777                         lport = htons(*lastport);
 778                 } while (in_pcblookup_localremote(portinfo, inp->inp_laddr,
 779                     lport, sin->sin_addr, sin->sin_port, cred));
 780         }
 781
 782         /* This could happen on loopback interface */
 783         if (sin->sin_port == lport &&
 784             sin->sin_addr.s_addr == inp->inp_laddr.s_addr) {
 785                 if (dup) {
 786                         /*
 787                          * Duplicate again; give up
 788                          */
 789                         error = EADDRNOTAVAIL;
 790                         goto done;
 791                 }
 792                 dup = 1;
 793                 goto again;
 794         }
 795         inp->inp_lport = lport;
 796         in_pcbinsporthash(portinfo, inp);
 797         error = 0;
 798 done:
 799         REL_PORT_TOKEN(portinfo);
 800
 801         if (error) {
 802                 /* Try next portinfo */
 803                 portinfo_idx++;
 804                 portinfo_idx &= pcbinfo->portinfo_mask;
 805                 if (portinfo_idx != portinfo_first)
 806                         goto loop;
 807                 inp->inp_laddr.s_addr = INADDR_ANY;
 808         }
 809         return error;
 810 }
 811
 812 /*
 813  *   Transform old in_pcbconnect() into an inner subroutine for new
 814  *   in_pcbconnect(): Do some validity-checking on the remote
 815  *   address (in mbuf 'nam') and then determine local host address
 816  *   (i.e., which interface) to use to access that remote host.
 817  *
 818  *   This preserves definition of in_pcbconnect(), while supporting a
 819  *   slightly different version for T/TCP.  (This is more than
 820  *   a bit of a kludge, but cleaning up the internal interfaces would
 821  *   have forced minor changes in every protocol).
 822  */
 823 int
 824 in_pcbladdr_find(struct inpcb *inp, struct sockaddr *nam,
 825     struct sockaddr_in **plocal_sin, struct thread *td, int find)
 826 {
 827         struct in_ifaddr *ia;
 828         struct ucred *cred = NULL;
 829         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 830         struct sockaddr *jsin;
 831         int jailed = 0, alloc_route = 0;
 832
 833         if (nam->sa_len != sizeof *sin)
 834                 return (EINVAL);
 835         if (sin->sin_family != AF_INET)
 836                 return (EAFNOSUPPORT);
 837         if (sin->sin_port == 0)
 838                 return (EADDRNOTAVAIL);
 839         if (td && td->td_proc && td->td_proc->p_ucred)
 840                 cred = td->td_proc->p_ucred;
 841         if (cred && cred->cr_prison)
 842                 jailed = 1;
 843         if (!TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) {
 844                 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia;
 845                 /*
 846                  * If the destination address is INADDR_ANY,
 847                  * use the primary local address.
 848                  * If the supplied address is INADDR_BROADCAST,
 849                  * and the primary interface supports broadcast,
 850                  * choose the broadcast address for that interface.
 851                  */
 852                 if (sin->sin_addr.s_addr == INADDR_ANY)
 853                         sin->sin_addr = IA_SIN(ia)->sin_addr;
 854                 else if (sin->sin_addr.s_addr == (u_long)INADDR_BROADCAST &&
 855                     (ia->ia_ifp->if_flags & IFF_BROADCAST))
 856                         sin->sin_addr = satosin(&ia->ia_broadaddr)->sin_addr;
 857         }
 858         if (find) {
 859                 struct route *ro;
 860
 861                 ia = NULL;
 862                 /*
 863                  * If route is known or can be allocated now,
 864                  * our src addr is taken from the i/f, else punt.
 865                  * Note that we should check the address family of the cached
 866                  * destination, in case of sharing the cache with IPv6.
 867                  */
 868                 ro = &inp->inp_route;
 869                 if (ro->ro_rt &&
 870                     (!(ro->ro_rt->rt_flags & RTF_UP) ||
 871                      ro->ro_dst.sa_family != AF_INET ||
 872                      satosin(&ro->ro_dst)->sin_addr.s_addr !=
 873                                       sin->sin_addr.s_addr ||
 874                      inp->inp_socket->so_options & SO_DONTROUTE)) {
 875                         RTFREE(ro->ro_rt);
 876                         ro->ro_rt = NULL;
 877                 }
 878                 if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/
 879                     (ro->ro_rt == NULL ||
 880                     ro->ro_rt->rt_ifp == NULL)) {
 881                         /* No route yet, so try to acquire one */
 882                         bzero(&ro->ro_dst, sizeof(struct sockaddr_in));
 883                         ro->ro_dst.sa_family = AF_INET;
 884                         ro->ro_dst.sa_len = sizeof(struct sockaddr_in);
 885                         ((struct sockaddr_in *) &ro->ro_dst)->sin_addr =
 886                                 sin->sin_addr;
 887                         rtalloc(ro);
 888                         alloc_route = 1;
 889                 }
 890                 /*
 891                  * If we found a route, use the address
 892                  * corresponding to the outgoing interface
 893                  * unless it is the loopback (in case a route
 894                  * to our address on another net goes to loopback).
 895                  */
 896                 if (ro->ro_rt &&
 897                     !(ro->ro_rt->rt_ifp->if_flags & IFF_LOOPBACK)) {
 898                         if (jailed) {
 899                                 if (jailed_ip(cred->cr_prison,
 900                                     ro->ro_rt->rt_ifa->ifa_addr)) {
 901                                         ia = ifatoia(ro->ro_rt->rt_ifa);
 902                                 }
 903                         } else {
 904                                 ia = ifatoia(ro->ro_rt->rt_ifa);
 905                         }
 906                 }
 907                 if (ia == NULL) {
 908                         u_short fport = sin->sin_port;
 909
 910                         sin->sin_port = 0;
 911                         ia = ifatoia(ifa_ifwithdstaddr(sintosa(sin)));
 912                         if (ia && jailed && !jailed_ip(cred->cr_prison,
 913                             sintosa(&ia->ia_addr)))
 914                                 ia = NULL;
 915                         if (ia == NULL)
 916                                 ia = ifatoia(ifa_ifwithnet(sintosa(sin)));
 917                         if (ia && jailed && !jailed_ip(cred->cr_prison,
 918                             sintosa(&ia->ia_addr)))
 919                                 ia = NULL;
 920                         sin->sin_port = fport;
 921                         if (ia == NULL &&
 922                             !TAILQ_EMPTY(&in_ifaddrheads[mycpuid]))
 923                                 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia;
 924                         if (ia && jailed && !jailed_ip(cred->cr_prison,
 925                             sintosa(&ia->ia_addr)))
 926                                 ia = NULL;
 927
 928                         if (!jailed && ia == NULL)
 929                                 goto fail;
 930                 }
 931                 /*
 932                  * If the destination address is multicast and an outgoing
 933                  * interface has been set as a multicast option, use the
 934                  * address of that interface as our source address.
 935                  */
 936                 if (!jailed && IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
 937                     inp->inp_moptions != NULL) {
 938                         struct ip_moptions *imo;
 939                         struct ifnet *ifp;
 940
 941                         imo = inp->inp_moptions;
 942                         if (imo->imo_multicast_ifp != NULL) {
 943                                 struct in_ifaddr_container *iac;
 944
 945                                 ifp = imo->imo_multicast_ifp;
 946                                 ia = NULL;
 947                                 TAILQ_FOREACH(iac,
 948                                 &in_ifaddrheads[mycpuid], ia_link) {
 949                                         if (iac->ia->ia_ifp == ifp) {
 950                                                 ia = iac->ia;
 951                                                 break;
 952                                         }
 953                                 }
 954                                 if (ia == NULL)
 955                                         goto fail;
 956                         }
 957                 }
 958                 /*
 959                  * Don't do pcblookup call here; return interface in plocal_sin
 960                  * and exit to caller, that will do the lookup.
 961                  */
 962                 if (ia == NULL && jailed) {
 963                         if ((jsin = prison_get_nonlocal(
 964                                 cred->cr_prison, AF_INET, NULL)) != NULL ||
 965                             (jsin = prison_get_local(
 966                                 cred->cr_prison, AF_INET, NULL)) != NULL) {
 967                                 *plocal_sin = satosin(jsin);
 968                         } else {
 969                                 /* IPv6 only Jail */
 970                                 goto fail;
 971                         }
 972                 } else {
 973                         *plocal_sin = &ia->ia_addr;
 974                 }
 975         }
 976         return (0);
 977 fail:
 978         if (alloc_route) {
 979                 struct route *ro = &inp->inp_route;
 980
 981                 if (ro->ro_rt != NULL)
 982                         RTFREE(ro->ro_rt);
 983                 bzero(ro, sizeof(*ro));
 984         }
 985         return (EADDRNOTAVAIL);
 986 }
 987
 988 int
 989 in_pcbladdr(struct inpcb *inp, struct sockaddr *nam,
 990     struct sockaddr_in **plocal_sin, struct thread *td)
 991 {
 992         return in_pcbladdr_find(inp, nam, plocal_sin, td,
 993             (inp->inp_laddr.s_addr == INADDR_ANY));
 994 }
 995
 996 /*
 997  * Outer subroutine:
 998  * Connect from a socket to a specified address.
 999  * Both address and port must be specified in argument sin.
1000  * If don't have a local address for this socket yet,
1001  * then pick one.
1002  */
1003 int
1004 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
1005 {
1006         struct sockaddr_in *if_sin;
1007         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
1008         int error;
1009
1010         /* Call inner routine to assign local interface address. */
1011         if ((error = in_pcbladdr(inp, nam, &if_sin, td)) != 0)
1012                 return (error);
1013
1014         if (in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port,
1015                               inp->inp_laddr.s_addr ?
1016                                 inp->inp_laddr : if_sin->sin_addr,
1017                               inp->inp_lport, FALSE, NULL) != NULL) {
1018                 return (EADDRINUSE);
1019         }
1020         if (inp->inp_laddr.s_addr == INADDR_ANY) {
1021                 if (inp->inp_lport == 0) {
1022                         error = in_pcbbind(inp, NULL, td);
1023                         if (error)
1024                                 return (error);
1025                 }
1026                 inp->inp_laddr = if_sin->sin_addr;
1027         }
1028         inp->inp_faddr = sin->sin_addr;
1029         inp->inp_fport = sin->sin_port;
1030         in_pcbinsconnhash(inp);
1031         return (0);
1032 }
1033
1034 void
1035 in_pcbdisconnect(struct inpcb *inp)
1036 {
1037
1038         in_pcbremconnhash(inp);
1039         inp->inp_faddr.s_addr = INADDR_ANY;
1040         inp->inp_fport = 0;
1041 }
1042
1043 void
1044 in_pcbdetach(struct inpcb *inp)
1045 {
1046         struct socket *so = inp->inp_socket;
1047         struct inpcbinfo *ipi = inp->inp_pcbinfo;
1048
1049 #ifdef IPSEC
1050         ipsec4_delete_pcbpolicy(inp);
1051 #endif /*IPSEC*/
1052         inp->inp_gencnt = ++ipi->ipi_gencnt;
1053         KKASSERT((so->so_state & SS_ASSERTINPROG) == 0);
1054         in_pcbremlists(inp);
1055         so->so_pcb = NULL;
1056         sofree(so);                     /* remove pcb ref */
1057         if (inp->inp_options)
1058                 m_free(inp->inp_options);
1059         if (inp->inp_route.ro_rt)
1060                 rtfree(inp->inp_route.ro_rt);
1061         ip_freemoptions(inp->inp_moptions);
1062         inp->inp_vflag = 0;
1063         kfree(inp, M_PCB);
1064 }
1065
1066 /*
1067  * The calling convention of in_setsockaddr() and in_setpeeraddr() was
1068  * modified to match the pru_sockaddr() and pru_peeraddr() entry points
1069  * in struct pr_usrreqs, so that protocols can just reference then directly
1070  * without the need for a wrapper function.  The socket must have a valid
1071  * (i.e., non-nil) PCB, but it should be impossible to get an invalid one
1072  * except through a kernel programming error, so it is acceptable to panic
1073  * (or in this case trap) if the PCB is invalid.  (Actually, we don't trap
1074  * because there actually /is/ a programming error somewhere... XXX)
1075  */
1076 int
1077 in_setsockaddr(struct socket *so, struct sockaddr **nam)
1078 {
1079         struct inpcb *inp;
1080         struct sockaddr_in *sin;
1081
1082         /*
1083          * Do the malloc first in case it blocks.
1084          */
1085         sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1086         sin->sin_family = AF_INET;
1087         sin->sin_len = sizeof *sin;
1088
1089         crit_enter();
1090         inp = so->so_pcb;
1091         if (!inp) {
1092                 crit_exit();
1093                 kfree(sin, M_SONAME);
1094                 return (ECONNRESET);
1095         }
1096         sin->sin_port = inp->inp_lport;
1097         sin->sin_addr = inp->inp_laddr;
1098         crit_exit();
1099
1100         *nam = (struct sockaddr *)sin;
1101         return (0);
1102 }
1103
1104 void
1105 in_setsockaddr_dispatch(netmsg_t msg)
1106 {
1107         int error;
1108
1109         error = in_setsockaddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1110         lwkt_replymsg(&msg->lmsg, error);
1111 }
1112
1113 int
1114 in_setpeeraddr(struct socket *so, struct sockaddr **nam)
1115 {
1116         struct inpcb *inp;
1117         struct sockaddr_in *sin;
1118
1119         /*
1120          * Do the malloc first in case it blocks.
1121          */
1122         sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1123         sin->sin_family = AF_INET;
1124         sin->sin_len = sizeof *sin;
1125
1126         crit_enter();
1127         inp = so->so_pcb;
1128         if (!inp) {
1129                 crit_exit();
1130                 kfree(sin, M_SONAME);
1131                 return (ECONNRESET);
1132         }
1133         sin->sin_port = inp->inp_fport;
1134         sin->sin_addr = inp->inp_faddr;
1135         crit_exit();
1136
1137         *nam = (struct sockaddr *)sin;
1138         return (0);
1139 }
1140
1141 void
1142 in_setpeeraddr_dispatch(netmsg_t msg)
1143 {
1144         int error;
1145
1146         error = in_setpeeraddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1147         lwkt_replymsg(&msg->lmsg, error);
1148 }
1149
1150 void
1151 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int err,
1152                 void (*notify)(struct inpcb *, int))
1153 {
1154         struct inpcb *inp, *marker;
1155
1156         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1157             ("not in the correct netisr"));
1158         marker = &in_pcbmarkers[mycpuid];
1159
1160         /*
1161          * NOTE:
1162          * - If INP_PLACEMARKER is set we must ignore the rest of the
1163          *   structure and skip it.
1164          * - It is safe to nuke inpcbs here, since we are in their own
1165          *   netisr.
1166          */
1167         GET_PCBINFO_TOKEN(pcbinfo);
1168
1169         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1170         while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1171                 LIST_REMOVE(marker, inp_list);
1172                 LIST_INSERT_AFTER(inp, marker, inp_list);
1173
1174                 if (inp->inp_flags & INP_PLACEMARKER)
1175                         continue;
1176 #ifdef INET6
1177                 if (!(inp->inp_vflag & INP_IPV4))
1178                         continue;
1179 #endif
1180                 if (inp->inp_faddr.s_addr != faddr.s_addr ||
1181                     inp->inp_socket == NULL)
1182                         continue;
1183                 (*notify)(inp, err);            /* can remove inp from list! */
1184         }
1185         LIST_REMOVE(marker, inp_list);
1186
1187         REL_PCBINFO_TOKEN(pcbinfo);
1188 }
1189
1190 void
1191 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1192 {
1193         struct inpcb *inp, *marker;
1194
1195         /*
1196          * We only need to make sure that we are in netisr0, where all
1197          * multicast operation happen.  We could check inpcbinfo which
1198          * does not belong to netisr0 by holding the inpcbinfo's token.
1199          * In this case, the pcbinfo must be able to be shared, i.e.
1200          * pcbinfo->infotoken is not NULL.
1201          */
1202         KASSERT(&curthread->td_msgport == netisr_cpuport(0),
1203             ("not in netisr0"));
1204         KASSERT(pcbinfo->cpu == 0 || pcbinfo->infotoken != NULL,
1205             ("pcbinfo could not be shared"));
1206
1207         /*
1208          * Get a marker for the current netisr (netisr0).
1209          *
1210          * It is possible that the multicast address deletion blocks,
1211          * which could cause temporary token releasing.  So we use
1212          * inpcb marker here to get a coherent view of the inpcb list.
1213          *
1214          * While, on the other hand, moptions are only added and deleted
1215          * in netisr0, so we would not see staled moption or miss moption
1216          * even if the token was released due to the blocking multicast
1217          * address deletion.
1218          */
1219         marker = &in_pcbmarkers[mycpuid];
1220
1221         GET_PCBINFO_TOKEN(pcbinfo);
1222
1223         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1224         while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1225                 struct ip_moptions *imo;
1226
1227                 LIST_REMOVE(marker, inp_list);
1228                 LIST_INSERT_AFTER(inp, marker, inp_list);
1229
1230                 if (inp->inp_flags & INP_PLACEMARKER)
1231                         continue;
1232                 imo = inp->inp_moptions;
1233                 if ((inp->inp_vflag & INP_IPV4) && imo != NULL) {
1234                         int i, gap;
1235
1236                         /*
1237                          * Unselect the outgoing interface if it is being
1238                          * detached.
1239                          */
1240                         if (imo->imo_multicast_ifp == ifp)
1241                                 imo->imo_multicast_ifp = NULL;
1242
1243                         /*
1244                          * Drop multicast group membership if we joined
1245                          * through the interface being detached.
1246                          */
1247                         for (i = 0, gap = 0; i < imo->imo_num_memberships;
1248                             i++) {
1249                                 if (imo->imo_membership[i]->inm_ifp == ifp) {
1250                                         /*
1251                                          * NOTE:
1252                                          * This could block and the pcbinfo
1253                                          * token could be passively released.
1254                                          */
1255                                         in_delmulti(imo->imo_membership[i]);
1256                                         gap++;
1257                                 } else if (gap != 0)
1258                                         imo->imo_membership[i - gap] =
1259                                             imo->imo_membership[i];
1260                         }
1261                         imo->imo_num_memberships -= gap;
1262                 }
1263         }
1264         LIST_REMOVE(marker, inp_list);
1265
1266         REL_PCBINFO_TOKEN(pcbinfo);
1267 }
1268
1269 /*
1270  * Check for alternatives when higher level complains
1271  * about service problems.  For now, invalidate cached
1272  * routing information.  If the route was created dynamically
1273  * (by a redirect), time to try a default gateway again.
1274  */
1275 void
1276 in_losing(struct inpcb *inp)
1277 {
1278         struct rtentry *rt;
1279         struct rt_addrinfo rtinfo;
1280
1281         if ((rt = inp->inp_route.ro_rt)) {
1282                 bzero(&rtinfo, sizeof(struct rt_addrinfo));
1283                 rtinfo.rti_info[RTAX_DST] = rt_key(rt);
1284                 rtinfo.rti_info[RTAX_GATEWAY] = rt->rt_gateway;
1285                 rtinfo.rti_info[RTAX_NETMASK] = rt_mask(rt);
1286                 rtinfo.rti_flags = rt->rt_flags;
1287                 rt_missmsg(RTM_LOSING, &rtinfo, rt->rt_flags, 0);
1288                 if (rt->rt_flags & RTF_DYNAMIC) {
1289                         rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway,
1290                             rt_mask(rt), rt->rt_flags, NULL);
1291                 }
1292                 inp->inp_route.ro_rt = NULL;
1293                 rtfree(rt);
1294                 /*
1295                  * A new route can be allocated
1296                  * the next time output is attempted.
1297                  */
1298         }
1299 }
1300
1301 /*
1302  * After a routing change, flush old routing
1303  * and allocate a (hopefully) better one.
1304  */
1305 void
1306 in_rtchange(struct inpcb *inp, int err)
1307 {
1308         if (inp->inp_route.ro_rt) {
1309                 rtfree(inp->inp_route.ro_rt);
1310                 inp->inp_route.ro_rt = NULL;
1311                 /*
1312                  * A new route can be allocated the next time
1313                  * output is attempted.
1314                  */
1315         }
1316 }
1317
1318 /*
1319  * Lookup a PCB based on the local address and port.
1320  */
1321 struct inpcb *
1322 in_pcblookup_local(struct inpcbportinfo *portinfo, struct in_addr laddr,
1323                    u_int lport_arg, int wild_okay, struct ucred *cred)
1324 {
1325         struct inpcb *inp;
1326         int matchwild = 3, wildcard;
1327         u_short lport = lport_arg;
1328         struct inpcbporthead *porthash;
1329         struct inpcbport *phd;
1330         struct inpcb *match = NULL;
1331
1332         /*
1333          * If the porthashbase is shared across several cpus, it must
1334          * have been locked.
1335          */
1336         ASSERT_PORT_TOKEN_HELD(portinfo);
1337
1338         /*
1339          * Best fit PCB lookup.
1340          *
1341          * First see if this local port is in use by looking on the
1342          * port hash list.
1343          */
1344         porthash = &portinfo->porthashbase[
1345                         INP_PCBPORTHASH(lport, portinfo->porthashmask)];
1346         LIST_FOREACH(phd, porthash, phd_hash) {
1347                 if (phd->phd_port == lport)
1348                         break;
1349         }
1350         if (phd != NULL) {
1351                 /*
1352                  * Port is in use by one or more PCBs. Look for best
1353                  * fit.
1354                  */
1355                 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
1356                         wildcard = 0;
1357 #ifdef INET6
1358                         if ((inp->inp_vflag & INP_IPV4) == 0)
1359                                 continue;
1360 #endif
1361                         if (inp->inp_faddr.s_addr != INADDR_ANY)
1362                                 wildcard++;
1363                         if (inp->inp_laddr.s_addr != INADDR_ANY) {
1364                                 if (laddr.s_addr == INADDR_ANY)
1365                                         wildcard++;
1366                                 else if (inp->inp_laddr.s_addr != laddr.s_addr)
1367                                         continue;
1368                         } else {
1369                                 if (laddr.s_addr != INADDR_ANY)
1370                                         wildcard++;
1371                         }
1372                         if (wildcard && !wild_okay)
1373                                 continue;
1374                         if (wildcard < matchwild &&
1375                             (cred == NULL ||
1376                              cred->cr_prison ==
1377                                         inp->inp_socket->so_cred->cr_prison)) {
1378                                 match = inp;
1379                                 matchwild = wildcard;
1380                                 if (matchwild == 0) {
1381                                         break;
1382                                 }
1383                         }
1384                 }
1385         }
1386         return (match);
1387 }
1388
1389 struct inpcb *
1390 in_pcblocalgroup_last(const struct inpcbinfo *pcbinfo,
1391     const struct inpcb *inp)
1392 {
1393         const struct inp_localgrphead *hdr;
1394         const struct inp_localgroup *grp;
1395         int i;
1396
1397         if (pcbinfo->localgrphashbase == NULL)
1398                 return NULL;
1399
1400         GET_PCBINFO_TOKEN(pcbinfo);
1401
1402         hdr = &pcbinfo->localgrphashbase[
1403             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
1404
1405         LIST_FOREACH(grp, hdr, il_list) {
1406                 if (grp->il_vflag == inp->inp_vflag &&
1407                     grp->il_lport == inp->inp_lport &&
1408                     memcmp(&grp->il_dependladdr,
1409                         &inp->inp_inc.inc_ie.ie_dependladdr,
1410                         sizeof(grp->il_dependladdr)) == 0) {
1411                         break;
1412                 }
1413         }
1414         if (grp == NULL || grp->il_inpcnt == 1) {
1415                 REL_PCBINFO_TOKEN(pcbinfo);
1416                 return NULL;
1417         }
1418
1419         KASSERT(grp->il_inpcnt >= 2,
1420             ("invalid localgroup inp count %d", grp->il_inpcnt));
1421         for (i = 0; i < grp->il_inpcnt; ++i) {
1422                 if (grp->il_inp[i] == inp) {
1423                         int last = grp->il_inpcnt - 1;
1424
1425                         if (i == last)
1426                                 last = grp->il_inpcnt - 2;
1427                         REL_PCBINFO_TOKEN(pcbinfo);
1428                         return grp->il_inp[last];
1429                 }
1430         }
1431         REL_PCBINFO_TOKEN(pcbinfo);
1432         return NULL;
1433 }
1434
1435 static struct inpcb *
1436 inp_localgroup_lookup(const struct inpcbinfo *pcbinfo,
1437     struct in_addr laddr, uint16_t lport, uint32_t pkt_hash)
1438 {
1439         struct inpcb *local_wild = NULL;
1440         const struct inp_localgrphead *hdr;
1441         const struct inp_localgroup *grp;
1442
1443         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1444
1445         hdr = &pcbinfo->localgrphashbase[
1446             INP_PCBLOCALGRPHASH(lport, pcbinfo->localgrphashmask)];
1447
1448         /*
1449          * Order of socket selection:
1450          * 1. non-wild.
1451          * 2. wild.
1452          *
1453          * NOTE:
1454          * - Local group does not contain jailed sockets
1455          * - Local group does not contain IPv4 mapped INET6 wild sockets
1456          */
1457         LIST_FOREACH(grp, hdr, il_list) {
1458 #ifdef INET6
1459                 if (!(grp->il_vflag & INP_IPV4))
1460                         continue;
1461 #endif
1462                 if (grp->il_lport == lport) {
1463                         int idx;
1464
1465                         /*
1466                          * Modulo-N is used here, which greatly reduces
1467                          * completion queue token contention, thus more
1468                          * cpu time is saved.
1469                          */
1470                         idx = pkt_hash % grp->il_inpcnt;
1471                         if (grp->il_laddr.s_addr == laddr.s_addr)
1472                                 return grp->il_inp[idx];
1473                         else if (grp->il_laddr.s_addr == INADDR_ANY)
1474                                 local_wild = grp->il_inp[idx];
1475                 }
1476         }
1477         if (local_wild != NULL)
1478                 return local_wild;
1479         return NULL;
1480 }
1481
1482 /*
1483  * Lookup PCB in hash list.
1484  */
1485 struct inpcb *
1486 in_pcblookup_pkthash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1487     u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1488     boolean_t wildcard, struct ifnet *ifp, const struct mbuf *m)
1489 {
1490         struct inpcbhead *head;
1491         struct inpcb *inp, *jinp=NULL;
1492         u_short fport = fport_arg, lport = lport_arg;
1493
1494         /*
1495          * First look for an exact match.
1496          */
1497         head = &pcbinfo->hashbase[INP_PCBCONNHASH(faddr.s_addr, fport,
1498             laddr.s_addr, lport, pcbinfo->hashmask)];
1499         LIST_FOREACH(inp, head, inp_hash) {
1500 #ifdef INET6
1501                 if (!(inp->inp_vflag & INP_IPV4))
1502                         continue;
1503 #endif
1504                 if (in_hosteq(inp->inp_faddr, faddr) &&
1505                     in_hosteq(inp->inp_laddr, laddr) &&
1506                     inp->inp_fport == fport && inp->inp_lport == lport) {
1507                         /* found */
1508                         if (inp->inp_socket == NULL ||
1509                             inp->inp_socket->so_cred->cr_prison == NULL) {
1510                                 return (inp);
1511                         } else {
1512                                 if  (jinp == NULL)
1513                                         jinp = inp;
1514                         }
1515                 }
1516         }
1517         if (jinp != NULL)
1518                 return (jinp);
1519
1520         if (wildcard) {
1521                 struct inpcb *local_wild = NULL;
1522                 struct inpcb *jinp_wild = NULL;
1523 #ifdef INET6
1524                 struct inpcb *local_wild_mapped = NULL;
1525 #endif
1526                 struct inpcontainer *ic;
1527                 struct inpcontainerhead *chead;
1528                 struct sockaddr_in jsin;
1529                 struct ucred *cred;
1530
1531                 GET_PCBINFO_TOKEN(pcbinfo);
1532
1533                 /*
1534                  * Check local group first
1535                  */
1536                 if (pcbinfo->localgrphashbase != NULL &&
1537                     m != NULL && (m->m_flags & M_HASH) &&
1538                     !(ifp && ifp->if_type == IFT_FAITH)) {
1539                         inp = inp_localgroup_lookup(pcbinfo,
1540                             laddr, lport, m->m_pkthdr.hash);
1541                         if (inp != NULL) {
1542                                 REL_PCBINFO_TOKEN(pcbinfo);
1543                                 return inp;
1544                         }
1545                 }
1546
1547                 /*
1548                  * Order of socket selection:
1549                  * 1. non-jailed, non-wild.
1550                  * 2. non-jailed, wild.
1551                  * 3. jailed, non-wild.
1552                  * 4. jailed, wild.
1553                  */
1554                 jsin.sin_family = AF_INET;
1555                 chead = &pcbinfo->wildcardhashbase[
1556                     INP_PCBWILDCARDHASH(lport, pcbinfo->wildcardhashmask)];
1557                 LIST_FOREACH(ic, chead, ic_list) {
1558                         inp = ic->ic_inp;
1559                         if (inp->inp_flags & INP_PLACEMARKER)
1560                                 continue;
1561
1562                         jsin.sin_addr.s_addr = laddr.s_addr;
1563 #ifdef INET6
1564                         if (!(inp->inp_vflag & INP_IPV4))
1565                                 continue;
1566 #endif
1567                         if (inp->inp_socket != NULL)
1568                                 cred = inp->inp_socket->so_cred;
1569                         else
1570                                 cred = NULL;
1571                         if (cred != NULL && jailed(cred)) {
1572                                 if (jinp != NULL)
1573                                         continue;
1574                                 else
1575                                         if (!jailed_ip(cred->cr_prison,
1576                                             (struct sockaddr *)&jsin))
1577                                                 continue;
1578                         }
1579                         if (inp->inp_lport == lport) {
1580                                 if (ifp && ifp->if_type == IFT_FAITH &&
1581                                     !(inp->inp_flags & INP_FAITH))
1582                                         continue;
1583                                 if (inp->inp_laddr.s_addr == laddr.s_addr) {
1584                                         if (cred != NULL && jailed(cred)) {
1585                                                 jinp = inp;
1586                                         } else {
1587                                                 REL_PCBINFO_TOKEN(pcbinfo);
1588                                                 return (inp);
1589                                         }
1590                                 }
1591                                 if (inp->inp_laddr.s_addr == INADDR_ANY) {
1592 #ifdef INET6
1593                                         if (INP_CHECK_SOCKAF(inp->inp_socket,
1594                                                              AF_INET6))
1595                                                 local_wild_mapped = inp;
1596                                         else
1597 #endif
1598                                                 if (cred != NULL &&
1599                                                     jailed(cred))
1600                                                         jinp_wild = inp;
1601                                                 else
1602                                                         local_wild = inp;
1603                                 }
1604                         }
1605                 }
1606
1607                 REL_PCBINFO_TOKEN(pcbinfo);
1608
1609                 if (local_wild != NULL)
1610                         return (local_wild);
1611 #ifdef INET6
1612                 if (local_wild_mapped != NULL)
1613                         return (local_wild_mapped);
1614 #endif
1615                 if (jinp != NULL)
1616                         return (jinp);
1617                 return (jinp_wild);
1618         }
1619
1620         /*
1621          * Not found.
1622          */
1623         return (NULL);
1624 }
1625
1626 struct inpcb *
1627 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1628     u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1629     boolean_t wildcard, struct ifnet *ifp)
1630 {
1631         return in_pcblookup_pkthash(pcbinfo, faddr, fport_arg,
1632             laddr, lport_arg, wildcard, ifp, NULL);
1633 }
1634
1635 /*
1636  * Insert PCB into connection hash table.
1637  */
1638 void
1639 in_pcbinsconnhash(struct inpcb *inp)
1640 {
1641         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1642         struct inpcbhead *bucket;
1643         u_int32_t hashkey_faddr, hashkey_laddr;
1644
1645 #ifdef INET6
1646         if (inp->inp_vflag & INP_IPV6) {
1647                 hashkey_faddr = inp->in6p_faddr.s6_addr32[3] /* XXX JH */;
1648                 hashkey_laddr = inp->in6p_laddr.s6_addr32[3] /* XXX JH */;
1649         } else {
1650 #endif
1651                 hashkey_faddr = inp->inp_faddr.s_addr;
1652                 hashkey_laddr = inp->inp_laddr.s_addr;
1653 #ifdef INET6
1654         }
1655 #endif
1656
1657         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1658             ("not in the correct netisr"));
1659         KASSERT(!(inp->inp_flags & INP_WILDCARD), ("already on wildcardhash"));
1660         KASSERT(!(inp->inp_flags & INP_CONNECTED), ("already on connhash"));
1661         inp->inp_flags |= INP_CONNECTED;
1662
1663         /*
1664          * Insert into the connection hash table.
1665          */
1666         bucket = &pcbinfo->hashbase[INP_PCBCONNHASH(hashkey_faddr,
1667             inp->inp_fport, hashkey_laddr, inp->inp_lport, pcbinfo->hashmask)];
1668         LIST_INSERT_HEAD(bucket, inp, inp_hash);
1669 }
1670
1671 /*
1672  * Remove PCB from connection hash table.
1673  */
1674 void
1675 in_pcbremconnhash(struct inpcb *inp)
1676 {
1677         struct inpcbinfo *pcbinfo __debugvar = inp->inp_pcbinfo;
1678
1679         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1680             ("not in the correct netisr"));
1681         KASSERT(inp->inp_flags & INP_CONNECTED, ("inp not connected"));
1682
1683         LIST_REMOVE(inp, inp_hash);
1684         inp->inp_flags &= ~INP_CONNECTED;
1685 }
1686
1687 /*
1688  * Insert PCB into port hash table.
1689  */
1690 void
1691 in_pcbinsporthash(struct inpcbportinfo *portinfo, struct inpcb *inp)
1692 {
1693         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1694         struct inpcbporthead *pcbporthash;
1695         struct inpcbport *phd;
1696
1697         /*
1698          * If the porthashbase is shared across several cpus, it must
1699          * have been locked.
1700          */
1701         ASSERT_PORT_TOKEN_HELD(portinfo);
1702
1703         /*
1704          * Insert into the port hash table.
1705          */
1706         pcbporthash = &portinfo->porthashbase[
1707             INP_PCBPORTHASH(inp->inp_lport, portinfo->porthashmask)];
1708
1709         /* Go through port list and look for a head for this lport. */
1710         LIST_FOREACH(phd, pcbporthash, phd_hash) {
1711                 if (phd->phd_port == inp->inp_lport)
1712                         break;
1713         }
1714
1715         /* If none exists, use saved one and tack it on. */
1716         if (phd == NULL) {
1717                 KKASSERT(pcbinfo->portsave != NULL);
1718                 phd = pcbinfo->portsave;
1719                 pcbinfo->portsave = NULL;
1720                 phd->phd_port = inp->inp_lport;
1721                 LIST_INIT(&phd->phd_pcblist);
1722                 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
1723         }
1724
1725         inp->inp_portinfo = portinfo;
1726         inp->inp_phd = phd;
1727         LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
1728
1729         /*
1730          * Malloc one inpcbport for later use.  It is safe to use
1731          * "wait" malloc here (port token would be released, if
1732          * malloc ever blocked), since all changes to the porthash
1733          * are done.
1734          */
1735         if (pcbinfo->portsave == NULL) {
1736                 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave),
1737                                             M_PCB, M_INTWAIT | M_ZERO);
1738         }
1739 }
1740
1741 void
1742 in_pcbinsporthash_lport(struct inpcb *inp)
1743 {
1744         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1745         struct inpcbportinfo *portinfo;
1746         u_short lport_ho;
1747
1748         /* Locate the proper portinfo based on lport */
1749         lport_ho = ntohs(inp->inp_lport);
1750         portinfo = &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask];
1751         KKASSERT((lport_ho & pcbinfo->portinfo_mask) == portinfo->offset);
1752
1753         GET_PORT_TOKEN(portinfo);
1754         in_pcbinsporthash(portinfo, inp);
1755         REL_PORT_TOKEN(portinfo);
1756 }
1757
1758 static struct inp_localgroup *
1759 inp_localgroup_alloc(u_char vflag,
1760     uint16_t port, const union in_dependaddr *addr, int size)
1761 {
1762         struct inp_localgroup *grp;
1763
1764         grp = kmalloc(__offsetof(struct inp_localgroup, il_inp[size]),
1765             M_TEMP, M_INTWAIT | M_ZERO);
1766         grp->il_vflag = vflag;
1767         grp->il_lport = port;
1768         grp->il_dependladdr = *addr;
1769         grp->il_inpsiz = size;
1770
1771         return grp;
1772 }
1773
1774 static void
1775 inp_localgroup_free(struct inp_localgroup *grp)
1776 {
1777         kfree(grp, M_TEMP);
1778 }
1779
1780 static void
1781 inp_localgroup_destroy(struct inp_localgroup *grp)
1782 {
1783         LIST_REMOVE(grp, il_list);
1784         inp_localgroup_free(grp);
1785 }
1786
1787 static void
1788 inp_localgroup_copy(struct inp_localgroup *grp,
1789     const struct inp_localgroup *old_grp)
1790 {
1791         int i;
1792
1793         KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
1794             ("invalid new local group size %d and old local group count %d",
1795              grp->il_inpsiz, old_grp->il_inpcnt));
1796         for (i = 0; i < old_grp->il_inpcnt; ++i)
1797                 grp->il_inp[i] = old_grp->il_inp[i];
1798         grp->il_inpcnt = old_grp->il_inpcnt;
1799 }
1800
1801 static void
1802 in_pcbinslocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
1803 {
1804         struct inp_localgrphead *hdr;
1805         struct inp_localgroup *grp, *grp_alloc = NULL;
1806         struct ucred *cred;
1807         int i, idx;
1808
1809         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1810
1811         if (pcbinfo->localgrphashbase == NULL)
1812                 return;
1813
1814         /*
1815          * XXX don't allow jailed socket to join local group
1816          */
1817         if (inp->inp_socket != NULL)
1818                 cred = inp->inp_socket->so_cred;
1819         else
1820                 cred = NULL;
1821         if (cred != NULL && jailed(cred))
1822                 return;
1823
1824 #ifdef INET6
1825         /*
1826          * XXX don't allow IPv4 mapped INET6 wild socket
1827          */
1828         if ((inp->inp_vflag & INP_IPV4) &&
1829             inp->inp_laddr.s_addr == INADDR_ANY &&
1830             INP_CHECK_SOCKAF(inp->inp_socket, AF_INET6))
1831                 return;
1832 #endif
1833
1834         hdr = &pcbinfo->localgrphashbase[
1835             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
1836
1837 again:
1838         LIST_FOREACH(grp, hdr, il_list) {
1839                 if (grp->il_vflag == inp->inp_vflag &&
1840                     grp->il_lport == inp->inp_lport &&
1841                     memcmp(&grp->il_dependladdr,
1842                         &inp->inp_inc.inc_ie.ie_dependladdr,
1843                         sizeof(grp->il_dependladdr)) == 0) {
1844                         break;
1845                 }
1846         }
1847         if (grp == NULL) {
1848                 /*
1849                  * Create a new local group
1850                  */
1851                 if (grp_alloc == NULL) {
1852                         grp_alloc = inp_localgroup_alloc(inp->inp_vflag,
1853                             inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
1854                             INP_LOCALGROUP_SIZMIN);
1855                         /*
1856                          * Local group allocation could block and the
1857                          * local group w/ the same property might have
1858                          * been added by others when we were blocked;
1859                          * check again.
1860                          */
1861                         goto again;
1862                 } else {
1863                         /* Local group has been allocated; link it */
1864                         grp = grp_alloc;
1865                         grp_alloc = NULL;
1866                         LIST_INSERT_HEAD(hdr, grp, il_list);
1867                 }
1868         } else if (grp->il_inpcnt == grp->il_inpsiz) {
1869                 if (grp->il_inpsiz >= INP_LOCALGROUP_SIZMAX) {
1870                         static int limit_logged = 0;
1871
1872                         if (!limit_logged) {
1873                                 limit_logged = 1;
1874                                 kprintf("local group port %d, "
1875                                     "limit reached\n", ntohs(grp->il_lport));
1876                         }
1877                         if (grp_alloc != NULL) {
1878                                 /*
1879                                  * This would happen if the local group
1880                                  * w/ the same property was expanded when
1881                                  * our local group allocation blocked.
1882                                  */
1883                                 inp_localgroup_free(grp_alloc);
1884                         }
1885                         return;
1886                 }
1887
1888                 /*
1889                  * Expand this local group
1890                  */
1891                 if (grp_alloc == NULL ||
1892                     grp->il_inpcnt >= grp_alloc->il_inpsiz) {
1893                         if (grp_alloc != NULL)
1894                                 inp_localgroup_free(grp_alloc);
1895                         grp_alloc = inp_localgroup_alloc(grp->il_vflag,
1896                             grp->il_lport, &grp->il_dependladdr,
1897                             grp->il_inpsiz * 2);
1898                         /*
1899                          * Local group allocation could block and the
1900                          * local group w/ the same property might have
1901                          * been expanded by others when we were blocked;
1902                          * check again.
1903                          */
1904                         goto again;
1905                 }
1906
1907                 /*
1908                  * Save the old local group, link the new one, and then
1909                  * destroy the old local group
1910                  */
1911                 inp_localgroup_copy(grp_alloc, grp);
1912                 LIST_INSERT_HEAD(hdr, grp_alloc, il_list);
1913                 inp_localgroup_destroy(grp);
1914
1915                 grp = grp_alloc;
1916                 grp_alloc = NULL;
1917         } else {
1918                 /*
1919                  * Found the local group
1920                  */
1921                 if (grp_alloc != NULL) {
1922                         /*
1923                          * This would happen if the local group w/ the
1924                          * same property was added or expanded when our
1925                          * local group allocation blocked.
1926                          */
1927                         inp_localgroup_free(grp_alloc);
1928                         grp_alloc = NULL;
1929                 }
1930         }
1931
1932         KASSERT(grp->il_inpcnt < grp->il_inpsiz,
1933             ("invalid local group size %d and count %d",
1934              grp->il_inpsiz, grp->il_inpcnt));
1935
1936         /*
1937          * Keep the local group sorted by the inpcb local group index
1938          * in ascending order.
1939          *
1940          * This eases the multi-process userland application which uses
1941          * SO_REUSEPORT sockets and binds process to the owner cpu of
1942          * the SO_REUSEPORT socket:
1943          * If we didn't sort the local group by the inpcb local group
1944          * index and one of the process owning an inpcb in this local
1945          * group restarted, e.g. crashed and restarted by watchdog,
1946          * other processes owning a inpcb in this local group would have
1947          * to detect that event, refetch its socket's owner cpu, and
1948          * re-bind.
1949          */
1950         idx = grp->il_inpcnt;
1951         for (i = 0; i < idx; ++i) {
1952                 struct inpcb *oinp = grp->il_inp[i];
1953
1954                 if (oinp->inp_lgrpindex > i) {
1955                         if (inp->inp_lgrpindex < 0) {
1956                                 inp->inp_lgrpindex = i;
1957                         } else if (inp->inp_lgrpindex != i) {
1958                                 if (bootverbose) {
1959                                         kprintf("inp %p: grpidx %d, "
1960                                             "assigned to %d, cpu%d\n",
1961                                             inp, inp->inp_lgrpindex, i,
1962                                             mycpuid);
1963                                 }
1964                         }
1965                         grp->il_inp[i] = inp;
1966
1967                         /* Pull down inpcbs */
1968                         for (; i < grp->il_inpcnt; ++i) {
1969                                 struct inpcb *oinp1 = grp->il_inp[i + 1];
1970
1971                                 grp->il_inp[i + 1] = oinp;
1972                                 oinp = oinp1;
1973                         }
1974                         grp->il_inpcnt++;
1975                         return;
1976                 }
1977         }
1978
1979         if (inp->inp_lgrpindex < 0) {
1980                 inp->inp_lgrpindex = idx;
1981         } else if (inp->inp_lgrpindex != idx) {
1982                 if (bootverbose) {
1983                         kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n",
1984                             inp, inp->inp_lgrpindex, idx, mycpuid);
1985                 }
1986         }
1987         grp->il_inp[idx] = inp;
1988         grp->il_inpcnt++;
1989 }
1990
1991 void
1992 in_pcbinswildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
1993 {
1994         struct inpcontainer *ic;
1995         struct inpcontainerhead *bucket;
1996
1997         GET_PCBINFO_TOKEN(pcbinfo);
1998
1999         in_pcbinslocalgrphash_oncpu(inp, pcbinfo);
2000
2001         bucket = &pcbinfo->wildcardhashbase[
2002             INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
2003
2004         ic = kmalloc(sizeof(struct inpcontainer), M_TEMP, M_INTWAIT);
2005         ic->ic_inp = inp;
2006         LIST_INSERT_HEAD(bucket, ic, ic_list);
2007
2008         REL_PCBINFO_TOKEN(pcbinfo);
2009 }
2010
2011 /*
2012  * Insert PCB into wildcard hash table.
2013  */
2014 void
2015 in_pcbinswildcardhash(struct inpcb *inp)
2016 {
2017         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2018
2019         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
2020             ("not in correct netisr"));
2021         KASSERT(!(inp->inp_flags & INP_CONNECTED),
2022                 ("already on connhash"));
2023         KASSERT(!(inp->inp_flags & INP_WILDCARD),
2024                 ("already on wildcardhash"));
2025         inp->inp_flags |= INP_WILDCARD;
2026
2027         in_pcbinswildcardhash_oncpu(inp, pcbinfo);
2028 }
2029
2030 static void
2031 in_pcbremlocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2032 {
2033         struct inp_localgrphead *hdr;
2034         struct inp_localgroup *grp;
2035
2036         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
2037
2038         if (pcbinfo->localgrphashbase == NULL)
2039                 return;
2040
2041         hdr = &pcbinfo->localgrphashbase[
2042             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
2043
2044         LIST_FOREACH(grp, hdr, il_list) {
2045                 int i;
2046
2047                 for (i = 0; i < grp->il_inpcnt; ++i) {
2048                         if (grp->il_inp[i] != inp)
2049                                 continue;
2050
2051                         if (grp->il_inpcnt == 1) {
2052                                 /* Destroy this local group */
2053                                 inp_localgroup_destroy(grp);
2054                         } else {
2055                                 /* Pull up inpcbs */
2056                                 for (; i + 1 < grp->il_inpcnt; ++i)
2057                                         grp->il_inp[i] = grp->il_inp[i + 1];
2058                                 grp->il_inpcnt--;
2059                         }
2060                         return;
2061                 }
2062         }
2063 }
2064
2065 void
2066 in_pcbremwildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2067 {
2068         struct inpcontainer *ic;
2069         struct inpcontainerhead *head;
2070
2071         GET_PCBINFO_TOKEN(pcbinfo);
2072
2073         in_pcbremlocalgrphash_oncpu(inp, pcbinfo);
2074
2075         /* find bucket */
2076         head = &pcbinfo->wildcardhashbase[
2077             INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
2078
2079         LIST_FOREACH(ic, head, ic_list) {
2080                 if (ic->ic_inp == inp)
2081                         goto found;
2082         }
2083         REL_PCBINFO_TOKEN(pcbinfo);
2084         return;                 /* not found! */
2085
2086 found:
2087         LIST_REMOVE(ic, ic_list);       /* remove container from bucket chain */
2088         REL_PCBINFO_TOKEN(pcbinfo);
2089         kfree(ic, M_TEMP);              /* deallocate container */
2090 }
2091
2092 /*
2093  * Remove PCB from wildcard hash table.
2094  */
2095 void
2096 in_pcbremwildcardhash(struct inpcb *inp)
2097 {
2098         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2099
2100         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
2101             ("not in correct netisr"));
2102         KASSERT(inp->inp_flags & INP_WILDCARD, ("inp not wildcard"));
2103
2104         in_pcbremwildcardhash_oncpu(inp, pcbinfo);
2105         inp->inp_lgrpindex = -1;
2106         inp->inp_flags &= ~INP_WILDCARD;
2107 }
2108
2109 /*
2110  * Remove PCB from various lists.
2111  */
2112 void
2113 in_pcbremlists(struct inpcb *inp)
2114 {
2115         if (inp->inp_lport) {
2116                 struct inpcbportinfo *portinfo;
2117                 struct inpcbport *phd;
2118
2119                 /*
2120                  * NOTE:
2121                  * inp->inp_portinfo is _not_ necessary same as
2122                  * inp->inp_pcbinfo->portinfo.
2123                  */
2124                 portinfo = inp->inp_portinfo;
2125                 GET_PORT_TOKEN(portinfo);
2126
2127                 phd = inp->inp_phd;
2128                 LIST_REMOVE(inp, inp_portlist);
2129                 if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
2130                         LIST_REMOVE(phd, phd_hash);
2131                         kfree(phd, M_PCB);
2132                 }
2133
2134                 REL_PORT_TOKEN(portinfo);
2135         }
2136         if (inp->inp_flags & INP_WILDCARD) {
2137                 in_pcbremwildcardhash(inp);
2138         } else if (inp->inp_flags & INP_CONNECTED) {
2139                 in_pcbremconnhash(inp);
2140         }
2141
2142         if (inp->inp_flags & INP_ONLIST)
2143                 in_pcbofflist(inp);
2144 }
2145
2146 int
2147 prison_xinpcb(struct thread *td, struct inpcb *inp)
2148 {
2149         struct ucred *cr;
2150
2151         if (td->td_proc == NULL)
2152                 return (0);
2153         cr = td->td_proc->p_ucred;
2154         if (cr->cr_prison == NULL)
2155                 return (0);
2156         if (inp->inp_socket && inp->inp_socket->so_cred &&
2157             inp->inp_socket->so_cred->cr_prison &&
2158             cr->cr_prison == inp->inp_socket->so_cred->cr_prison)
2159                 return (0);
2160         return (1);
2161 }
2162
2163 int
2164 in_pcblist_global(SYSCTL_HANDLER_ARGS)
2165 {
2166         struct inpcbinfo *pcbinfo_arr = arg1;
2167         int pcbinfo_arrlen = arg2;
2168         struct inpcb *marker;
2169         int cpu, origcpu;
2170         int error, n;
2171
2172         KASSERT(pcbinfo_arrlen <= ncpus && pcbinfo_arrlen >= 1,
2173             ("invalid pcbinfo count %d", pcbinfo_arrlen));
2174
2175         /*
2176          * The process of preparing the TCB list is too time-consuming and
2177          * resource-intensive to repeat twice on every request.
2178          */
2179         n = 0;
2180         if (req->oldptr == NULL) {
2181                 for (cpu = 0; cpu < pcbinfo_arrlen; ++cpu)
2182                         n += pcbinfo_arr[cpu].ipi_count;
2183                 req->oldidx = (n + n/8 + 10) * sizeof(struct xinpcb);
2184                 return 0;
2185         }
2186
2187         if (req->newptr != NULL)
2188                 return EPERM;
2189
2190         marker = kmalloc(sizeof(struct inpcb), M_TEMP, M_WAITOK|M_ZERO);
2191         marker->inp_flags |= INP_PLACEMARKER;
2192
2193         /*
2194          * OK, now we're committed to doing something.  Re-fetch ipi_count
2195          * after obtaining the generation count.
2196          */
2197         error = 0;
2198         origcpu = mycpuid;
2199         for (cpu = 0; cpu < pcbinfo_arrlen && error == 0; ++cpu) {
2200                 struct inpcbinfo *pcbinfo = &pcbinfo_arr[cpu];
2201                 struct inpcb *inp;
2202                 struct xinpcb xi;
2203                 int i;
2204
2205                 lwkt_migratecpu(cpu);
2206
2207                 GET_PCBINFO_TOKEN(pcbinfo);
2208
2209                 n = pcbinfo->ipi_count;
2210
2211                 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
2212                 i = 0;
2213                 while ((inp = LIST_NEXT(marker, inp_list)) != NULL && i < n) {
2214                         LIST_REMOVE(marker, inp_list);
2215                         LIST_INSERT_AFTER(inp, marker, inp_list);
2216
2217                         if (inp->inp_flags & INP_PLACEMARKER)
2218                                 continue;
2219                         if (prison_xinpcb(req->td, inp))
2220                                 continue;
2221
2222                         bzero(&xi, sizeof xi);
2223                         xi.xi_len = sizeof xi;
2224                         bcopy(inp, &xi.xi_inp, sizeof *inp);
2225                         if (inp->inp_socket)
2226                                 sotoxsocket(inp->inp_socket, &xi.xi_socket);
2227                         if ((error = SYSCTL_OUT(req, &xi, sizeof xi)) != 0)
2228                                 break;
2229                         ++i;
2230                 }
2231                 LIST_REMOVE(marker, inp_list);
2232
2233                 REL_PCBINFO_TOKEN(pcbinfo);
2234
2235                 if (error == 0 && i < n) {
2236                         bzero(&xi, sizeof xi);
2237                         xi.xi_len = sizeof xi;
2238                         while (i < n) {
2239                                 error = SYSCTL_OUT(req, &xi, sizeof xi);
2240                                 if (error)
2241                                         break;
2242                                 ++i;
2243                         }
2244                 }
2245         }
2246
2247         lwkt_migratecpu(origcpu);
2248         kfree(marker, M_TEMP);
2249         return error;
2250 }
2251
2252 int
2253 in_pcblist_global_ncpus2(SYSCTL_HANDLER_ARGS)
2254 {
2255         return in_pcblist_global(oidp, arg1, ncpus2, req);
2256 }
2257
2258 void
2259 in_savefaddr(struct socket *so, const struct sockaddr *faddr)
2260 {
2261         struct sockaddr_in *sin;
2262
2263         KASSERT(faddr->sa_family == AF_INET,
2264             ("not AF_INET faddr %d", faddr->sa_family));
2265
2266         sin = kmalloc(sizeof(*sin), M_SONAME, M_WAITOK | M_ZERO);
2267         sin->sin_family = AF_INET;
2268         sin->sin_len = sizeof(*sin);
2269         sin->sin_port = ((const struct sockaddr_in *)faddr)->sin_port;
2270         sin->sin_addr = ((const struct sockaddr_in *)faddr)->sin_addr;
2271
2272         so->so_faddr = (struct sockaddr *)sin;
2273 }
2274
2275 void
2276 in_pcbportinfo_init(struct inpcbportinfo *portinfo, int hashsize,
2277     boolean_t shared, u_short offset)
2278 {
2279         memset(portinfo, 0, sizeof(*portinfo));
2280
2281         portinfo->offset = offset;
2282         portinfo->lastport = offset;
2283         portinfo->lastlow = offset;
2284         portinfo->lasthi = offset;
2285
2286         portinfo->porthashbase = hashinit(hashsize, M_PCB,
2287             &portinfo->porthashmask);
2288
2289         if (shared) {
2290                 portinfo->porttoken = kmalloc(sizeof(struct lwkt_token),
2291                     M_PCB, M_WAITOK);
2292                 lwkt_token_init(portinfo->porttoken, "porttoken");
2293         }
2294 }
2295
2296 void
2297 in_pcbportrange(u_short *hi0, u_short *lo0, u_short ofs, u_short step)
2298 {
2299         int hi, lo;
2300
2301         if (step == 1)
2302                 return;
2303
2304         hi = *hi0;
2305         lo = *lo0;
2306
2307         hi = rounddown2(hi, step);
2308         hi += ofs;
2309         if (hi > (int)*hi0)
2310                 hi -= step;
2311
2312         lo = roundup2(lo, step);
2313         lo -= (step - ofs);
2314         if (lo < (int)*lo0)
2315                 lo += step;
2316
2317         *hi0 = hi;
2318         *lo0 = lo;
2319 }
2320
2321 void
2322 in_pcbglobalinit(void)
2323 {
2324         int cpu;
2325
2326         in_pcbmarkers = kmalloc(ncpus * sizeof(struct inpcb), M_PCB,
2327             M_WAITOK | M_ZERO);
2328         in_pcbcontainer_markers = kmalloc(ncpus * sizeof(struct inpcontainer),
2329             M_PCB, M_WAITOK | M_ZERO);
2330
2331         for (cpu = 0; cpu < ncpus; ++cpu) {
2332                 struct inpcontainer *ic = &in_pcbcontainer_markers[cpu];
2333                 struct inpcb *marker = &in_pcbmarkers[cpu];
2334
2335                 marker->inp_flags |= INP_PLACEMARKER;
2336                 ic->ic_inp = marker;
2337         }
2338 }
2339
2340 struct inpcb *
2341 in_pcbmarker(int cpuid)
2342 {
2343         KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid));
2344         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
2345
2346         return &in_pcbmarkers[cpuid];
2347 }
2348
2349 struct inpcontainer *
2350 in_pcbcontainer_marker(int cpuid)
2351 {
2352         KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid));
2353         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
2354
2355         return &in_pcbcontainer_markers[cpuid];
2356 }