sys/netinet/in_pcb.c

   1 /*
   2  * Copyright (c) 2004 Jeffrey M. Hsu.  All rights reserved.
   3  * Copyright (c) 2004 The DragonFly Project.  All rights reserved.
   4  *
   5  * This code is derived from software contributed to The DragonFly Project
   6  * by Jeffrey M. Hsu.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the name of The DragonFly Project nor the names of its
  17  *    contributors may be used to endorse or promote products derived
  18  *    from this software without specific, prior written permission.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  23  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  24  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  25  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  26  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  27  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  28  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  29  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  30  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  */
  33
  34 /*
  35  * Copyright (c) 1982, 1986, 1991, 1993, 1995
  36  *      The Regents of the University of California.  All rights reserved.
  37  *
  38  * Redistribution and use in source and binary forms, with or without
  39  * modification, are permitted provided that the following conditions
  40  * are met:
  41  * 1. Redistributions of source code must retain the above copyright
  42  *    notice, this list of conditions and the following disclaimer.
  43  * 2. Redistributions in binary form must reproduce the above copyright
  44  *    notice, this list of conditions and the following disclaimer in the
  45  *    documentation and/or other materials provided with the distribution.
  46  * 3. Neither the name of the University nor the names of its contributors
  47  *    may be used to endorse or promote products derived from this software
  48  *    without specific prior written permission.
  49  *
  50  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  51  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  52  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  53  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  54  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  55  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  56  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  57  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  58  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  59  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  60  * SUCH DAMAGE.
  61  *
  62  *      @(#)in_pcb.c    8.4 (Berkeley) 5/24/95
  63  * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $
  64  */
  65
  66 #include "opt_ipsec.h"
  67 #include "opt_inet6.h"
  68
  69 #include <sys/param.h>
  70 #include <sys/systm.h>
  71 #include <sys/malloc.h>
  72 #include <sys/mbuf.h>
  73 #include <sys/domain.h>
  74 #include <sys/protosw.h>
  75 #include <sys/socket.h>
  76 #include <sys/socketvar.h>
  77 #include <sys/proc.h>
  78 #include <sys/priv.h>
  79 #include <sys/jail.h>
  80 #include <sys/kernel.h>
  81 #include <sys/sysctl.h>
  82
  83 #include <sys/thread2.h>
  84 #include <sys/socketvar2.h>
  85 #include <sys/msgport2.h>
  86
  87 #include <machine/limits.h>
  88
  89 #include <net/if.h>
  90 #include <net/if_types.h>
  91 #include <net/route.h>
  92 #include <net/netisr2.h>
  93
  94 #include <netinet/in.h>
  95 #include <netinet/in_pcb.h>
  96 #include <netinet/in_var.h>
  97 #include <netinet/ip_var.h>
  98 #ifdef INET6
  99 #include <netinet/ip6.h>
 100 #include <netinet6/ip6_var.h>
 101 #endif /* INET6 */
 102
 103 #ifdef IPSEC
 104 #include <netinet6/ipsec.h>
 105 #include <netproto/key/key.h>
 106 #include <netproto/ipsec/esp_var.h>
 107 #endif
 108
 109 #ifdef FAST_IPSEC
 110 #if defined(IPSEC) || defined(IPSEC_ESP)
 111 #error "Bad idea: don't compile with both IPSEC and FAST_IPSEC!"
 112 #endif
 113
 114 #include <netproto/ipsec/ipsec.h>
 115 #include <netproto/ipsec/key.h>
 116 #define IPSEC
 117 #endif /* FAST_IPSEC */
 118
 119 #define INP_LOCALGROUP_SIZMIN   8
 120 #define INP_LOCALGROUP_SIZMAX   256
 121
 122 struct in_addr zeroin_addr;
 123
 124 /*
 125  * These configure the range of local port addresses assigned to
 126  * "unspecified" outgoing connections/packets/whatever.
 127  */
 128 int ipport_lowfirstauto = IPPORT_RESERVED - 1;  /* 1023 */
 129 int ipport_lowlastauto = IPPORT_RESERVEDSTART;  /* 600 */
 130
 131 int ipport_firstauto = IPPORT_RESERVED;         /* 1024 */
 132 int ipport_lastauto = IPPORT_USERRESERVED;      /* 5000 */
 133
 134 int ipport_hifirstauto = IPPORT_HIFIRSTAUTO;    /* 49152 */
 135 int ipport_hilastauto = IPPORT_HILASTAUTO;      /* 65535 */
 136
 137 #define RANGECHK(var, min, max) \
 138         if ((var) < (min)) { (var) = (min); } \
 139         else if ((var) > (max)) { (var) = (max); }
 140
 141 int udpencap_enable = 1;        /* enabled by default */
 142 int udpencap_port = 4500;       /* triggers decapsulation */
 143
 144 /*
 145  * Per-netisr inpcb markers.
 146  * NOTE: they should only be used in netisrs.
 147  */
 148 static struct inpcb             *in_pcbmarkers;
 149 static struct inpcontainer      *in_pcbcontainer_markers;
 150
 151 static int
 152 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
 153 {
 154         int error;
 155
 156         error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
 157         if (!error) {
 158                 RANGECHK(ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
 159                 RANGECHK(ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
 160
 161                 RANGECHK(ipport_firstauto, IPPORT_RESERVED, USHRT_MAX);
 162                 RANGECHK(ipport_lastauto, IPPORT_RESERVED, USHRT_MAX);
 163
 164                 RANGECHK(ipport_hifirstauto, IPPORT_RESERVED, USHRT_MAX);
 165                 RANGECHK(ipport_hilastauto, IPPORT_RESERVED, USHRT_MAX);
 166         }
 167         return (error);
 168 }
 169
 170 #undef RANGECHK
 171
 172 SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports");
 173
 174 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW,
 175            &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", "");
 176 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, CTLTYPE_INT|CTLFLAG_RW,
 177            &ipport_lowlastauto, 0, &sysctl_net_ipport_check, "I", "");
 178 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, CTLTYPE_INT|CTLFLAG_RW,
 179            &ipport_firstauto, 0, &sysctl_net_ipport_check, "I", "");
 180 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, CTLTYPE_INT|CTLFLAG_RW,
 181            &ipport_lastauto, 0, &sysctl_net_ipport_check, "I", "");
 182 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, CTLTYPE_INT|CTLFLAG_RW,
 183            &ipport_hifirstauto, 0, &sysctl_net_ipport_check, "I", "");
 184 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW,
 185            &ipport_hilastauto, 0, &sysctl_net_ipport_check, "I", "");
 186
 187 /*
 188  * in_pcb.c: manage the Protocol Control Blocks.
 189  *
 190  * NOTE: It is assumed that most of these functions will be called from
 191  * a critical section.  XXX - There are, unfortunately, a few exceptions
 192  * to this rule that should be fixed.
 193  *
 194  * NOTE: The caller should initialize the cpu field to the cpu running the
 195  * protocol stack associated with this inpcbinfo.
 196  */
 197
 198 void
 199 in_pcbinfo_init(struct inpcbinfo *pcbinfo, int cpu, boolean_t shared)
 200 {
 201         KASSERT(cpu >= 0 && cpu < ncpus, ("invalid cpu%d", cpu));
 202         pcbinfo->cpu = cpu;
 203
 204         LIST_INIT(&pcbinfo->pcblisthead);
 205         pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), M_PCB,
 206                                     M_WAITOK | M_ZERO);
 207
 208         if (shared) {
 209                 pcbinfo->infotoken = kmalloc(sizeof(struct lwkt_token),
 210                     M_PCB, M_WAITOK);
 211                 lwkt_token_init(pcbinfo->infotoken, "infotoken");
 212         } else {
 213                 pcbinfo->infotoken = NULL;
 214         }
 215 }
 216
 217 struct baddynamicports baddynamicports;
 218
 219 /*
 220  * Check if the specified port is invalid for dynamic allocation.
 221  */
 222 int
 223 in_baddynamic(u_int16_t port, u_int16_t proto)
 224 {
 225         switch (proto) {
 226         case IPPROTO_TCP:
 227                 return (DP_ISSET(baddynamicports.tcp, port));
 228         case IPPROTO_UDP:
 229 #ifdef IPSEC
 230                 /* Cannot preset this as it is a sysctl */
 231                 if (port == udpencap_port)
 232                         return (1);
 233 #endif
 234                 return (DP_ISSET(baddynamicports.udp, port));
 235         default:
 236                 return (0);
 237         }
 238 }
 239
 240 void
 241 in_pcbonlist(struct inpcb *inp)
 242 {
 243         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 244
 245         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
 246             ("not in the correct netisr"));
 247         KASSERT((inp->inp_flags & INP_ONLIST) == 0, ("already on pcblist"));
 248         inp->inp_flags |= INP_ONLIST;
 249
 250         GET_PCBINFO_TOKEN(pcbinfo);
 251         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list);
 252         pcbinfo->ipi_count++;
 253         REL_PCBINFO_TOKEN(pcbinfo);
 254 }
 255
 256 void
 257 in_pcbofflist(struct inpcb *inp)
 258 {
 259         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 260
 261         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
 262             ("not in the correct netisr"));
 263         KASSERT(inp->inp_flags & INP_ONLIST, ("not on pcblist"));
 264         inp->inp_flags &= ~INP_ONLIST;
 265
 266         GET_PCBINFO_TOKEN(pcbinfo);
 267         LIST_REMOVE(inp, inp_list);
 268         KASSERT(pcbinfo->ipi_count > 0,
 269             ("invalid inpcb count %d", pcbinfo->ipi_count));
 270         pcbinfo->ipi_count--;
 271         REL_PCBINFO_TOKEN(pcbinfo);
 272 }
 273
 274 /*
 275  * Allocate a PCB and associate it with the socket.
 276  */
 277 int
 278 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
 279 {
 280         struct inpcb *inp;
 281 #ifdef IPSEC
 282         int error;
 283 #endif
 284
 285         inp = kmalloc(pcbinfo->ipi_size, M_PCB, M_WAITOK|M_ZERO|M_NULLOK);
 286         if (inp == NULL)
 287                 return (ENOMEM);
 288         inp->inp_lgrpindex = -1;
 289         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
 290         inp->inp_pcbinfo = pcbinfo;
 291         inp->inp_socket = so;
 292 #ifdef IPSEC
 293         error = ipsec_init_policy(so, &inp->inp_sp);
 294         if (error != 0) {
 295                 kfree(inp, M_PCB);
 296                 return (error);
 297         }
 298 #endif
 299 #ifdef INET6
 300         if (INP_SOCKAF(so) == AF_INET6 && ip6_v6only)
 301                 inp->inp_flags |= IN6P_IPV6_V6ONLY;
 302         if (ip6_auto_flowlabel)
 303                 inp->inp_flags |= IN6P_AUTOFLOWLABEL;
 304 #endif
 305         soreference(so);
 306         so->so_pcb = inp;
 307
 308         in_pcbonlist(inp);
 309         return (0);
 310 }
 311
 312 /*
 313  * Unlink a pcb with the intention of moving it to another cpu with a
 314  * different pcbinfo.  While unlinked nothing should attempt to dereference
 315  * inp_pcbinfo, NULL it out so we assert if it does.
 316  */
 317 void
 318 in_pcbunlink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags)
 319 {
 320         KASSERT(inp->inp_pcbinfo == pcbinfo, ("pcbinfo mismatch"));
 321         KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0,
 322             ("already linked"));
 323
 324         in_pcbofflist(inp);
 325         inp->inp_pcbinfo = NULL;
 326 }
 327
 328 void
 329 in_pcbunlink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
 330 {
 331         in_pcbunlink_flags(inp, pcbinfo, INP_WILDCARD);
 332 }
 333
 334 /*
 335  * Relink a pcb into a new pcbinfo.
 336  */
 337 void
 338 in_pcblink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags)
 339 {
 340         KASSERT(inp->inp_pcbinfo == NULL, ("has pcbinfo"));
 341         KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0,
 342             ("already linked"));
 343
 344         inp->inp_pcbinfo = pcbinfo;
 345         in_pcbonlist(inp);
 346 }
 347
 348 void
 349 in_pcblink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
 350 {
 351         return in_pcblink_flags(inp, pcbinfo, INP_WILDCARD);
 352 }
 353
 354 static int
 355 in_pcbsetlport(struct inpcb *inp, int wild, struct ucred *cred)
 356 {
 357         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 358         struct inpcbportinfo *portinfo;
 359         u_short first, last, lport, step;
 360         u_short *lastport;
 361         int count, error;
 362         int portinfo_first, portinfo_idx;
 363
 364         inp->inp_flags |= INP_ANONPORT;
 365
 366         step = pcbinfo->portinfo_mask + 1;
 367         portinfo_first = mycpuid & pcbinfo->portinfo_mask;
 368         portinfo_idx = portinfo_first;
 369 loop:
 370         portinfo = &pcbinfo->portinfo[portinfo_idx];
 371
 372         if (inp->inp_flags & INP_HIGHPORT) {
 373                 first = ipport_hifirstauto;     /* sysctl */
 374                 last  = ipport_hilastauto;
 375                 lastport = &portinfo->lasthi;
 376         } else if (inp->inp_flags & INP_LOWPORT) {
 377                 if (cred &&
 378                     (error =
 379                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 380                         inp->inp_laddr.s_addr = INADDR_ANY;
 381                         return error;
 382                 }
 383                 first = ipport_lowfirstauto;    /* 1023 */
 384                 last  = ipport_lowlastauto;     /* 600 */
 385                 lastport = &portinfo->lastlow;
 386         } else {
 387                 first = ipport_firstauto;       /* sysctl */
 388                 last  = ipport_lastauto;
 389                 lastport = &portinfo->lastport;
 390         }
 391
 392         /*
 393          * This has to be atomic.  If the porthash is shared across multiple
 394          * protocol threads (aka tcp) then the token must be held.
 395          */
 396         GET_PORT_TOKEN(portinfo);
 397
 398         /*
 399          * Simple check to ensure all ports are not used up causing
 400          * a deadlock here.
 401          *
 402          * We split the two cases (up and down) so that the direction
 403          * is not being tested on each round of the loop.
 404          */
 405         if (first > last) {
 406                 /*
 407                  * counting down
 408                  */
 409                 in_pcbportrange(&first, &last, portinfo->offset, step);
 410                 count = (first - last) / step;
 411
 412                 do {
 413                         if (count-- < 0) {      /* completely used? */
 414                                 error = EADDRNOTAVAIL;
 415                                 goto done;
 416                         }
 417                         *lastport -= step;
 418                         if (*lastport > first || *lastport < last)
 419                                 *lastport = first;
 420                         KKASSERT((*lastport & pcbinfo->portinfo_mask) ==
 421                             portinfo->offset);
 422                         lport = htons(*lastport);
 423                 } while (in_pcblookup_local(portinfo, inp->inp_laddr, lport,
 424                     wild, cred));
 425         } else {
 426                 /*
 427                  * counting up
 428                  */
 429                 in_pcbportrange(&last, &first, portinfo->offset, step);
 430                 count = (last - first) / step;
 431
 432                 do {
 433                         if (count-- < 0) {      /* completely used? */
 434                                 error = EADDRNOTAVAIL;
 435                                 goto done;
 436                         }
 437                         *lastport += step;
 438                         if (*lastport < first || *lastport > last)
 439                                 *lastport = first;
 440                         KKASSERT((*lastport & pcbinfo->portinfo_mask) ==
 441                             portinfo->offset);
 442                         lport = htons(*lastport);
 443                 } while (in_pcblookup_local(portinfo, inp->inp_laddr, lport,
 444                     wild, cred));
 445         }
 446         inp->inp_lport = lport;
 447         in_pcbinsporthash(portinfo, inp);
 448         error = 0;
 449 done:
 450         REL_PORT_TOKEN(portinfo);
 451
 452         if (error) {
 453                 /* Try next portinfo */
 454                 portinfo_idx++;
 455                 portinfo_idx &= pcbinfo->portinfo_mask;
 456                 if (portinfo_idx != portinfo_first)
 457                         goto loop;
 458                 inp->inp_laddr.s_addr = INADDR_ANY;
 459         }
 460         return error;
 461 }
 462
 463 int
 464 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
 465 {
 466         struct socket *so = inp->inp_socket;
 467         struct sockaddr_in jsin;
 468         struct ucred *cred = NULL;
 469         int wild = 0;
 470
 471         if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */
 472                 return (EADDRNOTAVAIL);
 473         if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
 474                 return (EINVAL);        /* already bound */
 475
 476         if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT)))
 477                 wild = 1;    /* neither SO_REUSEADDR nor SO_REUSEPORT is set */
 478         if (td->td_proc)
 479                 cred = td->td_proc->p_ucred;
 480
 481         if (nam != NULL) {
 482                 struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 483                 struct inpcbinfo *pcbinfo;
 484                 struct inpcbportinfo *portinfo;
 485                 struct inpcb *t;
 486                 u_short lport, lport_ho;
 487                 int reuseport = (so->so_options & SO_REUSEPORT);
 488                 int error;
 489
 490                 if (nam->sa_len != sizeof *sin)
 491                         return (EINVAL);
 492 #ifdef notdef
 493                 /*
 494                  * We should check the family, but old programs
 495                  * incorrectly fail to initialize it.
 496                  */
 497                 if (sin->sin_family != AF_INET)
 498                         return (EAFNOSUPPORT);
 499 #endif
 500                 if (!prison_replace_wildcards(td, nam))
 501                         return (EINVAL);
 502
 503                 lport = sin->sin_port;
 504                 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 505                         /*
 506                          * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
 507                          * allow complete duplication of binding if
 508                          * SO_REUSEPORT is set, or if SO_REUSEADDR is set
 509                          * and a multicast address is bound on both
 510                          * new and duplicated sockets.
 511                          */
 512                         if (so->so_options & SO_REUSEADDR)
 513                                 reuseport = SO_REUSEADDR | SO_REUSEPORT;
 514                 } else if (sin->sin_addr.s_addr != INADDR_ANY) {
 515                         sin->sin_port = 0;              /* yech... */
 516                         bzero(&sin->sin_zero, sizeof sin->sin_zero);
 517                         if (ifa_ifwithaddr((struct sockaddr *)sin) == NULL)
 518                                 return (EADDRNOTAVAIL);
 519                 }
 520
 521                 inp->inp_laddr = sin->sin_addr;
 522
 523                 jsin.sin_family = AF_INET;
 524                 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 525                 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 526                         inp->inp_laddr.s_addr = INADDR_ANY;
 527                         return (EINVAL);
 528                 }
 529                 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 530
 531                 if (lport == 0) {
 532                         /* Auto-select local port */
 533                         return in_pcbsetlport(inp, wild, cred);
 534                 }
 535                 lport_ho = ntohs(lport);
 536
 537                 /* GROSS */
 538                 if (lport_ho < IPPORT_RESERVED && cred &&
 539                     (error =
 540                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 541                         inp->inp_laddr.s_addr = INADDR_ANY;
 542                         return (error);
 543                 }
 544
 545                 /*
 546                  * Locate the proper portinfo based on lport
 547                  */
 548                 pcbinfo = inp->inp_pcbinfo;
 549                 portinfo =
 550                     &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask];
 551                 KKASSERT((lport_ho & pcbinfo->portinfo_mask) ==
 552                     portinfo->offset);
 553
 554                 /*
 555                  * This has to be atomic.  If the porthash is shared across
 556                  * multiple protocol threads (aka tcp) then the token must
 557                  * be held.
 558                  */
 559                 GET_PORT_TOKEN(portinfo);
 560
 561                 if (so->so_cred->cr_uid != 0 &&
 562                     !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 563                         t = in_pcblookup_local(portinfo, sin->sin_addr, lport,
 564                             INPLOOKUP_WILDCARD, cred);
 565                         if (t &&
 566                             (!in_nullhost(sin->sin_addr) ||
 567                              !in_nullhost(t->inp_laddr) ||
 568                              (t->inp_socket->so_options & SO_REUSEPORT) == 0) &&
 569                             (so->so_cred->cr_uid !=
 570                              t->inp_socket->so_cred->cr_uid)) {
 571 #ifdef INET6
 572                                 if (!in_nullhost(sin->sin_addr) ||
 573                                     !in_nullhost(t->inp_laddr) ||
 574                                     INP_SOCKAF(so) == INP_SOCKAF(t->inp_socket))
 575 #endif
 576                                 {
 577                                         inp->inp_laddr.s_addr = INADDR_ANY;
 578                                         error = EADDRINUSE;
 579                                         goto done;
 580                                 }
 581                         }
 582                 }
 583                 if (cred && !prison_replace_wildcards(td, nam)) {
 584                         inp->inp_laddr.s_addr = INADDR_ANY;
 585                         error = EADDRNOTAVAIL;
 586                         goto done;
 587                 }
 588                 t = in_pcblookup_local(portinfo, sin->sin_addr, lport,
 589                     wild, cred);
 590                 if (t && !(reuseport & t->inp_socket->so_options)) {
 591 #ifdef INET6
 592                         if (!in_nullhost(sin->sin_addr) ||
 593                             !in_nullhost(t->inp_laddr) ||
 594                             INP_SOCKAF(so) == INP_SOCKAF(t->inp_socket))
 595 #endif
 596                         {
 597                                 inp->inp_laddr.s_addr = INADDR_ANY;
 598                                 error = EADDRINUSE;
 599                                 goto done;
 600                         }
 601                 }
 602                 inp->inp_lport = lport;
 603                 in_pcbinsporthash(portinfo, inp);
 604                 error = 0;
 605 done:
 606                 REL_PORT_TOKEN(portinfo);
 607                 return (error);
 608         } else {
 609                 jsin.sin_family = AF_INET;
 610                 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 611                 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 612                         inp->inp_laddr.s_addr = INADDR_ANY;
 613                         return (EINVAL);
 614                 }
 615                 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 616
 617                 return in_pcbsetlport(inp, wild, cred);
 618         }
 619 }
 620
 621 static struct inpcb *
 622 in_pcblookup_localremote(struct inpcbportinfo *portinfo, struct in_addr laddr,
 623     u_short lport, struct in_addr faddr, u_short fport, struct ucred *cred)
 624 {
 625         struct inpcb *inp;
 626         struct inpcbporthead *porthash;
 627         struct inpcbport *phd;
 628         struct inpcb *match = NULL;
 629
 630         /*
 631          * If the porthashbase is shared across several cpus, it must
 632          * have been locked.
 633          */
 634         ASSERT_PORT_TOKEN_HELD(portinfo);
 635
 636         /*
 637          * Best fit PCB lookup.
 638          *
 639          * First see if this local port is in use by looking on the
 640          * port hash list.
 641          */
 642         porthash = &portinfo->porthashbase[
 643                         INP_PCBPORTHASH(lport, portinfo->porthashmask)];
 644         LIST_FOREACH(phd, porthash, phd_hash) {
 645                 if (phd->phd_port == lport)
 646                         break;
 647         }
 648         if (phd != NULL) {
 649                 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
 650 #ifdef INET6
 651                         if ((inp->inp_vflag & INP_IPV4) == 0)
 652                                 continue;
 653 #endif
 654                         if (inp->inp_laddr.s_addr != INADDR_ANY &&
 655                             inp->inp_laddr.s_addr != laddr.s_addr)
 656                                 continue;
 657
 658                         if (inp->inp_faddr.s_addr != INADDR_ANY &&
 659                             inp->inp_faddr.s_addr != faddr.s_addr)
 660                                 continue;
 661
 662                         if (inp->inp_fport != 0 && inp->inp_fport != fport)
 663                                 continue;
 664
 665                         if (cred == NULL ||
 666                             cred->cr_prison ==
 667                             inp->inp_socket->so_cred->cr_prison) {
 668                                 match = inp;
 669                                 break;
 670                         }
 671                 }
 672         }
 673         return (match);
 674 }
 675
 676 int
 677 in_pcbbind_remote(struct inpcb *inp, const struct sockaddr *remote,
 678     struct thread *td)
 679 {
 680         struct proc *p = td->td_proc;
 681         unsigned short *lastport;
 682         const struct sockaddr_in *sin = (const struct sockaddr_in *)remote;
 683         struct sockaddr_in jsin;
 684         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 685         struct inpcbportinfo *portinfo;
 686         struct ucred *cred = NULL;
 687         u_short first, last, lport, step;
 688         int count, error, dup;
 689         int portinfo_first, portinfo_idx;
 690
 691         if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */
 692                 return (EADDRNOTAVAIL);
 693
 694         KKASSERT(inp->inp_laddr.s_addr != INADDR_ANY);
 695         if (inp->inp_lport != 0)
 696                 return (EINVAL);        /* already bound */
 697
 698         KKASSERT(p);
 699         cred = p->p_ucred;
 700
 701         jsin.sin_family = AF_INET;
 702         jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
 703         if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
 704                 inp->inp_laddr.s_addr = INADDR_ANY;
 705                 return (EINVAL);
 706         }
 707         inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
 708
 709         inp->inp_flags |= INP_ANONPORT;
 710
 711         step = pcbinfo->portinfo_mask + 1;
 712         portinfo_first = mycpuid & pcbinfo->portinfo_mask;
 713         portinfo_idx = portinfo_first;
 714 loop:
 715         portinfo = &pcbinfo->portinfo[portinfo_idx];
 716         dup = 0;
 717
 718         if (inp->inp_flags & INP_HIGHPORT) {
 719                 first = ipport_hifirstauto;     /* sysctl */
 720                 last  = ipport_hilastauto;
 721                 lastport = &portinfo->lasthi;
 722         } else if (inp->inp_flags & INP_LOWPORT) {
 723                 if (cred &&
 724                     (error =
 725                      priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
 726                         inp->inp_laddr.s_addr = INADDR_ANY;
 727                         return (error);
 728                 }
 729                 first = ipport_lowfirstauto;    /* 1023 */
 730                 last  = ipport_lowlastauto;     /* 600 */
 731                 lastport = &portinfo->lastlow;
 732         } else {
 733                 first = ipport_firstauto;       /* sysctl */
 734                 last  = ipport_lastauto;
 735                 lastport = &portinfo->lastport;
 736         }
 737
 738         /*
 739          * This has to be atomic.  If the porthash is shared across multiple
 740          * protocol threads (aka tcp) then the token must be held.
 741          */
 742         GET_PORT_TOKEN(portinfo);
 743
 744 again:
 745         /*
 746          * Simple check to ensure all ports are not used up causing
 747          * a deadlock here.
 748          *
 749          * We split the two cases (up and down) so that the direction
 750          * is not being tested on each round of the loop.
 751          */
 752         if (first > last) {
 753                 /*
 754                  * counting down
 755                  */
 756                 in_pcbportrange(&first, &last, portinfo->offset, step);
 757                 count = (first - last) / step;
 758
 759                 do {
 760                         if (count-- < 0) {      /* completely used? */
 761                                 error = EADDRNOTAVAIL;
 762                                 goto done;
 763                         }
 764                         *lastport -= step;
 765                         if (*lastport > first || *lastport < last)
 766                                 *lastport = first;
 767                         KKASSERT((*lastport & pcbinfo->portinfo_mask) ==
 768                             portinfo->offset);
 769                         lport = htons(*lastport);
 770                 } while (in_pcblookup_localremote(portinfo, inp->inp_laddr,
 771                     lport, sin->sin_addr, sin->sin_port, cred));
 772         } else {
 773                 /*
 774                  * counting up
 775                  */
 776                 in_pcbportrange(&last, &first, portinfo->offset, step);
 777                 count = (last - first) / step;
 778
 779                 do {
 780                         if (count-- < 0) {      /* completely used? */
 781                                 error = EADDRNOTAVAIL;
 782                                 goto done;
 783                         }
 784                         *lastport += step;
 785                         if (*lastport < first || *lastport > last)
 786                                 *lastport = first;
 787                         KKASSERT((*lastport & pcbinfo->portinfo_mask) ==
 788                             portinfo->offset);
 789                         lport = htons(*lastport);
 790                 } while (in_pcblookup_localremote(portinfo, inp->inp_laddr,
 791                     lport, sin->sin_addr, sin->sin_port, cred));
 792         }
 793
 794         /* This could happen on loopback interface */
 795         if (sin->sin_port == lport &&
 796             sin->sin_addr.s_addr == inp->inp_laddr.s_addr) {
 797                 if (dup) {
 798                         /*
 799                          * Duplicate again; give up
 800                          */
 801                         error = EADDRNOTAVAIL;
 802                         goto done;
 803                 }
 804                 dup = 1;
 805                 goto again;
 806         }
 807         inp->inp_lport = lport;
 808         in_pcbinsporthash(portinfo, inp);
 809         error = 0;
 810 done:
 811         REL_PORT_TOKEN(portinfo);
 812
 813         if (error) {
 814                 /* Try next portinfo */
 815                 portinfo_idx++;
 816                 portinfo_idx &= pcbinfo->portinfo_mask;
 817                 if (portinfo_idx != portinfo_first)
 818                         goto loop;
 819                 inp->inp_laddr.s_addr = INADDR_ANY;
 820         }
 821         return error;
 822 }
 823
 824 /*
 825  *   Transform old in_pcbconnect() into an inner subroutine for new
 826  *   in_pcbconnect(): Do some validity-checking on the remote
 827  *   address (in mbuf 'nam') and then determine local host address
 828  *   (i.e., which interface) to use to access that remote host.
 829  *
 830  *   This preserves definition of in_pcbconnect(), while supporting a
 831  *   slightly different version for T/TCP.  (This is more than
 832  *   a bit of a kludge, but cleaning up the internal interfaces would
 833  *   have forced minor changes in every protocol).
 834  */
 835 int
 836 in_pcbladdr_find(struct inpcb *inp, struct sockaddr *nam,
 837     struct sockaddr_in **plocal_sin, struct thread *td, int find)
 838 {
 839         struct in_ifaddr *ia;
 840         struct ucred *cred = NULL;
 841         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 842         struct sockaddr *jsin;
 843         int jailed = 0, alloc_route = 0;
 844
 845         if (nam->sa_len != sizeof *sin)
 846                 return (EINVAL);
 847         if (sin->sin_family != AF_INET)
 848                 return (EAFNOSUPPORT);
 849         if (sin->sin_port == 0)
 850                 return (EADDRNOTAVAIL);
 851         if (td && td->td_proc && td->td_proc->p_ucred)
 852                 cred = td->td_proc->p_ucred;
 853         if (cred && cred->cr_prison)
 854                 jailed = 1;
 855         if (!TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) {
 856                 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia;
 857                 /*
 858                  * If the destination address is INADDR_ANY,
 859                  * use the primary local address.
 860                  * If the supplied address is INADDR_BROADCAST,
 861                  * and the primary interface supports broadcast,
 862                  * choose the broadcast address for that interface.
 863                  */
 864                 if (sin->sin_addr.s_addr == INADDR_ANY)
 865                         sin->sin_addr = IA_SIN(ia)->sin_addr;
 866                 else if (sin->sin_addr.s_addr == (u_long)INADDR_BROADCAST &&
 867                     (ia->ia_ifp->if_flags & IFF_BROADCAST))
 868                         sin->sin_addr = satosin(&ia->ia_broadaddr)->sin_addr;
 869         }
 870         if (find) {
 871                 struct route *ro;
 872
 873                 ia = NULL;
 874                 /*
 875                  * If route is known or can be allocated now,
 876                  * our src addr is taken from the i/f, else punt.
 877                  * Note that we should check the address family of the cached
 878                  * destination, in case of sharing the cache with IPv6.
 879                  */
 880                 ro = &inp->inp_route;
 881                 if (ro->ro_rt &&
 882                     (!(ro->ro_rt->rt_flags & RTF_UP) ||
 883                      ro->ro_dst.sa_family != AF_INET ||
 884                      satosin(&ro->ro_dst)->sin_addr.s_addr !=
 885                                       sin->sin_addr.s_addr ||
 886                      inp->inp_socket->so_options & SO_DONTROUTE)) {
 887                         RTFREE(ro->ro_rt);
 888                         ro->ro_rt = NULL;
 889                 }
 890                 if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/
 891                     (ro->ro_rt == NULL ||
 892                     ro->ro_rt->rt_ifp == NULL)) {
 893                         /* No route yet, so try to acquire one */
 894                         bzero(&ro->ro_dst, sizeof(struct sockaddr_in));
 895                         ro->ro_dst.sa_family = AF_INET;
 896                         ro->ro_dst.sa_len = sizeof(struct sockaddr_in);
 897                         ((struct sockaddr_in *) &ro->ro_dst)->sin_addr =
 898                                 sin->sin_addr;
 899                         rtalloc(ro);
 900                         alloc_route = 1;
 901                 }
 902                 /*
 903                  * If we found a route, use the address
 904                  * corresponding to the outgoing interface
 905                  * unless it is the loopback (in case a route
 906                  * to our address on another net goes to loopback).
 907                  */
 908                 if (ro->ro_rt &&
 909                     !(ro->ro_rt->rt_ifp->if_flags & IFF_LOOPBACK)) {
 910                         if (jailed) {
 911                                 if (jailed_ip(cred->cr_prison,
 912                                     ro->ro_rt->rt_ifa->ifa_addr)) {
 913                                         ia = ifatoia(ro->ro_rt->rt_ifa);
 914                                 }
 915                         } else {
 916                                 ia = ifatoia(ro->ro_rt->rt_ifa);
 917                         }
 918                 }
 919                 if (ia == NULL) {
 920                         u_short fport = sin->sin_port;
 921
 922                         sin->sin_port = 0;
 923                         ia = ifatoia(ifa_ifwithdstaddr(sintosa(sin)));
 924                         if (ia && jailed && !jailed_ip(cred->cr_prison,
 925                             sintosa(&ia->ia_addr)))
 926                                 ia = NULL;
 927                         if (ia == NULL)
 928                                 ia = ifatoia(ifa_ifwithnet(sintosa(sin)));
 929                         if (ia && jailed && !jailed_ip(cred->cr_prison,
 930                             sintosa(&ia->ia_addr)))
 931                                 ia = NULL;
 932                         sin->sin_port = fport;
 933                         if (ia == NULL &&
 934                             !TAILQ_EMPTY(&in_ifaddrheads[mycpuid]))
 935                                 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia;
 936                         if (ia && jailed && !jailed_ip(cred->cr_prison,
 937                             sintosa(&ia->ia_addr)))
 938                                 ia = NULL;
 939
 940                         if (!jailed && ia == NULL)
 941                                 goto fail;
 942                 }
 943                 /*
 944                  * If the destination address is multicast and an outgoing
 945                  * interface has been set as a multicast option, use the
 946                  * address of that interface as our source address.
 947                  */
 948                 if (!jailed && IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
 949                     inp->inp_moptions != NULL) {
 950                         struct ip_moptions *imo;
 951                         struct ifnet *ifp;
 952
 953                         imo = inp->inp_moptions;
 954                         if (imo->imo_multicast_ifp != NULL) {
 955                                 struct in_ifaddr_container *iac;
 956
 957                                 ifp = imo->imo_multicast_ifp;
 958                                 ia = NULL;
 959                                 TAILQ_FOREACH(iac,
 960                                 &in_ifaddrheads[mycpuid], ia_link) {
 961                                         if (iac->ia->ia_ifp == ifp) {
 962                                                 ia = iac->ia;
 963                                                 break;
 964                                         }
 965                                 }
 966                                 if (ia == NULL)
 967                                         goto fail;
 968                         }
 969                 }
 970                 /*
 971                  * Don't do pcblookup call here; return interface in plocal_sin
 972                  * and exit to caller, that will do the lookup.
 973                  */
 974                 if (ia == NULL && jailed) {
 975                         if ((jsin = prison_get_nonlocal(
 976                                 cred->cr_prison, AF_INET, NULL)) != NULL ||
 977                             (jsin = prison_get_local(
 978                                 cred->cr_prison, AF_INET, NULL)) != NULL) {
 979                                 *plocal_sin = satosin(jsin);
 980                         } else {
 981                                 /* IPv6 only Jail */
 982                                 goto fail;
 983                         }
 984                 } else {
 985                         *plocal_sin = &ia->ia_addr;
 986                 }
 987         }
 988         return (0);
 989 fail:
 990         if (alloc_route)
 991                 in_pcbresetroute(inp);
 992         return (EADDRNOTAVAIL);
 993 }
 994
 995 int
 996 in_pcbladdr(struct inpcb *inp, struct sockaddr *nam,
 997     struct sockaddr_in **plocal_sin, struct thread *td)
 998 {
 999         return in_pcbladdr_find(inp, nam, plocal_sin, td,
1000             (inp->inp_laddr.s_addr == INADDR_ANY));
1001 }
1002
1003 /*
1004  * Outer subroutine:
1005  * Connect from a socket to a specified address.
1006  * Both address and port must be specified in argument sin.
1007  * If don't have a local address for this socket yet,
1008  * then pick one.
1009  */
1010 int
1011 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
1012 {
1013         struct sockaddr_in *if_sin;
1014         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
1015         int error;
1016
1017         /* Call inner routine to assign local interface address. */
1018         if ((error = in_pcbladdr(inp, nam, &if_sin, td)) != 0)
1019                 return (error);
1020
1021         if (in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port,
1022                               inp->inp_laddr.s_addr ?
1023                                 inp->inp_laddr : if_sin->sin_addr,
1024                               inp->inp_lport, FALSE, NULL) != NULL) {
1025                 return (EADDRINUSE);
1026         }
1027         if (inp->inp_laddr.s_addr == INADDR_ANY) {
1028                 if (inp->inp_lport == 0) {
1029                         error = in_pcbbind(inp, NULL, td);
1030                         if (error)
1031                                 return (error);
1032                 }
1033                 inp->inp_laddr = if_sin->sin_addr;
1034         }
1035         inp->inp_faddr = sin->sin_addr;
1036         inp->inp_fport = sin->sin_port;
1037         in_pcbinsconnhash(inp);
1038         return (0);
1039 }
1040
1041 void
1042 in_pcbdisconnect(struct inpcb *inp)
1043 {
1044
1045         in_pcbremconnhash(inp);
1046         inp->inp_faddr.s_addr = INADDR_ANY;
1047         inp->inp_fport = 0;
1048 }
1049
1050 void
1051 in_pcbdetach(struct inpcb *inp)
1052 {
1053         struct socket *so = inp->inp_socket;
1054         struct inpcbinfo *ipi = inp->inp_pcbinfo;
1055
1056 #ifdef IPSEC
1057         ipsec4_delete_pcbpolicy(inp);
1058 #endif /*IPSEC*/
1059         inp->inp_gencnt = ++ipi->ipi_gencnt;
1060         KKASSERT((so->so_state & SS_ASSERTINPROG) == 0);
1061         in_pcbremlists(inp);
1062         so->so_pcb = NULL;
1063         sofree(so);                     /* remove pcb ref */
1064         if (inp->inp_options)
1065                 m_free(inp->inp_options);
1066         if (inp->inp_route.ro_rt)
1067                 rtfree(inp->inp_route.ro_rt);
1068         ip_freemoptions(inp->inp_moptions);
1069         inp->inp_vflag = 0;
1070         kfree(inp, M_PCB);
1071 }
1072
1073 /*
1074  * The calling convention of in_setsockaddr() and in_setpeeraddr() was
1075  * modified to match the pru_sockaddr() and pru_peeraddr() entry points
1076  * in struct pr_usrreqs, so that protocols can just reference then directly
1077  * without the need for a wrapper function.  The socket must have a valid
1078  * (i.e., non-nil) PCB, but it should be impossible to get an invalid one
1079  * except through a kernel programming error, so it is acceptable to panic
1080  * (or in this case trap) if the PCB is invalid.  (Actually, we don't trap
1081  * because there actually /is/ a programming error somewhere... XXX)
1082  */
1083 int
1084 in_setsockaddr(struct socket *so, struct sockaddr **nam)
1085 {
1086         struct inpcb *inp;
1087         struct sockaddr_in *sin;
1088
1089         /*
1090          * Do the malloc first in case it blocks.
1091          */
1092         sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1093         sin->sin_family = AF_INET;
1094         sin->sin_len = sizeof *sin;
1095
1096         crit_enter();
1097         inp = so->so_pcb;
1098         if (!inp) {
1099                 crit_exit();
1100                 kfree(sin, M_SONAME);
1101                 return (ECONNRESET);
1102         }
1103         sin->sin_port = inp->inp_lport;
1104         sin->sin_addr = inp->inp_laddr;
1105         crit_exit();
1106
1107         *nam = (struct sockaddr *)sin;
1108         return (0);
1109 }
1110
1111 void
1112 in_setsockaddr_dispatch(netmsg_t msg)
1113 {
1114         int error;
1115
1116         error = in_setsockaddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1117         lwkt_replymsg(&msg->lmsg, error);
1118 }
1119
1120 int
1121 in_setpeeraddr(struct socket *so, struct sockaddr **nam)
1122 {
1123         struct inpcb *inp;
1124         struct sockaddr_in *sin;
1125
1126         /*
1127          * Do the malloc first in case it blocks.
1128          */
1129         sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1130         sin->sin_family = AF_INET;
1131         sin->sin_len = sizeof *sin;
1132
1133         crit_enter();
1134         inp = so->so_pcb;
1135         if (!inp) {
1136                 crit_exit();
1137                 kfree(sin, M_SONAME);
1138                 return (ECONNRESET);
1139         }
1140         sin->sin_port = inp->inp_fport;
1141         sin->sin_addr = inp->inp_faddr;
1142         crit_exit();
1143
1144         *nam = (struct sockaddr *)sin;
1145         return (0);
1146 }
1147
1148 void
1149 in_setpeeraddr_dispatch(netmsg_t msg)
1150 {
1151         int error;
1152
1153         error = in_setpeeraddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1154         lwkt_replymsg(&msg->lmsg, error);
1155 }
1156
1157 void
1158 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int err,
1159                 void (*notify)(struct inpcb *, int))
1160 {
1161         struct inpcb *inp, *marker;
1162
1163         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1164             ("not in the correct netisr"));
1165         marker = &in_pcbmarkers[mycpuid];
1166
1167         /*
1168          * NOTE:
1169          * - If INP_PLACEMARKER is set we must ignore the rest of the
1170          *   structure and skip it.
1171          * - It is safe to nuke inpcbs here, since we are in their own
1172          *   netisr.
1173          */
1174         GET_PCBINFO_TOKEN(pcbinfo);
1175
1176         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1177         while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1178                 LIST_REMOVE(marker, inp_list);
1179                 LIST_INSERT_AFTER(inp, marker, inp_list);
1180
1181                 if (inp->inp_flags & INP_PLACEMARKER)
1182                         continue;
1183 #ifdef INET6
1184                 if (!(inp->inp_vflag & INP_IPV4))
1185                         continue;
1186 #endif
1187                 if (inp->inp_faddr.s_addr != faddr.s_addr ||
1188                     inp->inp_socket == NULL)
1189                         continue;
1190                 (*notify)(inp, err);            /* can remove inp from list! */
1191         }
1192         LIST_REMOVE(marker, inp_list);
1193
1194         REL_PCBINFO_TOKEN(pcbinfo);
1195 }
1196
1197 void
1198 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1199 {
1200         struct inpcb *inp, *marker;
1201
1202         /*
1203          * We only need to make sure that we are in netisr0, where all
1204          * multicast operation happen.  We could check inpcbinfo which
1205          * does not belong to netisr0 by holding the inpcbinfo's token.
1206          * In this case, the pcbinfo must be able to be shared, i.e.
1207          * pcbinfo->infotoken is not NULL.
1208          */
1209         KASSERT(&curthread->td_msgport == netisr_cpuport(0),
1210             ("not in netisr0"));
1211         KASSERT(pcbinfo->cpu == 0 || pcbinfo->infotoken != NULL,
1212             ("pcbinfo could not be shared"));
1213
1214         /*
1215          * Get a marker for the current netisr (netisr0).
1216          *
1217          * It is possible that the multicast address deletion blocks,
1218          * which could cause temporary token releasing.  So we use
1219          * inpcb marker here to get a coherent view of the inpcb list.
1220          *
1221          * While, on the other hand, moptions are only added and deleted
1222          * in netisr0, so we would not see staled moption or miss moption
1223          * even if the token was released due to the blocking multicast
1224          * address deletion.
1225          */
1226         marker = &in_pcbmarkers[mycpuid];
1227
1228         GET_PCBINFO_TOKEN(pcbinfo);
1229
1230         LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1231         while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1232                 struct ip_moptions *imo;
1233
1234                 LIST_REMOVE(marker, inp_list);
1235                 LIST_INSERT_AFTER(inp, marker, inp_list);
1236
1237                 if (inp->inp_flags & INP_PLACEMARKER)
1238                         continue;
1239                 imo = inp->inp_moptions;
1240                 if ((inp->inp_vflag & INP_IPV4) && imo != NULL) {
1241                         int i, gap;
1242
1243                         /*
1244                          * Unselect the outgoing interface if it is being
1245                          * detached.
1246                          */
1247                         if (imo->imo_multicast_ifp == ifp)
1248                                 imo->imo_multicast_ifp = NULL;
1249
1250                         /*
1251                          * Drop multicast group membership if we joined
1252                          * through the interface being detached.
1253                          */
1254                         for (i = 0, gap = 0; i < imo->imo_num_memberships;
1255                             i++) {
1256                                 if (imo->imo_membership[i]->inm_ifp == ifp) {
1257                                         /*
1258                                          * NOTE:
1259                                          * This could block and the pcbinfo
1260                                          * token could be passively released.
1261                                          */
1262                                         in_delmulti(imo->imo_membership[i]);
1263                                         gap++;
1264                                 } else if (gap != 0)
1265                                         imo->imo_membership[i - gap] =
1266                                             imo->imo_membership[i];
1267                         }
1268                         imo->imo_num_memberships -= gap;
1269                 }
1270         }
1271         LIST_REMOVE(marker, inp_list);
1272
1273         REL_PCBINFO_TOKEN(pcbinfo);
1274 }
1275
1276 /*
1277  * Check for alternatives when higher level complains
1278  * about service problems.  For now, invalidate cached
1279  * routing information.  If the route was created dynamically
1280  * (by a redirect), time to try a default gateway again.
1281  */
1282 void
1283 in_losing(struct inpcb *inp)
1284 {
1285         struct rtentry *rt;
1286         struct rt_addrinfo rtinfo;
1287
1288         if ((rt = inp->inp_route.ro_rt)) {
1289                 bzero(&rtinfo, sizeof(struct rt_addrinfo));
1290                 rtinfo.rti_info[RTAX_DST] = rt_key(rt);
1291                 rtinfo.rti_info[RTAX_GATEWAY] = rt->rt_gateway;
1292                 rtinfo.rti_info[RTAX_NETMASK] = rt_mask(rt);
1293                 rtinfo.rti_flags = rt->rt_flags;
1294                 rt_missmsg(RTM_LOSING, &rtinfo, rt->rt_flags, 0);
1295                 if (rt->rt_flags & RTF_DYNAMIC) {
1296                         rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway,
1297                             rt_mask(rt), rt->rt_flags, NULL);
1298                 }
1299                 inp->inp_route.ro_rt = NULL;
1300                 rtfree(rt);
1301                 /*
1302                  * A new route can be allocated
1303                  * the next time output is attempted.
1304                  */
1305         }
1306 }
1307
1308 /*
1309  * After a routing change, flush old routing
1310  * and allocate a (hopefully) better one.
1311  */
1312 void
1313 in_rtchange(struct inpcb *inp, int err)
1314 {
1315         if (inp->inp_route.ro_rt) {
1316                 rtfree(inp->inp_route.ro_rt);
1317                 inp->inp_route.ro_rt = NULL;
1318                 /*
1319                  * A new route can be allocated the next time
1320                  * output is attempted.
1321                  */
1322         }
1323 }
1324
1325 /*
1326  * Lookup a PCB based on the local address and port.
1327  */
1328 struct inpcb *
1329 in_pcblookup_local(struct inpcbportinfo *portinfo, struct in_addr laddr,
1330                    u_int lport_arg, int wild_okay, struct ucred *cred)
1331 {
1332         struct inpcb *inp;
1333         int matchwild = 3, wildcard;
1334         u_short lport = lport_arg;
1335         struct inpcbporthead *porthash;
1336         struct inpcbport *phd;
1337         struct inpcb *match = NULL;
1338
1339         /*
1340          * If the porthashbase is shared across several cpus, it must
1341          * have been locked.
1342          */
1343         ASSERT_PORT_TOKEN_HELD(portinfo);
1344
1345         /*
1346          * Best fit PCB lookup.
1347          *
1348          * First see if this local port is in use by looking on the
1349          * port hash list.
1350          */
1351         porthash = &portinfo->porthashbase[
1352                         INP_PCBPORTHASH(lport, portinfo->porthashmask)];
1353         LIST_FOREACH(phd, porthash, phd_hash) {
1354                 if (phd->phd_port == lport)
1355                         break;
1356         }
1357         if (phd != NULL) {
1358                 /*
1359                  * Port is in use by one or more PCBs. Look for best
1360                  * fit.
1361                  */
1362                 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
1363                         wildcard = 0;
1364 #ifdef INET6
1365                         if ((inp->inp_vflag & INP_IPV4) == 0)
1366                                 continue;
1367 #endif
1368                         if (inp->inp_faddr.s_addr != INADDR_ANY)
1369                                 wildcard++;
1370                         if (inp->inp_laddr.s_addr != INADDR_ANY) {
1371                                 if (laddr.s_addr == INADDR_ANY)
1372                                         wildcard++;
1373                                 else if (inp->inp_laddr.s_addr != laddr.s_addr)
1374                                         continue;
1375                         } else {
1376                                 if (laddr.s_addr != INADDR_ANY)
1377                                         wildcard++;
1378                         }
1379                         if (wildcard && !wild_okay)
1380                                 continue;
1381                         if (wildcard < matchwild &&
1382                             (cred == NULL ||
1383                              cred->cr_prison ==
1384                                         inp->inp_socket->so_cred->cr_prison)) {
1385                                 match = inp;
1386                                 matchwild = wildcard;
1387                                 if (matchwild == 0) {
1388                                         break;
1389                                 }
1390                         }
1391                 }
1392         }
1393         return (match);
1394 }
1395
1396 struct inpcb *
1397 in_pcblocalgroup_last(const struct inpcbinfo *pcbinfo,
1398     const struct inpcb *inp)
1399 {
1400         const struct inp_localgrphead *hdr;
1401         const struct inp_localgroup *grp;
1402         int i;
1403
1404         if (pcbinfo->localgrphashbase == NULL)
1405                 return NULL;
1406
1407         GET_PCBINFO_TOKEN(pcbinfo);
1408
1409         hdr = &pcbinfo->localgrphashbase[
1410             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
1411
1412         LIST_FOREACH(grp, hdr, il_list) {
1413                 if (grp->il_vflag == inp->inp_vflag &&
1414                     grp->il_lport == inp->inp_lport &&
1415                     memcmp(&grp->il_dependladdr,
1416                         &inp->inp_inc.inc_ie.ie_dependladdr,
1417                         sizeof(grp->il_dependladdr)) == 0) {
1418                         break;
1419                 }
1420         }
1421         if (grp == NULL || grp->il_inpcnt == 1) {
1422                 REL_PCBINFO_TOKEN(pcbinfo);
1423                 return NULL;
1424         }
1425
1426         KASSERT(grp->il_inpcnt >= 2,
1427             ("invalid localgroup inp count %d", grp->il_inpcnt));
1428         for (i = 0; i < grp->il_inpcnt; ++i) {
1429                 if (grp->il_inp[i] == inp) {
1430                         int last = grp->il_inpcnt - 1;
1431
1432                         if (i == last)
1433                                 last = grp->il_inpcnt - 2;
1434                         REL_PCBINFO_TOKEN(pcbinfo);
1435                         return grp->il_inp[last];
1436                 }
1437         }
1438         REL_PCBINFO_TOKEN(pcbinfo);
1439         return NULL;
1440 }
1441
1442 static struct inpcb *
1443 inp_localgroup_lookup(const struct inpcbinfo *pcbinfo,
1444     struct in_addr laddr, uint16_t lport, uint32_t pkt_hash)
1445 {
1446         struct inpcb *local_wild = NULL;
1447         const struct inp_localgrphead *hdr;
1448         const struct inp_localgroup *grp;
1449
1450         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1451
1452         hdr = &pcbinfo->localgrphashbase[
1453             INP_PCBLOCALGRPHASH(lport, pcbinfo->localgrphashmask)];
1454
1455         /*
1456          * Order of socket selection:
1457          * 1. non-wild.
1458          * 2. wild.
1459          *
1460          * NOTE:
1461          * - Local group does not contain jailed sockets
1462          * - Local group does not contain IPv4 mapped INET6 wild sockets
1463          */
1464         LIST_FOREACH(grp, hdr, il_list) {
1465 #ifdef INET6
1466                 if (!(grp->il_vflag & INP_IPV4))
1467                         continue;
1468 #endif
1469                 if (grp->il_lport == lport) {
1470                         int idx;
1471
1472                         /*
1473                          * Modulo-N is used here, which greatly reduces
1474                          * completion queue token contention, thus more
1475                          * cpu time is saved.
1476                          */
1477                         idx = pkt_hash % grp->il_inpcnt;
1478                         if (grp->il_laddr.s_addr == laddr.s_addr)
1479                                 return grp->il_inp[idx];
1480                         else if (grp->il_laddr.s_addr == INADDR_ANY)
1481                                 local_wild = grp->il_inp[idx];
1482                 }
1483         }
1484         if (local_wild != NULL)
1485                 return local_wild;
1486         return NULL;
1487 }
1488
1489 /*
1490  * Lookup PCB in hash list.
1491  */
1492 struct inpcb *
1493 in_pcblookup_pkthash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1494     u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1495     boolean_t wildcard, struct ifnet *ifp, const struct mbuf *m)
1496 {
1497         struct inpcbhead *head;
1498         struct inpcb *inp, *jinp=NULL;
1499         u_short fport = fport_arg, lport = lport_arg;
1500
1501         /*
1502          * First look for an exact match.
1503          */
1504         head = &pcbinfo->hashbase[INP_PCBCONNHASH(faddr.s_addr, fport,
1505             laddr.s_addr, lport, pcbinfo->hashmask)];
1506         LIST_FOREACH(inp, head, inp_hash) {
1507 #ifdef INET6
1508                 if (!(inp->inp_vflag & INP_IPV4))
1509                         continue;
1510 #endif
1511                 if (in_hosteq(inp->inp_faddr, faddr) &&
1512                     in_hosteq(inp->inp_laddr, laddr) &&
1513                     inp->inp_fport == fport && inp->inp_lport == lport) {
1514                         /* found */
1515                         if (inp->inp_socket == NULL ||
1516                             inp->inp_socket->so_cred->cr_prison == NULL) {
1517                                 return (inp);
1518                         } else {
1519                                 if  (jinp == NULL)
1520                                         jinp = inp;
1521                         }
1522                 }
1523         }
1524         if (jinp != NULL)
1525                 return (jinp);
1526
1527         if (wildcard) {
1528                 struct inpcb *local_wild = NULL;
1529                 struct inpcb *jinp_wild = NULL;
1530 #ifdef INET6
1531                 struct inpcb *local_wild_mapped = NULL;
1532 #endif
1533                 struct inpcontainer *ic;
1534                 struct inpcontainerhead *chead;
1535                 struct sockaddr_in jsin;
1536                 struct ucred *cred;
1537
1538                 GET_PCBINFO_TOKEN(pcbinfo);
1539
1540                 /*
1541                  * Check local group first
1542                  */
1543                 if (pcbinfo->localgrphashbase != NULL &&
1544                     m != NULL && (m->m_flags & M_HASH) &&
1545                     !(ifp && ifp->if_type == IFT_FAITH)) {
1546                         inp = inp_localgroup_lookup(pcbinfo,
1547                             laddr, lport, m->m_pkthdr.hash);
1548                         if (inp != NULL) {
1549                                 REL_PCBINFO_TOKEN(pcbinfo);
1550                                 return inp;
1551                         }
1552                 }
1553
1554                 /*
1555                  * Order of socket selection:
1556                  * 1. non-jailed, non-wild.
1557                  * 2. non-jailed, wild.
1558                  * 3. jailed, non-wild.
1559                  * 4. jailed, wild.
1560                  */
1561                 jsin.sin_family = AF_INET;
1562                 chead = &pcbinfo->wildcardhashbase[
1563                     INP_PCBWILDCARDHASH(lport, pcbinfo->wildcardhashmask)];
1564                 LIST_FOREACH(ic, chead, ic_list) {
1565                         inp = ic->ic_inp;
1566                         if (inp->inp_flags & INP_PLACEMARKER)
1567                                 continue;
1568
1569                         jsin.sin_addr.s_addr = laddr.s_addr;
1570 #ifdef INET6
1571                         if (!(inp->inp_vflag & INP_IPV4))
1572                                 continue;
1573 #endif
1574                         if (inp->inp_socket != NULL)
1575                                 cred = inp->inp_socket->so_cred;
1576                         else
1577                                 cred = NULL;
1578                         if (cred != NULL && jailed(cred)) {
1579                                 if (jinp != NULL)
1580                                         continue;
1581                                 else
1582                                         if (!jailed_ip(cred->cr_prison,
1583                                             (struct sockaddr *)&jsin))
1584                                                 continue;
1585                         }
1586                         if (inp->inp_lport == lport) {
1587                                 if (ifp && ifp->if_type == IFT_FAITH &&
1588                                     !(inp->inp_flags & INP_FAITH))
1589                                         continue;
1590                                 if (inp->inp_laddr.s_addr == laddr.s_addr) {
1591                                         if (cred != NULL && jailed(cred)) {
1592                                                 jinp = inp;
1593                                         } else {
1594                                                 REL_PCBINFO_TOKEN(pcbinfo);
1595                                                 return (inp);
1596                                         }
1597                                 }
1598                                 if (inp->inp_laddr.s_addr == INADDR_ANY) {
1599 #ifdef INET6
1600                                         if (INP_CHECK_SOCKAF(inp->inp_socket,
1601                                                              AF_INET6))
1602                                                 local_wild_mapped = inp;
1603                                         else
1604 #endif
1605                                                 if (cred != NULL &&
1606                                                     jailed(cred))
1607                                                         jinp_wild = inp;
1608                                                 else
1609                                                         local_wild = inp;
1610                                 }
1611                         }
1612                 }
1613
1614                 REL_PCBINFO_TOKEN(pcbinfo);
1615
1616                 if (local_wild != NULL)
1617                         return (local_wild);
1618 #ifdef INET6
1619                 if (local_wild_mapped != NULL)
1620                         return (local_wild_mapped);
1621 #endif
1622                 if (jinp != NULL)
1623                         return (jinp);
1624                 return (jinp_wild);
1625         }
1626
1627         /*
1628          * Not found.
1629          */
1630         return (NULL);
1631 }
1632
1633 struct inpcb *
1634 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1635     u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1636     boolean_t wildcard, struct ifnet *ifp)
1637 {
1638         return in_pcblookup_pkthash(pcbinfo, faddr, fport_arg,
1639             laddr, lport_arg, wildcard, ifp, NULL);
1640 }
1641
1642 /*
1643  * Insert PCB into connection hash table.
1644  */
1645 void
1646 in_pcbinsconnhash(struct inpcb *inp)
1647 {
1648         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1649         struct inpcbhead *bucket;
1650         u_int32_t hashkey_faddr, hashkey_laddr;
1651
1652 #ifdef INET6
1653         if (inp->inp_vflag & INP_IPV6) {
1654                 hashkey_faddr = inp->in6p_faddr.s6_addr32[3] /* XXX JH */;
1655                 hashkey_laddr = inp->in6p_laddr.s6_addr32[3] /* XXX JH */;
1656         } else {
1657 #endif
1658                 hashkey_faddr = inp->inp_faddr.s_addr;
1659                 hashkey_laddr = inp->inp_laddr.s_addr;
1660 #ifdef INET6
1661         }
1662 #endif
1663
1664         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1665             ("not in the correct netisr"));
1666         ASSERT_INP_NOTINHASH(inp);
1667         inp->inp_flags |= INP_CONNECTED;
1668
1669         /*
1670          * Insert into the connection hash table.
1671          */
1672         bucket = &pcbinfo->hashbase[INP_PCBCONNHASH(hashkey_faddr,
1673             inp->inp_fport, hashkey_laddr, inp->inp_lport, pcbinfo->hashmask)];
1674         LIST_INSERT_HEAD(bucket, inp, inp_hash);
1675 }
1676
1677 /*
1678  * Remove PCB from connection hash table.
1679  */
1680 void
1681 in_pcbremconnhash(struct inpcb *inp)
1682 {
1683         struct inpcbinfo *pcbinfo __debugvar = inp->inp_pcbinfo;
1684
1685         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1686             ("not in the correct netisr"));
1687         KASSERT(inp->inp_flags & INP_CONNECTED, ("inp not connected"));
1688
1689         LIST_REMOVE(inp, inp_hash);
1690         inp->inp_flags &= ~INP_CONNECTED;
1691 }
1692
1693 /*
1694  * Insert PCB into port hash table.
1695  */
1696 void
1697 in_pcbinsporthash(struct inpcbportinfo *portinfo, struct inpcb *inp)
1698 {
1699         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1700         struct inpcbporthead *pcbporthash;
1701         struct inpcbport *phd;
1702
1703         /*
1704          * If the porthashbase is shared across several cpus, it must
1705          * have been locked.
1706          */
1707         ASSERT_PORT_TOKEN_HELD(portinfo);
1708
1709         /*
1710          * Insert into the port hash table.
1711          */
1712         pcbporthash = &portinfo->porthashbase[
1713             INP_PCBPORTHASH(inp->inp_lport, portinfo->porthashmask)];
1714
1715         /* Go through port list and look for a head for this lport. */
1716         LIST_FOREACH(phd, pcbporthash, phd_hash) {
1717                 if (phd->phd_port == inp->inp_lport)
1718                         break;
1719         }
1720
1721         /* If none exists, use saved one and tack it on. */
1722         if (phd == NULL) {
1723                 KKASSERT(pcbinfo->portsave != NULL);
1724                 phd = pcbinfo->portsave;
1725                 pcbinfo->portsave = NULL;
1726                 phd->phd_port = inp->inp_lport;
1727                 LIST_INIT(&phd->phd_pcblist);
1728                 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
1729         }
1730
1731         inp->inp_portinfo = portinfo;
1732         inp->inp_phd = phd;
1733         LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
1734
1735         /*
1736          * Malloc one inpcbport for later use.  It is safe to use
1737          * "wait" malloc here (port token would be released, if
1738          * malloc ever blocked), since all changes to the porthash
1739          * are done.
1740          */
1741         if (pcbinfo->portsave == NULL) {
1742                 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave),
1743                                             M_PCB, M_INTWAIT | M_ZERO);
1744         }
1745 }
1746
1747 void
1748 in_pcbinsporthash_lport(struct inpcb *inp)
1749 {
1750         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1751         struct inpcbportinfo *portinfo;
1752         u_short lport_ho;
1753
1754         /* Locate the proper portinfo based on lport */
1755         lport_ho = ntohs(inp->inp_lport);
1756         portinfo = &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask];
1757         KKASSERT((lport_ho & pcbinfo->portinfo_mask) == portinfo->offset);
1758
1759         GET_PORT_TOKEN(portinfo);
1760         in_pcbinsporthash(portinfo, inp);
1761         REL_PORT_TOKEN(portinfo);
1762 }
1763
1764 static struct inp_localgroup *
1765 inp_localgroup_alloc(u_char vflag,
1766     uint16_t port, const union in_dependaddr *addr, int size)
1767 {
1768         struct inp_localgroup *grp;
1769
1770         grp = kmalloc(__offsetof(struct inp_localgroup, il_inp[size]),
1771             M_TEMP, M_INTWAIT | M_ZERO);
1772         grp->il_vflag = vflag;
1773         grp->il_lport = port;
1774         grp->il_dependladdr = *addr;
1775         grp->il_inpsiz = size;
1776
1777         return grp;
1778 }
1779
1780 static void
1781 inp_localgroup_free(struct inp_localgroup *grp)
1782 {
1783         kfree(grp, M_TEMP);
1784 }
1785
1786 static void
1787 inp_localgroup_destroy(struct inp_localgroup *grp)
1788 {
1789         LIST_REMOVE(grp, il_list);
1790         inp_localgroup_free(grp);
1791 }
1792
1793 static void
1794 inp_localgroup_copy(struct inp_localgroup *grp,
1795     const struct inp_localgroup *old_grp)
1796 {
1797         int i;
1798
1799         KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
1800             ("invalid new local group size %d and old local group count %d",
1801              grp->il_inpsiz, old_grp->il_inpcnt));
1802         for (i = 0; i < old_grp->il_inpcnt; ++i)
1803                 grp->il_inp[i] = old_grp->il_inp[i];
1804         grp->il_inpcnt = old_grp->il_inpcnt;
1805 }
1806
1807 static void
1808 in_pcbinslocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
1809 {
1810         struct inp_localgrphead *hdr;
1811         struct inp_localgroup *grp, *grp_alloc = NULL;
1812         struct ucred *cred;
1813         int i, idx;
1814
1815         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1816
1817         if (pcbinfo->localgrphashbase == NULL)
1818                 return;
1819
1820         /*
1821          * XXX don't allow jailed socket to join local group
1822          */
1823         if (inp->inp_socket != NULL)
1824                 cred = inp->inp_socket->so_cred;
1825         else
1826                 cred = NULL;
1827         if (cred != NULL && jailed(cred))
1828                 return;
1829
1830 #ifdef INET6
1831         /*
1832          * XXX don't allow IPv4 mapped INET6 wild socket
1833          */
1834         if ((inp->inp_vflag & INP_IPV4) &&
1835             inp->inp_laddr.s_addr == INADDR_ANY &&
1836             INP_CHECK_SOCKAF(inp->inp_socket, AF_INET6))
1837                 return;
1838 #endif
1839
1840         hdr = &pcbinfo->localgrphashbase[
1841             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
1842
1843 again:
1844         LIST_FOREACH(grp, hdr, il_list) {
1845                 if (grp->il_vflag == inp->inp_vflag &&
1846                     grp->il_lport == inp->inp_lport &&
1847                     memcmp(&grp->il_dependladdr,
1848                         &inp->inp_inc.inc_ie.ie_dependladdr,
1849                         sizeof(grp->il_dependladdr)) == 0) {
1850                         break;
1851                 }
1852         }
1853         if (grp == NULL) {
1854                 /*
1855                  * Create a new local group
1856                  */
1857                 if (grp_alloc == NULL) {
1858                         grp_alloc = inp_localgroup_alloc(inp->inp_vflag,
1859                             inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
1860                             INP_LOCALGROUP_SIZMIN);
1861                         /*
1862                          * Local group allocation could block and the
1863                          * local group w/ the same property might have
1864                          * been added by others when we were blocked;
1865                          * check again.
1866                          */
1867                         goto again;
1868                 } else {
1869                         /* Local group has been allocated; link it */
1870                         grp = grp_alloc;
1871                         grp_alloc = NULL;
1872                         LIST_INSERT_HEAD(hdr, grp, il_list);
1873                 }
1874         } else if (grp->il_inpcnt == grp->il_inpsiz) {
1875                 if (grp->il_inpsiz >= INP_LOCALGROUP_SIZMAX) {
1876                         static int limit_logged = 0;
1877
1878                         if (!limit_logged) {
1879                                 limit_logged = 1;
1880                                 kprintf("local group port %d, "
1881                                     "limit reached\n", ntohs(grp->il_lport));
1882                         }
1883                         if (grp_alloc != NULL) {
1884                                 /*
1885                                  * This would happen if the local group
1886                                  * w/ the same property was expanded when
1887                                  * our local group allocation blocked.
1888                                  */
1889                                 inp_localgroup_free(grp_alloc);
1890                         }
1891                         return;
1892                 }
1893
1894                 /*
1895                  * Expand this local group
1896                  */
1897                 if (grp_alloc == NULL ||
1898                     grp->il_inpcnt >= grp_alloc->il_inpsiz) {
1899                         if (grp_alloc != NULL)
1900                                 inp_localgroup_free(grp_alloc);
1901                         grp_alloc = inp_localgroup_alloc(grp->il_vflag,
1902                             grp->il_lport, &grp->il_dependladdr,
1903                             grp->il_inpsiz * 2);
1904                         /*
1905                          * Local group allocation could block and the
1906                          * local group w/ the same property might have
1907                          * been expanded by others when we were blocked;
1908                          * check again.
1909                          */
1910                         goto again;
1911                 }
1912
1913                 /*
1914                  * Save the old local group, link the new one, and then
1915                  * destroy the old local group
1916                  */
1917                 inp_localgroup_copy(grp_alloc, grp);
1918                 LIST_INSERT_HEAD(hdr, grp_alloc, il_list);
1919                 inp_localgroup_destroy(grp);
1920
1921                 grp = grp_alloc;
1922                 grp_alloc = NULL;
1923         } else {
1924                 /*
1925                  * Found the local group
1926                  */
1927                 if (grp_alloc != NULL) {
1928                         /*
1929                          * This would happen if the local group w/ the
1930                          * same property was added or expanded when our
1931                          * local group allocation blocked.
1932                          */
1933                         inp_localgroup_free(grp_alloc);
1934                         grp_alloc = NULL;
1935                 }
1936         }
1937
1938         KASSERT(grp->il_inpcnt < grp->il_inpsiz,
1939             ("invalid local group size %d and count %d",
1940              grp->il_inpsiz, grp->il_inpcnt));
1941
1942         /*
1943          * Keep the local group sorted by the inpcb local group index
1944          * in ascending order.
1945          *
1946          * This eases the multi-process userland application which uses
1947          * SO_REUSEPORT sockets and binds process to the owner cpu of
1948          * the SO_REUSEPORT socket:
1949          * If we didn't sort the local group by the inpcb local group
1950          * index and one of the process owning an inpcb in this local
1951          * group restarted, e.g. crashed and restarted by watchdog,
1952          * other processes owning a inpcb in this local group would have
1953          * to detect that event, refetch its socket's owner cpu, and
1954          * re-bind.
1955          */
1956         idx = grp->il_inpcnt;
1957         for (i = 0; i < idx; ++i) {
1958                 struct inpcb *oinp = grp->il_inp[i];
1959
1960                 if (oinp->inp_lgrpindex > i) {
1961                         if (inp->inp_lgrpindex < 0) {
1962                                 inp->inp_lgrpindex = i;
1963                         } else if (inp->inp_lgrpindex != i) {
1964                                 if (bootverbose) {
1965                                         kprintf("inp %p: grpidx %d, "
1966                                             "assigned to %d, cpu%d\n",
1967                                             inp, inp->inp_lgrpindex, i,
1968                                             mycpuid);
1969                                 }
1970                         }
1971                         grp->il_inp[i] = inp;
1972
1973                         /* Pull down inpcbs */
1974                         for (; i < grp->il_inpcnt; ++i) {
1975                                 struct inpcb *oinp1 = grp->il_inp[i + 1];
1976
1977                                 grp->il_inp[i + 1] = oinp;
1978                                 oinp = oinp1;
1979                         }
1980                         grp->il_inpcnt++;
1981                         return;
1982                 }
1983         }
1984
1985         if (inp->inp_lgrpindex < 0) {
1986                 inp->inp_lgrpindex = idx;
1987         } else if (inp->inp_lgrpindex != idx) {
1988                 if (bootverbose) {
1989                         kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n",
1990                             inp, inp->inp_lgrpindex, idx, mycpuid);
1991                 }
1992         }
1993         grp->il_inp[idx] = inp;
1994         grp->il_inpcnt++;
1995 }
1996
1997 void
1998 in_pcbinswildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
1999 {
2000         struct inpcontainer *ic;
2001         struct inpcontainerhead *bucket;
2002
2003         GET_PCBINFO_TOKEN(pcbinfo);
2004
2005         in_pcbinslocalgrphash_oncpu(inp, pcbinfo);
2006
2007         bucket = &pcbinfo->wildcardhashbase[
2008             INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
2009
2010         ic = kmalloc(sizeof(struct inpcontainer), M_TEMP, M_INTWAIT);
2011         ic->ic_inp = inp;
2012         LIST_INSERT_HEAD(bucket, ic, ic_list);
2013
2014         REL_PCBINFO_TOKEN(pcbinfo);
2015 }
2016
2017 /*
2018  * Insert PCB into wildcard hash table.
2019  */
2020 void
2021 in_pcbinswildcardhash(struct inpcb *inp)
2022 {
2023         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2024
2025         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
2026             ("not in correct netisr"));
2027         ASSERT_INP_NOTINHASH(inp);
2028         inp->inp_flags |= INP_WILDCARD;
2029
2030         in_pcbinswildcardhash_oncpu(inp, pcbinfo);
2031 }
2032
2033 static void
2034 in_pcbremlocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2035 {
2036         struct inp_localgrphead *hdr;
2037         struct inp_localgroup *grp;
2038
2039         ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
2040
2041         if (pcbinfo->localgrphashbase == NULL)
2042                 return;
2043
2044         hdr = &pcbinfo->localgrphashbase[
2045             INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
2046
2047         LIST_FOREACH(grp, hdr, il_list) {
2048                 int i;
2049
2050                 for (i = 0; i < grp->il_inpcnt; ++i) {
2051                         if (grp->il_inp[i] != inp)
2052                                 continue;
2053
2054                         if (grp->il_inpcnt == 1) {
2055                                 /* Destroy this local group */
2056                                 inp_localgroup_destroy(grp);
2057                         } else {
2058                                 /* Pull up inpcbs */
2059                                 for (; i + 1 < grp->il_inpcnt; ++i)
2060                                         grp->il_inp[i] = grp->il_inp[i + 1];
2061                                 grp->il_inpcnt--;
2062                         }
2063                         return;
2064                 }
2065         }
2066 }
2067
2068 void
2069 in_pcbremwildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2070 {
2071         struct inpcontainer *ic;
2072         struct inpcontainerhead *head;
2073
2074         GET_PCBINFO_TOKEN(pcbinfo);
2075
2076         in_pcbremlocalgrphash_oncpu(inp, pcbinfo);
2077
2078         /* find bucket */
2079         head = &pcbinfo->wildcardhashbase[
2080             INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
2081
2082         LIST_FOREACH(ic, head, ic_list) {
2083                 if (ic->ic_inp == inp)
2084                         goto found;
2085         }
2086         REL_PCBINFO_TOKEN(pcbinfo);
2087         return;                 /* not found! */
2088
2089 found:
2090         LIST_REMOVE(ic, ic_list);       /* remove container from bucket chain */
2091         REL_PCBINFO_TOKEN(pcbinfo);
2092         kfree(ic, M_TEMP);              /* deallocate container */
2093 }
2094
2095 /*
2096  * Remove PCB from wildcard hash table.
2097  */
2098 void
2099 in_pcbremwildcardhash(struct inpcb *inp)
2100 {
2101         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2102
2103         KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
2104             ("not in correct netisr"));
2105         KASSERT(inp->inp_flags & INP_WILDCARD, ("inp not wildcard"));
2106
2107         in_pcbremwildcardhash_oncpu(inp, pcbinfo);
2108         inp->inp_lgrpindex = -1;
2109         inp->inp_flags &= ~INP_WILDCARD;
2110 }
2111
2112 /*
2113  * Remove PCB from various lists.
2114  */
2115 void
2116 in_pcbremlists(struct inpcb *inp)
2117 {
2118         if (inp->inp_lport) {
2119                 struct inpcbportinfo *portinfo;
2120                 struct inpcbport *phd;
2121
2122                 /*
2123                  * NOTE:
2124                  * inp->inp_portinfo is _not_ necessary same as
2125                  * inp->inp_pcbinfo->portinfo.
2126                  */
2127                 portinfo = inp->inp_portinfo;
2128                 GET_PORT_TOKEN(portinfo);
2129
2130                 phd = inp->inp_phd;
2131                 LIST_REMOVE(inp, inp_portlist);
2132                 if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
2133                         LIST_REMOVE(phd, phd_hash);
2134                         kfree(phd, M_PCB);
2135                 }
2136
2137                 REL_PORT_TOKEN(portinfo);
2138         }
2139         if (inp->inp_flags & INP_WILDCARD) {
2140                 in_pcbremwildcardhash(inp);
2141         } else if (inp->inp_flags & INP_CONNECTED) {
2142                 in_pcbremconnhash(inp);
2143         }
2144
2145         if (inp->inp_flags & INP_ONLIST)
2146                 in_pcbofflist(inp);
2147 }
2148
2149 int
2150 prison_xinpcb(struct thread *td, struct inpcb *inp)
2151 {
2152         struct ucred *cr;
2153
2154         if (td->td_proc == NULL)
2155                 return (0);
2156         cr = td->td_proc->p_ucred;
2157         if (cr->cr_prison == NULL)
2158                 return (0);
2159         if (inp->inp_socket && inp->inp_socket->so_cred &&
2160             inp->inp_socket->so_cred->cr_prison &&
2161             cr->cr_prison == inp->inp_socket->so_cred->cr_prison)
2162                 return (0);
2163         return (1);
2164 }
2165
2166 int
2167 in_pcblist_global(SYSCTL_HANDLER_ARGS)
2168 {
2169         struct inpcbinfo *pcbinfo_arr = arg1;
2170         int pcbinfo_arrlen = arg2;
2171         struct inpcb *marker;
2172         int cpu, origcpu;
2173         int error, n;
2174
2175         KASSERT(pcbinfo_arrlen <= ncpus && pcbinfo_arrlen >= 1,
2176             ("invalid pcbinfo count %d", pcbinfo_arrlen));
2177
2178         /*
2179          * The process of preparing the TCB list is too time-consuming and
2180          * resource-intensive to repeat twice on every request.
2181          */
2182         n = 0;
2183         if (req->oldptr == NULL) {
2184                 for (cpu = 0; cpu < pcbinfo_arrlen; ++cpu)
2185                         n += pcbinfo_arr[cpu].ipi_count;
2186                 req->oldidx = (n + n/8 + 10) * sizeof(struct xinpcb);
2187                 return 0;
2188         }
2189
2190         if (req->newptr != NULL)
2191                 return EPERM;
2192
2193         marker = kmalloc(sizeof(struct inpcb), M_TEMP, M_WAITOK|M_ZERO);
2194         marker->inp_flags |= INP_PLACEMARKER;
2195
2196         /*
2197          * OK, now we're committed to doing something.  Re-fetch ipi_count
2198          * after obtaining the generation count.
2199          */
2200         error = 0;
2201         origcpu = mycpuid;
2202         for (cpu = 0; cpu < pcbinfo_arrlen && error == 0; ++cpu) {
2203                 struct inpcbinfo *pcbinfo = &pcbinfo_arr[cpu];
2204                 struct inpcb *inp;
2205                 struct xinpcb xi;
2206                 int i;
2207
2208                 lwkt_migratecpu(cpu);
2209
2210                 GET_PCBINFO_TOKEN(pcbinfo);
2211
2212                 n = pcbinfo->ipi_count;
2213
2214                 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
2215                 i = 0;
2216                 while ((inp = LIST_NEXT(marker, inp_list)) != NULL && i < n) {
2217                         LIST_REMOVE(marker, inp_list);
2218                         LIST_INSERT_AFTER(inp, marker, inp_list);
2219
2220                         if (inp->inp_flags & INP_PLACEMARKER)
2221                                 continue;
2222                         if (prison_xinpcb(req->td, inp))
2223                                 continue;
2224
2225                         bzero(&xi, sizeof xi);
2226                         xi.xi_len = sizeof xi;
2227                         bcopy(inp, &xi.xi_inp, sizeof *inp);
2228                         if (inp->inp_socket)
2229                                 sotoxsocket(inp->inp_socket, &xi.xi_socket);
2230                         if ((error = SYSCTL_OUT(req, &xi, sizeof xi)) != 0)
2231                                 break;
2232                         ++i;
2233                 }
2234                 LIST_REMOVE(marker, inp_list);
2235
2236                 REL_PCBINFO_TOKEN(pcbinfo);
2237
2238                 if (error == 0 && i < n) {
2239                         bzero(&xi, sizeof xi);
2240                         xi.xi_len = sizeof xi;
2241                         while (i < n) {
2242                                 error = SYSCTL_OUT(req, &xi, sizeof xi);
2243                                 if (error)
2244                                         break;
2245                                 ++i;
2246                         }
2247                 }
2248         }
2249
2250         lwkt_migratecpu(origcpu);
2251         kfree(marker, M_TEMP);
2252         return error;
2253 }
2254
2255 int
2256 in_pcblist_global_ncpus2(SYSCTL_HANDLER_ARGS)
2257 {
2258         return in_pcblist_global(oidp, arg1, ncpus2, req);
2259 }
2260
2261 void
2262 in_savefaddr(struct socket *so, const struct sockaddr *faddr)
2263 {
2264         struct sockaddr_in *sin;
2265
2266         KASSERT(faddr->sa_family == AF_INET,
2267             ("not AF_INET faddr %d", faddr->sa_family));
2268
2269         sin = kmalloc(sizeof(*sin), M_SONAME, M_WAITOK | M_ZERO);
2270         sin->sin_family = AF_INET;
2271         sin->sin_len = sizeof(*sin);
2272         sin->sin_port = ((const struct sockaddr_in *)faddr)->sin_port;
2273         sin->sin_addr = ((const struct sockaddr_in *)faddr)->sin_addr;
2274
2275         so->so_faddr = (struct sockaddr *)sin;
2276 }
2277
2278 void
2279 in_pcbportinfo_init(struct inpcbportinfo *portinfo, int hashsize,
2280     boolean_t shared, u_short offset)
2281 {
2282         memset(portinfo, 0, sizeof(*portinfo));
2283
2284         portinfo->offset = offset;
2285         portinfo->lastport = offset;
2286         portinfo->lastlow = offset;
2287         portinfo->lasthi = offset;
2288
2289         portinfo->porthashbase = hashinit(hashsize, M_PCB,
2290             &portinfo->porthashmask);
2291
2292         if (shared) {
2293                 portinfo->porttoken = kmalloc(sizeof(struct lwkt_token),
2294                     M_PCB, M_WAITOK);
2295                 lwkt_token_init(portinfo->porttoken, "porttoken");
2296         }
2297 }
2298
2299 void
2300 in_pcbportrange(u_short *hi0, u_short *lo0, u_short ofs, u_short step)
2301 {
2302         int hi, lo;
2303
2304         if (step == 1)
2305                 return;
2306
2307         hi = *hi0;
2308         lo = *lo0;
2309
2310         hi = rounddown2(hi, step);
2311         hi += ofs;
2312         if (hi > (int)*hi0)
2313                 hi -= step;
2314
2315         lo = roundup2(lo, step);
2316         lo -= (step - ofs);
2317         if (lo < (int)*lo0)
2318                 lo += step;
2319
2320         *hi0 = hi;
2321         *lo0 = lo;
2322 }
2323
2324 void
2325 in_pcbglobalinit(void)
2326 {
2327         int cpu;
2328
2329         in_pcbmarkers = kmalloc(ncpus * sizeof(struct inpcb), M_PCB,
2330             M_WAITOK | M_ZERO);
2331         in_pcbcontainer_markers = kmalloc(ncpus * sizeof(struct inpcontainer),
2332             M_PCB, M_WAITOK | M_ZERO);
2333
2334         for (cpu = 0; cpu < ncpus; ++cpu) {
2335                 struct inpcontainer *ic = &in_pcbcontainer_markers[cpu];
2336                 struct inpcb *marker = &in_pcbmarkers[cpu];
2337
2338                 marker->inp_flags |= INP_PLACEMARKER;
2339                 ic->ic_inp = marker;
2340         }
2341 }
2342
2343 struct inpcb *
2344 in_pcbmarker(int cpuid)
2345 {
2346         KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid));
2347         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
2348
2349         return &in_pcbmarkers[cpuid];
2350 }
2351
2352 struct inpcontainer *
2353 in_pcbcontainer_marker(int cpuid)
2354 {
2355         KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid));
2356         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
2357
2358         return &in_pcbcontainer_markers[cpuid];
2359 }
2360
2361 void
2362 in_pcbresetroute(struct inpcb *inp)
2363 {
2364         struct route *ro = &inp->inp_route;
2365
2366         if (ro->ro_rt != NULL)
2367                 RTFREE(ro->ro_rt);
2368         bzero(ro, sizeof(*ro));
2369 }