sys/netinet/ip_input.c

   1 /*
   2  * Copyright (c) 2003, 2004 Jeffrey M. Hsu.  All rights reserved.
   3  * Copyright (c) 2003, 2004 The DragonFly Project.  All rights reserved.
   4  *
   5  * This code is derived from software contributed to The DragonFly Project
   6  * by Jeffrey M. Hsu.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the name of The DragonFly Project nor the names of its
  17  *    contributors may be used to endorse or promote products derived
  18  *    from this software without specific, prior written permission.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  23  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  24  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  25  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  26  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  27  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  28  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  29  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  30  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  */
  33
  34 /*
  35  * Copyright (c) 1982, 1986, 1988, 1993
  36  *      The Regents of the University of California.  All rights reserved.
  37  *
  38  * Redistribution and use in source and binary forms, with or without
  39  * modification, are permitted provided that the following conditions
  40  * are met:
  41  * 1. Redistributions of source code must retain the above copyright
  42  *    notice, this list of conditions and the following disclaimer.
  43  * 2. Redistributions in binary form must reproduce the above copyright
  44  *    notice, this list of conditions and the following disclaimer in the
  45  *    documentation and/or other materials provided with the distribution.
  46  * 3. Neither the name of the University nor the names of its contributors
  47  *    may be used to endorse or promote products derived from this software
  48  *    without specific prior written permission.
  49  *
  50  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  51  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  52  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  53  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  54  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  55  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  56  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  57  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  58  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  59  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  60  * SUCH DAMAGE.
  61  *
  62  *      @(#)ip_input.c  8.2 (Berkeley) 1/4/94
  63  * $FreeBSD: src/sys/netinet/ip_input.c,v 1.130.2.52 2003/03/07 07:01:28 silby Exp $
  64  */
  65
  66 #define _IP_VHL
  67
  68 #include "opt_bootp.h"
  69 #include "opt_ipdn.h"
  70 #include "opt_ipdivert.h"
  71 #include "opt_ipstealth.h"
  72 #include "opt_ipsec.h"
  73 #include "opt_rss.h"
  74
  75 #include <sys/param.h>
  76 #include <sys/systm.h>
  77 #include <sys/mbuf.h>
  78 #include <sys/malloc.h>
  79 #include <sys/mpipe.h>
  80 #include <sys/domain.h>
  81 #include <sys/protosw.h>
  82 #include <sys/socket.h>
  83 #include <sys/time.h>
  84 #include <sys/globaldata.h>
  85 #include <sys/thread.h>
  86 #include <sys/kernel.h>
  87 #include <sys/syslog.h>
  88 #include <sys/sysctl.h>
  89 #include <sys/in_cksum.h>
  90 #include <sys/lock.h>
  91
  92 #include <sys/mplock2.h>
  93
  94 #include <machine/stdarg.h>
  95
  96 #include <net/if.h>
  97 #include <net/if_types.h>
  98 #include <net/if_var.h>
  99 #include <net/if_dl.h>
 100 #include <net/pfil.h>
 101 #include <net/route.h>
 102 #include <net/netisr2.h>
 103
 104 #include <netinet/in.h>
 105 #include <netinet/in_systm.h>
 106 #include <netinet/in_var.h>
 107 #include <netinet/ip.h>
 108 #include <netinet/in_pcb.h>
 109 #include <netinet/ip_var.h>
 110 #include <netinet/ip_icmp.h>
 111 #include <netinet/ip_divert.h>
 112 #include <netinet/ip_flow.h>
 113
 114 #include <sys/thread2.h>
 115 #include <sys/msgport2.h>
 116 #include <net/netmsg2.h>
 117
 118 #include <sys/socketvar.h>
 119
 120 #include <net/ipfw/ip_fw.h>
 121 #include <net/dummynet/ip_dummynet.h>
 122
 123 #ifdef IPSEC
 124 #include <netinet6/ipsec.h>
 125 #include <netproto/key/key.h>
 126 #endif
 127
 128 #ifdef FAST_IPSEC
 129 #include <netproto/ipsec/ipsec.h>
 130 #include <netproto/ipsec/key.h>
 131 #endif
 132
 133 int rsvp_on = 0;
 134 static int ip_rsvp_on;
 135 struct socket *ip_rsvpd;
 136
 137 int ipforwarding = 0;
 138 SYSCTL_INT(_net_inet_ip, IPCTL_FORWARDING, forwarding, CTLFLAG_RW,
 139     &ipforwarding, 0, "Enable IP forwarding between interfaces");
 140
 141 static int ipsendredirects = 1; /* XXX */
 142 SYSCTL_INT(_net_inet_ip, IPCTL_SENDREDIRECTS, redirect, CTLFLAG_RW,
 143     &ipsendredirects, 0, "Enable sending IP redirects");
 144
 145 int ip_defttl = IPDEFTTL;
 146 SYSCTL_INT(_net_inet_ip, IPCTL_DEFTTL, ttl, CTLFLAG_RW,
 147     &ip_defttl, 0, "Maximum TTL on IP packets");
 148
 149 static int ip_dosourceroute = 0;
 150 SYSCTL_INT(_net_inet_ip, IPCTL_SOURCEROUTE, sourceroute, CTLFLAG_RW,
 151     &ip_dosourceroute, 0, "Enable forwarding source routed IP packets");
 152
 153 static int ip_acceptsourceroute = 0;
 154 SYSCTL_INT(_net_inet_ip, IPCTL_ACCEPTSOURCEROUTE, accept_sourceroute,
 155     CTLFLAG_RW, &ip_acceptsourceroute, 0,
 156     "Enable accepting source routed IP packets");
 157
 158 static int ip_keepfaith = 0;
 159 SYSCTL_INT(_net_inet_ip, IPCTL_KEEPFAITH, keepfaith, CTLFLAG_RW,
 160     &ip_keepfaith, 0,
 161     "Enable packet capture for FAITH IPv4->IPv6 translator daemon");
 162
 163 static int maxnipq;
 164 SYSCTL_INT(_net_inet_ip, OID_AUTO, maxfragpackets, CTLFLAG_RW,
 165     &maxnipq, 0,
 166     "Maximum number of IPv4 fragment reassembly queue entries");
 167
 168 static int maxfragsperpacket;
 169 SYSCTL_INT(_net_inet_ip, OID_AUTO, maxfragsperpacket, CTLFLAG_RW,
 170     &maxfragsperpacket, 0,
 171     "Maximum number of IPv4 fragments allowed per packet");
 172
 173 static int ip_sendsourcequench = 0;
 174 SYSCTL_INT(_net_inet_ip, OID_AUTO, sendsourcequench, CTLFLAG_RW,
 175     &ip_sendsourcequench, 0,
 176     "Enable the transmission of source quench packets");
 177
 178 int ip_do_randomid = 1;
 179 SYSCTL_INT(_net_inet_ip, OID_AUTO, random_id, CTLFLAG_RW,
 180     &ip_do_randomid, 0,
 181     "Assign random ip_id values");
 182 /*
 183  * XXX - Setting ip_checkinterface mostly implements the receive side of
 184  * the Strong ES model described in RFC 1122, but since the routing table
 185  * and transmit implementation do not implement the Strong ES model,
 186  * setting this to 1 results in an odd hybrid.
 187  *
 188  * XXX - ip_checkinterface currently must be disabled if you use ipnat
 189  * to translate the destination address to another local interface.
 190  *
 191  * XXX - ip_checkinterface must be disabled if you add IP aliases
 192  * to the loopback interface instead of the interface where the
 193  * packets for those addresses are received.
 194  */
 195 static int ip_checkinterface = 0;
 196 SYSCTL_INT(_net_inet_ip, OID_AUTO, check_interface, CTLFLAG_RW,
 197     &ip_checkinterface, 0, "Verify packet arrives on correct interface");
 198
 199 static u_long ip_hash_count = 0;
 200 SYSCTL_ULONG(_net_inet_ip, OID_AUTO, hash_count, CTLFLAG_RD,
 201     &ip_hash_count, 0, "Number of packets hashed by IP");
 202
 203 #ifdef RSS_DEBUG
 204 static u_long ip_rehash_count = 0;
 205 SYSCTL_ULONG(_net_inet_ip, OID_AUTO, rehash_count, CTLFLAG_RD,
 206     &ip_rehash_count, 0, "Number of packets rehashed by IP");
 207
 208 static u_long ip_dispatch_fast = 0;
 209 SYSCTL_ULONG(_net_inet_ip, OID_AUTO, dispatch_fast_count, CTLFLAG_RD,
 210     &ip_dispatch_fast, 0, "Number of packets handled on current CPU");
 211
 212 static u_long ip_dispatch_slow = 0;
 213 SYSCTL_ULONG(_net_inet_ip, OID_AUTO, dispatch_slow_count, CTLFLAG_RD,
 214     &ip_dispatch_slow, 0, "Number of packets messaged to another CPU");
 215 #endif
 216
 217 #ifdef DIAGNOSTIC
 218 static int ipprintfs = 0;
 219 #endif
 220
 221 extern  struct domain inetdomain;
 222 extern  struct protosw inetsw[];
 223 u_char  ip_protox[IPPROTO_MAX];
 224 struct  in_ifaddrhead in_ifaddrheads[MAXCPU];   /* first inet address */
 225 struct  in_ifaddrhashhead *in_ifaddrhashtbls[MAXCPU];
 226                                                 /* inet addr hash table */
 227 u_long  in_ifaddrhmask;                         /* mask for hash table */
 228
 229 static struct mbuf *ipforward_mtemp[MAXCPU];
 230
 231 struct ip_stats ipstats_percpu[MAXCPU] __cachealign;
 232
 233 static int
 234 sysctl_ipstats(SYSCTL_HANDLER_ARGS)
 235 {
 236         int cpu, error = 0;
 237
 238         for (cpu = 0; cpu < ncpus; ++cpu) {
 239                 if ((error = SYSCTL_OUT(req, &ipstats_percpu[cpu],
 240                                         sizeof(struct ip_stats))))
 241                         break;
 242                 if ((error = SYSCTL_IN(req, &ipstats_percpu[cpu],
 243                                        sizeof(struct ip_stats))))
 244                         break;
 245         }
 246
 247         return (error);
 248 }
 249 SYSCTL_PROC(_net_inet_ip, IPCTL_STATS, stats, (CTLTYPE_OPAQUE | CTLFLAG_RW),
 250     0, 0, sysctl_ipstats, "S,ip_stats", "IP statistics");
 251
 252 /* Packet reassembly stuff */
 253 #define IPREASS_NHASH_LOG2      6
 254 #define IPREASS_NHASH           (1 << IPREASS_NHASH_LOG2)
 255 #define IPREASS_HMASK           (IPREASS_NHASH - 1)
 256 #define IPREASS_HASH(x,y)                                               \
 257     (((((x) & 0xF) | ((((x) >> 8) & 0xF) << 4)) ^ (y)) & IPREASS_HMASK)
 258
 259 TAILQ_HEAD(ipqhead, ipq);
 260 struct ipfrag_queue {
 261         int                     nipq;
 262         struct netmsg_base      timeo_netmsg;
 263         struct netmsg_base      drain_netmsg;
 264         struct ipqhead          ipq[IPREASS_NHASH];
 265 } __cachealign;
 266
 267 static struct ipfrag_queue      ipfrag_queue_pcpu[MAXCPU];
 268
 269 #ifdef IPCTL_DEFMTU
 270 SYSCTL_INT(_net_inet_ip, IPCTL_DEFMTU, mtu, CTLFLAG_RW,
 271     &ip_mtu, 0, "Default MTU");
 272 #endif
 273
 274 #ifdef IPSTEALTH
 275 static int ipstealth = 0;
 276 SYSCTL_INT(_net_inet_ip, OID_AUTO, stealth, CTLFLAG_RW, &ipstealth, 0, "");
 277 #else
 278 static const int ipstealth = 0;
 279 #endif
 280
 281 struct mbuf *(*ip_divert_p)(struct mbuf *, int, int);
 282
 283 struct pfil_head inet_pfil_hook;
 284
 285 /*
 286  * struct ip_srcrt_opt is used to store packet state while it travels
 287  * through the stack.
 288  *
 289  * XXX Note that the code even makes assumptions on the size and
 290  * alignment of fields inside struct ip_srcrt so e.g. adding some
 291  * fields will break the code.  This needs to be fixed.
 292  *
 293  * We need to save the IP options in case a protocol wants to respond
 294  * to an incoming packet over the same route if the packet got here
 295  * using IP source routing.  This allows connection establishment and
 296  * maintenance when the remote end is on a network that is not known
 297  * to us.
 298  */
 299 struct ip_srcrt {
 300         struct  in_addr dst;                    /* final destination */
 301         char    nop;                            /* one NOP to align */
 302         char    srcopt[IPOPT_OFFSET + 1];       /* OPTVAL, OLEN and OFFSET */
 303         struct  in_addr route[MAX_IPOPTLEN/sizeof(struct in_addr)];
 304 };
 305
 306 struct ip_srcrt_opt {
 307         int             ip_nhops;
 308         struct ip_srcrt ip_srcrt;
 309 };
 310
 311 #define IPFRAG_MPIPE_MAX        4096
 312 #define MAXIPFRAG_MIN           ((IPFRAG_MPIPE_MAX * 2) / 256)
 313
 314 static MALLOC_DEFINE(M_IPQ, "ipq", "IP Fragment Management");
 315 static struct malloc_pipe ipq_mpipe;
 316
 317 static void             save_rte(struct mbuf *, u_char *, struct in_addr);
 318 static int              ip_dooptions(struct mbuf *m, int, struct sockaddr_in *);
 319 static void             ip_freef(struct ipfrag_queue *, struct ipqhead *,
 320                             struct ipq *);
 321 static void             ip_input_handler(netmsg_t);
 322
 323 static void             ipfrag_timeo_dispatch(netmsg_t);
 324 static void             ipfrag_drain_dispatch(netmsg_t);
 325
 326 /*
 327  * IP initialization: fill in IP protocol switch table.
 328  * All protocols not implemented in kernel go to raw IP protocol handler.
 329  */
 330 void
 331 ip_init(void)
 332 {
 333         struct protosw *pr;
 334         int cpu, i;
 335
 336         /*
 337          * Make sure we can handle a reasonable number of fragments but
 338          * cap it at IPFRAG_MPIPE_MAX.
 339          */
 340         mpipe_init(&ipq_mpipe, M_IPQ, sizeof(struct ipq),
 341             IFQ_MAXLEN, IPFRAG_MPIPE_MAX, 0, NULL, NULL, NULL);
 342         for (cpu = 0; cpu < ncpus; ++cpu) {
 343                 TAILQ_INIT(&in_ifaddrheads[cpu]);
 344                 in_ifaddrhashtbls[cpu] =
 345                     hashinit(INADDR_NHASH, M_IFADDR, &in_ifaddrhmask);
 346         }
 347         pr = pffindproto(PF_INET, IPPROTO_RAW, SOCK_RAW);
 348         if (pr == NULL)
 349                 panic("ip_init");
 350         for (i = 0; i < IPPROTO_MAX; i++)
 351                 ip_protox[i] = pr - inetsw;
 352         for (pr = inetdomain.dom_protosw;
 353              pr < inetdomain.dom_protoswNPROTOSW; pr++) {
 354                 if (pr->pr_domain->dom_family == PF_INET && pr->pr_protocol) {
 355                         if (pr->pr_protocol != IPPROTO_RAW)
 356                                 ip_protox[pr->pr_protocol] = pr - inetsw;
 357                 }
 358         }
 359
 360         inet_pfil_hook.ph_type = PFIL_TYPE_AF;
 361         inet_pfil_hook.ph_af = AF_INET;
 362         if ((i = pfil_head_register(&inet_pfil_hook)) != 0) {
 363                 kprintf("%s: WARNING: unable to register pfil hook, "
 364                         "error %d\n", __func__, i);
 365         }
 366
 367         maxnipq = (nmbclusters / 32) / ncpus2;
 368         if (maxnipq < MAXIPFRAG_MIN)
 369                 maxnipq = MAXIPFRAG_MIN;
 370         maxfragsperpacket = 16;
 371
 372         ip_id = time_second & 0xffff;   /* time_second survives reboots */
 373
 374         for (cpu = 0; cpu < ncpus; ++cpu) {
 375                 /*
 376                  * Initialize IP statistics counters for each CPU.
 377                  */
 378                 bzero(&ipstats_percpu[cpu], sizeof(struct ip_stats));
 379
 380                 /*
 381                  * Preallocate mbuf template for forwarding
 382                  */
 383                 MGETHDR(ipforward_mtemp[cpu], MB_WAIT, MT_DATA);
 384
 385                 /*
 386                  * Initialize per-cpu ip fragments queues
 387                  */
 388                 for (i = 0; i < IPREASS_NHASH; i++) {
 389                         struct ipfrag_queue *fragq = &ipfrag_queue_pcpu[cpu];
 390
 391                         TAILQ_INIT(&fragq->ipq[i]);
 392                         netmsg_init(&fragq->timeo_netmsg, NULL,
 393                             &netisr_adone_rport, MSGF_PRIORITY,
 394                             ipfrag_timeo_dispatch);
 395                         netmsg_init(&fragq->drain_netmsg, NULL,
 396                             &netisr_adone_rport, MSGF_PRIORITY,
 397                             ipfrag_drain_dispatch);
 398                 }
 399         }
 400
 401         netisr_register(NETISR_IP, ip_input_handler, ip_hashfn);
 402         netisr_register_hashcheck(NETISR_IP, ip_hashcheck);
 403 }
 404
 405 /* Do transport protocol processing. */
 406 static void
 407 transport_processing_oncpu(struct mbuf *m, int hlen, struct ip *ip)
 408 {
 409         const struct protosw *pr = &inetsw[ip_protox[ip->ip_p]];
 410
 411         /*
 412          * Switch out to protocol's input routine.
 413          */
 414         PR_GET_MPLOCK(pr);
 415         pr->pr_input(&m, &hlen, ip->ip_p);
 416         PR_REL_MPLOCK(pr);
 417 }
 418
 419 static void
 420 transport_processing_handler(netmsg_t msg)
 421 {
 422         struct netmsg_packet *pmsg = &msg->packet;
 423         struct ip *ip;
 424         int hlen;
 425
 426         ip = mtod(pmsg->nm_packet, struct ip *);
 427         hlen = pmsg->base.lmsg.u.ms_result;
 428
 429         transport_processing_oncpu(pmsg->nm_packet, hlen, ip);
 430         /* msg was embedded in the mbuf, do not reply! */
 431 }
 432
 433 static void
 434 ip_input_handler(netmsg_t msg)
 435 {
 436         ip_input(msg->packet.nm_packet);
 437         /* msg was embedded in the mbuf, do not reply! */
 438 }
 439
 440 /*
 441  * IP input routine.  Checksum and byte swap header.  If fragmented
 442  * try to reassemble.  Process options.  Pass to next level.
 443  */
 444 void
 445 ip_input(struct mbuf *m)
 446 {
 447         struct ip *ip;
 448         struct in_ifaddr *ia = NULL;
 449         struct in_ifaddr_container *iac;
 450         int hlen, checkif;
 451         u_short sum;
 452         struct in_addr pkt_dst;
 453         boolean_t using_srcrt = FALSE;          /* forward (by PFIL_HOOKS) */
 454         struct in_addr odst;                    /* original dst address(NAT) */
 455         struct m_tag *mtag;
 456         struct sockaddr_in *next_hop = NULL;
 457         lwkt_port_t port;
 458 #ifdef FAST_IPSEC
 459         struct tdb_ident *tdbi;
 460         struct secpolicy *sp;
 461         int error;
 462 #endif
 463
 464         M_ASSERTPKTHDR(m);
 465
 466         /*
 467          * This routine is called from numerous places which may not have
 468          * characterized the packet.
 469          */
 470         ip = mtod(m, struct ip *);
 471         if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr)) ||
 472             ntohs(ip->ip_off) & (IP_MF | IP_OFFMASK)) {
 473                 /*
 474                  * Force hash recalculation for fragments and multicast
 475                  * packets; hardware may not do it correctly.
 476                  * XXX add flag to indicate the hash is from hardware
 477                  */
 478                 m->m_flags &= ~M_HASH;
 479         }
 480         if ((m->m_flags & M_HASH) == 0) {
 481                 ip_hashfn(&m, 0);
 482                 if (m == NULL)
 483                         return;
 484                 KKASSERT(m->m_flags & M_HASH);
 485
 486                 if (&curthread->td_msgport !=
 487                     netisr_hashport(m->m_pkthdr.hash)) {
 488                         netisr_queue(NETISR_IP, m);
 489                         /* Requeued to other netisr msgport; done */
 490                         return;
 491                 }
 492
 493                 /* mbuf could have been changed */
 494                 ip = mtod(m, struct ip *);
 495         }
 496
 497         /*
 498          * Pull out certain tags
 499          */
 500         if (m->m_pkthdr.fw_flags & IPFORWARD_MBUF_TAGGED) {
 501                 /* Next hop */
 502                 mtag = m_tag_find(m, PACKET_TAG_IPFORWARD, NULL);
 503                 KKASSERT(mtag != NULL);
 504                 next_hop = m_tag_data(mtag);
 505         }
 506
 507         if (m->m_pkthdr.fw_flags & DUMMYNET_MBUF_TAGGED) {
 508                 /* dummynet already filtered us */
 509                 ip = mtod(m, struct ip *);
 510                 hlen = IP_VHL_HL(ip->ip_vhl) << 2;
 511                 goto iphack;
 512         }
 513
 514         ipstat.ips_total++;
 515
 516         /* length checks already done in ip_hashfn() */
 517         KASSERT(m->m_len >= sizeof(struct ip), ("IP header not in one mbuf"));
 518
 519         if (IP_VHL_V(ip->ip_vhl) != IPVERSION) {
 520                 ipstat.ips_badvers++;
 521                 goto bad;
 522         }
 523
 524         hlen = IP_VHL_HL(ip->ip_vhl) << 2;
 525         /* length checks already done in ip_hashfn() */
 526         KASSERT(hlen >= sizeof(struct ip), ("IP header len too small"));
 527         KASSERT(m->m_len >= hlen, ("complete IP header not in one mbuf"));
 528
 529         /* 127/8 must not appear on wire - RFC1122 */
 530         if ((ntohl(ip->ip_dst.s_addr) >> IN_CLASSA_NSHIFT) == IN_LOOPBACKNET ||
 531             (ntohl(ip->ip_src.s_addr) >> IN_CLASSA_NSHIFT) == IN_LOOPBACKNET) {
 532                 if (!(m->m_pkthdr.rcvif->if_flags & IFF_LOOPBACK)) {
 533                         ipstat.ips_badaddr++;
 534                         goto bad;
 535                 }
 536         }
 537
 538         if (m->m_pkthdr.csum_flags & CSUM_IP_CHECKED) {
 539                 sum = !(m->m_pkthdr.csum_flags & CSUM_IP_VALID);
 540         } else {
 541                 if (hlen == sizeof(struct ip))
 542                         sum = in_cksum_hdr(ip);
 543                 else
 544                         sum = in_cksum(m, hlen);
 545         }
 546         if (sum != 0) {
 547                 ipstat.ips_badsum++;
 548                 goto bad;
 549         }
 550
 551 #ifdef ALTQ
 552         if (altq_input != NULL && (*altq_input)(m, AF_INET) == 0) {
 553                 /* packet is dropped by traffic conditioner */
 554                 return;
 555         }
 556 #endif
 557         /*
 558          * Convert fields to host representation.
 559          */
 560         ip->ip_len = ntohs(ip->ip_len);
 561         ip->ip_off = ntohs(ip->ip_off);
 562
 563         /* length checks already done in ip_hashfn() */
 564         KASSERT(ip->ip_len >= hlen, ("total length less then header length"));
 565         KASSERT(m->m_pkthdr.len >= ip->ip_len, ("mbuf too short"));
 566
 567         /*
 568          * Trim mbufs if longer than the IP header would have us expect.
 569          */
 570         if (m->m_pkthdr.len > ip->ip_len) {
 571                 if (m->m_len == m->m_pkthdr.len) {
 572                         m->m_len = ip->ip_len;
 573                         m->m_pkthdr.len = ip->ip_len;
 574                 } else {
 575                         m_adj(m, ip->ip_len - m->m_pkthdr.len);
 576                 }
 577         }
 578 #if defined(IPSEC) && !defined(IPSEC_FILTERGIF)
 579         /*
 580          * Bypass packet filtering for packets from a tunnel (gif).
 581          */
 582         if (ipsec_gethist(m, NULL))
 583                 goto pass;
 584 #endif
 585
 586         /*
 587          * IpHack's section.
 588          * Right now when no processing on packet has done
 589          * and it is still fresh out of network we do our black
 590          * deals with it.
 591          * - Firewall: deny/allow/divert
 592          * - Xlate: translate packet's addr/port (NAT).
 593          * - Pipe: pass pkt through dummynet.
 594          * - Wrap: fake packet's addr/port <unimpl.>
 595          * - Encapsulate: put it in another IP and send out. <unimp.>
 596          */
 597
 598 iphack:
 599         /*
 600          * If we've been forwarded from the output side, then
 601          * skip the firewall a second time
 602          */
 603         if (next_hop != NULL)
 604                 goto ours;
 605
 606         /* No pfil hooks */
 607         if (!pfil_has_hooks(&inet_pfil_hook)) {
 608                 if (m->m_pkthdr.fw_flags & DUMMYNET_MBUF_TAGGED) {
 609                         /*
 610                          * Strip dummynet tags from stranded packets
 611                          */
 612                         mtag = m_tag_find(m, PACKET_TAG_DUMMYNET, NULL);
 613                         KKASSERT(mtag != NULL);
 614                         m_tag_delete(m, mtag);
 615                         m->m_pkthdr.fw_flags &= ~DUMMYNET_MBUF_TAGGED;
 616                 }
 617                 goto pass;
 618         }
 619
 620         /*
 621          * Run through list of hooks for input packets.
 622          *
 623          * NOTE!  If the packet is rewritten pf/ipfw/whoever must
 624          *        clear M_HASH.
 625          */
 626         odst = ip->ip_dst;
 627         if (pfil_run_hooks(&inet_pfil_hook, &m, m->m_pkthdr.rcvif, PFIL_IN))
 628                 return;
 629         if (m == NULL)  /* consumed by filter */
 630                 return;
 631         ip = mtod(m, struct ip *);
 632         hlen = IP_VHL_HL(ip->ip_vhl) << 2;
 633         using_srcrt = (odst.s_addr != ip->ip_dst.s_addr);
 634
 635         if (m->m_pkthdr.fw_flags & IPFORWARD_MBUF_TAGGED) {
 636                 mtag = m_tag_find(m, PACKET_TAG_IPFORWARD, NULL);
 637                 KKASSERT(mtag != NULL);
 638                 next_hop = m_tag_data(mtag);
 639         }
 640         if (m->m_pkthdr.fw_flags & DUMMYNET_MBUF_TAGGED) {
 641                 ip_dn_queue(m);
 642                 return;
 643         }
 644         if (m->m_pkthdr.fw_flags & FW_MBUF_REDISPATCH) {
 645                 m->m_pkthdr.fw_flags &= ~FW_MBUF_REDISPATCH;
 646         }
 647 pass:
 648         /*
 649          * Process options and, if not destined for us,
 650          * ship it on.  ip_dooptions returns 1 when an
 651          * error was detected (causing an icmp message
 652          * to be sent and the original packet to be freed).
 653          */
 654         if (hlen > sizeof(struct ip) && ip_dooptions(m, 0, next_hop))
 655                 return;
 656
 657         /* greedy RSVP, snatches any PATH packet of the RSVP protocol and no
 658          * matter if it is destined to another node, or whether it is
 659          * a multicast one, RSVP wants it! and prevents it from being forwarded
 660          * anywhere else. Also checks if the rsvp daemon is running before
 661          * grabbing the packet.
 662          */
 663         if (rsvp_on && ip->ip_p == IPPROTO_RSVP)
 664                 goto ours;
 665
 666         /*
 667          * Check our list of addresses, to see if the packet is for us.
 668          * If we don't have any addresses, assume any unicast packet
 669          * we receive might be for us (and let the upper layers deal
 670          * with it).
 671          */
 672         if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid]) &&
 673             !(m->m_flags & (M_MCAST | M_BCAST)))
 674                 goto ours;
 675
 676         /*
 677          * Cache the destination address of the packet; this may be
 678          * changed by use of 'ipfw fwd'.
 679          */
 680         pkt_dst = next_hop ? next_hop->sin_addr : ip->ip_dst;
 681
 682         /*
 683          * Enable a consistency check between the destination address
 684          * and the arrival interface for a unicast packet (the RFC 1122
 685          * strong ES model) if IP forwarding is disabled and the packet
 686          * is not locally generated and the packet is not subject to
 687          * 'ipfw fwd'.
 688          *
 689          * XXX - Checking also should be disabled if the destination
 690          * address is ipnat'ed to a different interface.
 691          *
 692          * XXX - Checking is incompatible with IP aliases added
 693          * to the loopback interface instead of the interface where
 694          * the packets are received.
 695          */
 696         checkif = ip_checkinterface &&
 697                   !ipforwarding &&
 698                   m->m_pkthdr.rcvif != NULL &&
 699                   !(m->m_pkthdr.rcvif->if_flags & IFF_LOOPBACK) &&
 700                   next_hop == NULL;
 701
 702         /*
 703          * Check for exact addresses in the hash bucket.
 704          */
 705         LIST_FOREACH(iac, INADDR_HASH(pkt_dst.s_addr), ia_hash) {
 706                 ia = iac->ia;
 707
 708                 /*
 709                  * If the address matches, verify that the packet
 710                  * arrived via the correct interface if checking is
 711                  * enabled.
 712                  */
 713                 if (IA_SIN(ia)->sin_addr.s_addr == pkt_dst.s_addr &&
 714                     (!checkif || ia->ia_ifp == m->m_pkthdr.rcvif))
 715                         goto ours;
 716         }
 717         ia = NULL;
 718
 719         /*
 720          * Check for broadcast addresses.
 721          *
 722          * Only accept broadcast packets that arrive via the matching
 723          * interface.  Reception of forwarded directed broadcasts would
 724          * be handled via ip_forward() and ether_output() with the loopback
 725          * into the stack for SIMPLEX interfaces handled by ether_output().
 726          */
 727         if (m->m_pkthdr.rcvif != NULL &&
 728             m->m_pkthdr.rcvif->if_flags & IFF_BROADCAST) {
 729                 struct ifaddr_container *ifac;
 730
 731                 TAILQ_FOREACH(ifac, &m->m_pkthdr.rcvif->if_addrheads[mycpuid],
 732                               ifa_link) {
 733                         struct ifaddr *ifa = ifac->ifa;
 734
 735                         if (ifa->ifa_addr == NULL) /* shutdown/startup race */
 736                                 continue;
 737                         if (ifa->ifa_addr->sa_family != AF_INET)
 738                                 continue;
 739                         ia = ifatoia(ifa);
 740                         if (satosin(&ia->ia_broadaddr)->sin_addr.s_addr ==
 741                                                                 pkt_dst.s_addr)
 742                                 goto ours;
 743                         if (ia->ia_netbroadcast.s_addr == pkt_dst.s_addr)
 744                                 goto ours;
 745 #ifdef BOOTP_COMPAT
 746                         if (IA_SIN(ia)->sin_addr.s_addr == INADDR_ANY)
 747                                 goto ours;
 748 #endif
 749                 }
 750         }
 751         if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr))) {
 752                 struct in_multi *inm;
 753
 754                 if (ip_mrouter != NULL) {
 755                         /* XXX Multicast routing is not MPSAFE yet */
 756                         get_mplock();
 757
 758                         /*
 759                          * If we are acting as a multicast router, all
 760                          * incoming multicast packets are passed to the
 761                          * kernel-level multicast forwarding function.
 762                          * The packet is returned (relatively) intact; if
 763                          * ip_mforward() returns a non-zero value, the packet
 764                          * must be discarded, else it may be accepted below.
 765                          */
 766                         if (ip_mforward != NULL &&
 767                             ip_mforward(ip, m->m_pkthdr.rcvif, m, NULL) != 0) {
 768                                 rel_mplock();
 769                                 ipstat.ips_cantforward++;
 770                                 m_freem(m);
 771                                 return;
 772                         }
 773
 774                         rel_mplock();
 775
 776                         /*
 777                          * The process-level routing daemon needs to receive
 778                          * all multicast IGMP packets, whether or not this
 779                          * host belongs to their destination groups.
 780                          */
 781                         if (ip->ip_p == IPPROTO_IGMP)
 782                                 goto ours;
 783                         ipstat.ips_forward++;
 784                 }
 785                 /*
 786                  * See if we belong to the destination multicast group on the
 787                  * arrival interface.
 788                  */
 789                 inm = IN_LOOKUP_MULTI(&ip->ip_dst, m->m_pkthdr.rcvif);
 790                 if (inm == NULL) {
 791                         ipstat.ips_notmember++;
 792                         m_freem(m);
 793                         return;
 794                 }
 795                 goto ours;
 796         }
 797         if (ip->ip_dst.s_addr == INADDR_BROADCAST)
 798                 goto ours;
 799         if (ip->ip_dst.s_addr == INADDR_ANY)
 800                 goto ours;
 801
 802         /*
 803          * FAITH(Firewall Aided Internet Translator)
 804          */
 805         if (m->m_pkthdr.rcvif && m->m_pkthdr.rcvif->if_type == IFT_FAITH) {
 806                 if (ip_keepfaith) {
 807                         if (ip->ip_p == IPPROTO_TCP || ip->ip_p == IPPROTO_ICMP)
 808                                 goto ours;
 809                 }
 810                 m_freem(m);
 811                 return;
 812         }
 813
 814         /*
 815          * Not for us; forward if possible and desirable.
 816          */
 817         if (!ipforwarding) {
 818                 ipstat.ips_cantforward++;
 819                 m_freem(m);
 820         } else {
 821 #ifdef IPSEC
 822                 /*
 823                  * Enforce inbound IPsec SPD.
 824                  */
 825                 if (ipsec4_in_reject(m, NULL)) {
 826                         ipsecstat.in_polvio++;
 827                         goto bad;
 828                 }
 829 #endif
 830 #ifdef FAST_IPSEC
 831                 mtag = m_tag_find(m, PACKET_TAG_IPSEC_IN_DONE, NULL);
 832                 crit_enter();
 833                 if (mtag != NULL) {
 834                         tdbi = (struct tdb_ident *)m_tag_data(mtag);
 835                         sp = ipsec_getpolicy(tdbi, IPSEC_DIR_INBOUND);
 836                 } else {
 837                         sp = ipsec_getpolicybyaddr(m, IPSEC_DIR_INBOUND,
 838                                                    IP_FORWARDING, &error);
 839                 }
 840                 if (sp == NULL) {       /* NB: can happen if error */
 841                         crit_exit();
 842                         /*XXX error stat???*/
 843                         DPRINTF(("ip_input: no SP for forwarding\n"));  /*XXX*/
 844                         goto bad;
 845                 }
 846
 847                 /*
 848                  * Check security policy against packet attributes.
 849                  */
 850                 error = ipsec_in_reject(sp, m);
 851                 KEY_FREESP(&sp);
 852                 crit_exit();
 853                 if (error) {
 854                         ipstat.ips_cantforward++;
 855                         goto bad;
 856                 }
 857 #endif
 858                 ip_forward(m, using_srcrt, next_hop);
 859         }
 860         return;
 861
 862 ours:
 863
 864         /*
 865          * IPSTEALTH: Process non-routing options only
 866          * if the packet is destined for us.
 867          */
 868         if (ipstealth &&
 869             hlen > sizeof(struct ip) &&
 870             ip_dooptions(m, 1, next_hop))
 871                 return;
 872
 873         /* Count the packet in the ip address stats */
 874         if (ia != NULL) {
 875                 IFA_STAT_INC(&ia->ia_ifa, ipackets, 1);
 876                 IFA_STAT_INC(&ia->ia_ifa, ibytes, m->m_pkthdr.len);
 877         }
 878
 879         /*
 880          * If offset or IP_MF are set, must reassemble.
 881          * Otherwise, nothing need be done.
 882          * (We could look in the reassembly queue to see
 883          * if the packet was previously fragmented,
 884          * but it's not worth the time; just let them time out.)
 885          */
 886         if (ip->ip_off & (IP_MF | IP_OFFMASK)) {
 887                 /*
 888                  * Attempt reassembly; if it succeeds, proceed.  ip_reass()
 889                  * will return a different mbuf.
 890                  *
 891                  * NOTE: ip_reass() returns m with M_HASH cleared to force
 892                  *       us to recharacterize the packet.
 893                  */
 894                 m = ip_reass(m);
 895                 if (m == NULL)
 896                         return;
 897                 ip = mtod(m, struct ip *);
 898
 899                 /* Get the header length of the reassembled packet */
 900                 hlen = IP_VHL_HL(ip->ip_vhl) << 2;
 901         } else {
 902                 ip->ip_len -= hlen;
 903         }
 904
 905 #ifdef IPSEC
 906         /*
 907          * enforce IPsec policy checking if we are seeing last header.
 908          * note that we do not visit this with protocols with pcb layer
 909          * code - like udp/tcp/raw ip.
 910          */
 911         if ((inetsw[ip_protox[ip->ip_p]].pr_flags & PR_LASTHDR) &&
 912             ipsec4_in_reject(m, NULL)) {
 913                 ipsecstat.in_polvio++;
 914                 goto bad;
 915         }
 916 #endif
 917 #ifdef FAST_IPSEC
 918         /*
 919          * enforce IPsec policy checking if we are seeing last header.
 920          * note that we do not visit this with protocols with pcb layer
 921          * code - like udp/tcp/raw ip.
 922          */
 923         if (inetsw[ip_protox[ip->ip_p]].pr_flags & PR_LASTHDR) {
 924                 /*
 925                  * Check if the packet has already had IPsec processing
 926                  * done.  If so, then just pass it along.  This tag gets
 927                  * set during AH, ESP, etc. input handling, before the
 928                  * packet is returned to the ip input queue for delivery.
 929                  */
 930                 mtag = m_tag_find(m, PACKET_TAG_IPSEC_IN_DONE, NULL);
 931                 crit_enter();
 932                 if (mtag != NULL) {
 933                         tdbi = (struct tdb_ident *)m_tag_data(mtag);
 934                         sp = ipsec_getpolicy(tdbi, IPSEC_DIR_INBOUND);
 935                 } else {
 936                         sp = ipsec_getpolicybyaddr(m, IPSEC_DIR_INBOUND,
 937                                                    IP_FORWARDING, &error);
 938                 }
 939                 if (sp != NULL) {
 940                         /*
 941                          * Check security policy against packet attributes.
 942                          */
 943                         error = ipsec_in_reject(sp, m);
 944                         KEY_FREESP(&sp);
 945                 } else {
 946                         /* XXX error stat??? */
 947                         error = EINVAL;
 948 DPRINTF(("ip_input: no SP, packet discarded\n"));/*XXX*/
 949                         crit_exit();
 950                         goto bad;
 951                 }
 952                 crit_exit();
 953                 if (error)
 954                         goto bad;
 955         }
 956 #endif /* FAST_IPSEC */
 957
 958         /*
 959          * We must forward the packet to the correct protocol thread if
 960          * we are not already in it.
 961          *
 962          * NOTE: ip_len is now in host form.  ip_len is not adjusted
 963          *       further for protocol processing, instead we pass hlen
 964          *       to the protosw and let it deal with it.
 965          */
 966         ipstat.ips_delivered++;
 967
 968         if ((m->m_flags & M_HASH) == 0) {
 969 #ifdef RSS_DEBUG
 970                 atomic_add_long(&ip_rehash_count, 1);
 971 #endif
 972                 ip->ip_len = htons(ip->ip_len + hlen);
 973                 ip->ip_off = htons(ip->ip_off);
 974
 975                 ip_hashfn(&m, 0);
 976                 if (m == NULL)
 977                         return;
 978
 979                 ip = mtod(m, struct ip *);
 980                 ip->ip_len = ntohs(ip->ip_len) - hlen;
 981                 ip->ip_off = ntohs(ip->ip_off);
 982                 KKASSERT(m->m_flags & M_HASH);
 983         }
 984         port = netisr_hashport(m->m_pkthdr.hash);
 985
 986         if (port != &curthread->td_msgport) {
 987                 struct netmsg_packet *pmsg;
 988
 989 #ifdef RSS_DEBUG
 990                 atomic_add_long(&ip_dispatch_slow, 1);
 991 #endif
 992
 993                 pmsg = &m->m_hdr.mh_netmsg;
 994                 netmsg_init(&pmsg->base, NULL, &netisr_apanic_rport,
 995                             0, transport_processing_handler);
 996                 pmsg->nm_packet = m;
 997                 pmsg->base.lmsg.u.ms_result = hlen;
 998                 lwkt_sendmsg(port, &pmsg->base.lmsg);
 999         } else {
1000 #ifdef RSS_DEBUG
1001                 atomic_add_long(&ip_dispatch_fast, 1);
1002 #endif
1003                 transport_processing_oncpu(m, hlen, ip);
1004         }
1005         return;
1006
1007 bad:
1008         m_freem(m);
1009 }
1010
1011 /*
1012  * Take incoming datagram fragment and try to reassemble it into
1013  * whole datagram.  If a chain for reassembly of this datagram already
1014  * exists, then it is given as fp; otherwise have to make a chain.
1015  */
1016 struct mbuf *
1017 ip_reass(struct mbuf *m)
1018 {
1019         struct ipfrag_queue *fragq = &ipfrag_queue_pcpu[mycpuid];
1020         struct ip *ip = mtod(m, struct ip *);
1021         struct mbuf *p = NULL, *q, *nq;
1022         struct mbuf *n;
1023         struct ipq *fp = NULL;
1024         struct ipqhead *head;
1025         int hlen = IP_VHL_HL(ip->ip_vhl) << 2;
1026         int i, next;
1027         u_short sum;
1028
1029         /* If maxnipq is 0, never accept fragments. */
1030         if (maxnipq == 0) {
1031                 ipstat.ips_fragments++;
1032                 ipstat.ips_fragdropped++;
1033                 m_freem(m);
1034                 return NULL;
1035         }
1036
1037         sum = IPREASS_HASH(ip->ip_src.s_addr, ip->ip_id);
1038         /*
1039          * Look for queue of fragments of this datagram.
1040          */
1041         head = &fragq->ipq[sum];
1042         TAILQ_FOREACH(fp, head, ipq_list) {
1043                 if (ip->ip_id == fp->ipq_id &&
1044                     ip->ip_src.s_addr == fp->ipq_src.s_addr &&
1045                     ip->ip_dst.s_addr == fp->ipq_dst.s_addr &&
1046                     ip->ip_p == fp->ipq_p)
1047                         goto found;
1048         }
1049
1050         fp = NULL;
1051
1052         /*
1053          * Enforce upper bound on number of fragmented packets
1054          * for which we attempt reassembly;
1055          * If maxnipq is -1, accept all fragments without limitation.
1056          */
1057         if (fragq->nipq > maxnipq && maxnipq > 0) {
1058                 /*
1059                  * drop something from the tail of the current queue
1060                  * before proceeding further
1061                  */
1062                 struct ipq *q = TAILQ_LAST(head, ipqhead);
1063                 if (q == NULL) {
1064                         /*
1065                          * The current queue is empty,
1066                          * so drop from one of the others.
1067                          */
1068                         for (i = 0; i < IPREASS_NHASH; i++) {
1069                                 struct ipq *r = TAILQ_LAST(&fragq->ipq[i],
1070                                     ipqhead);
1071                                 if (r) {
1072                                         ipstat.ips_fragtimeout += r->ipq_nfrags;
1073                                         ip_freef(fragq, &fragq->ipq[i], r);
1074                                         break;
1075                                 }
1076                         }
1077                 } else {
1078                         ipstat.ips_fragtimeout += q->ipq_nfrags;
1079                         ip_freef(fragq, head, q);
1080                 }
1081         }
1082 found:
1083         /*
1084          * Adjust ip_len to not reflect header,
1085          * convert offset of this to bytes.
1086          */
1087         ip->ip_len -= hlen;
1088         if (ip->ip_off & IP_MF) {
1089                 /*
1090                  * Make sure that fragments have a data length
1091                  * that's a non-zero multiple of 8 bytes.
1092                  */
1093                 if (ip->ip_len == 0 || (ip->ip_len & 0x7) != 0) {
1094                         ipstat.ips_toosmall++; /* XXX */
1095                         m_freem(m);
1096                         goto done;
1097                 }
1098                 m->m_flags |= M_FRAG;
1099         } else {
1100                 m->m_flags &= ~M_FRAG;
1101         }
1102         ip->ip_off <<= 3;
1103
1104         ipstat.ips_fragments++;
1105         m->m_pkthdr.header = ip;
1106
1107         /*
1108          * If the hardware has not done csum over this fragment
1109          * then csum_data is not valid at all.
1110          */
1111         if ((m->m_pkthdr.csum_flags & (CSUM_FRAG_NOT_CHECKED | CSUM_DATA_VALID))
1112             == (CSUM_FRAG_NOT_CHECKED | CSUM_DATA_VALID)) {
1113                 m->m_pkthdr.csum_data = 0;
1114                 m->m_pkthdr.csum_flags &= ~(CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1115         }
1116
1117         /*
1118          * Presence of header sizes in mbufs
1119          * would confuse code below.
1120          */
1121         m->m_data += hlen;
1122         m->m_len -= hlen;
1123
1124         /*
1125          * If first fragment to arrive, create a reassembly queue.
1126          */
1127         if (fp == NULL) {
1128                 if ((fp = mpipe_alloc_nowait(&ipq_mpipe)) == NULL)
1129                         goto dropfrag;
1130                 TAILQ_INSERT_HEAD(head, fp, ipq_list);
1131                 fragq->nipq++;
1132                 fp->ipq_nfrags = 1;
1133                 fp->ipq_ttl = IPFRAGTTL;
1134                 fp->ipq_p = ip->ip_p;
1135                 fp->ipq_id = ip->ip_id;
1136                 fp->ipq_src = ip->ip_src;
1137                 fp->ipq_dst = ip->ip_dst;
1138                 fp->ipq_frags = m;
1139                 m->m_nextpkt = NULL;
1140                 goto inserted;
1141         } else {
1142                 fp->ipq_nfrags++;
1143         }
1144
1145 #define GETIP(m)        ((struct ip*)((m)->m_pkthdr.header))
1146
1147         /*
1148          * Find a segment which begins after this one does.
1149          */
1150         for (p = NULL, q = fp->ipq_frags; q; p = q, q = q->m_nextpkt) {
1151                 if (GETIP(q)->ip_off > ip->ip_off)
1152                         break;
1153         }
1154
1155         /*
1156          * If there is a preceding segment, it may provide some of
1157          * our data already.  If so, drop the data from the incoming
1158          * segment.  If it provides all of our data, drop us, otherwise
1159          * stick new segment in the proper place.
1160          *
1161          * If some of the data is dropped from the the preceding
1162          * segment, then it's checksum is invalidated.
1163          */
1164         if (p) {
1165                 i = GETIP(p)->ip_off + GETIP(p)->ip_len - ip->ip_off;
1166                 if (i > 0) {
1167                         if (i >= ip->ip_len)
1168                                 goto dropfrag;
1169                         m_adj(m, i);
1170                         m->m_pkthdr.csum_flags = 0;
1171                         ip->ip_off += i;
1172                         ip->ip_len -= i;
1173                 }
1174                 m->m_nextpkt = p->m_nextpkt;
1175                 p->m_nextpkt = m;
1176         } else {
1177                 m->m_nextpkt = fp->ipq_frags;
1178                 fp->ipq_frags = m;
1179         }
1180
1181         /*
1182          * While we overlap succeeding segments trim them or,
1183          * if they are completely covered, dequeue them.
1184          */
1185         for (; q != NULL && ip->ip_off + ip->ip_len > GETIP(q)->ip_off;
1186              q = nq) {
1187                 i = (ip->ip_off + ip->ip_len) - GETIP(q)->ip_off;
1188                 if (i < GETIP(q)->ip_len) {
1189                         GETIP(q)->ip_len -= i;
1190                         GETIP(q)->ip_off += i;
1191                         m_adj(q, i);
1192                         q->m_pkthdr.csum_flags = 0;
1193                         break;
1194                 }
1195                 nq = q->m_nextpkt;
1196                 m->m_nextpkt = nq;
1197                 ipstat.ips_fragdropped++;
1198                 fp->ipq_nfrags--;
1199                 q->m_nextpkt = NULL;
1200                 m_freem(q);
1201         }
1202
1203 inserted:
1204         /*
1205          * Check for complete reassembly and perform frag per packet
1206          * limiting.
1207          *
1208          * Frag limiting is performed here so that the nth frag has
1209          * a chance to complete the packet before we drop the packet.
1210          * As a result, n+1 frags are actually allowed per packet, but
1211          * only n will ever be stored. (n = maxfragsperpacket.)
1212          *
1213          */
1214         next = 0;
1215         for (p = NULL, q = fp->ipq_frags; q; p = q, q = q->m_nextpkt) {
1216                 if (GETIP(q)->ip_off != next) {
1217                         if (fp->ipq_nfrags > maxfragsperpacket) {
1218                                 ipstat.ips_fragdropped += fp->ipq_nfrags;
1219                                 ip_freef(fragq, head, fp);
1220                         }
1221                         goto done;
1222                 }
1223                 next += GETIP(q)->ip_len;
1224         }
1225         /* Make sure the last packet didn't have the IP_MF flag */
1226         if (p->m_flags & M_FRAG) {
1227                 if (fp->ipq_nfrags > maxfragsperpacket) {
1228                         ipstat.ips_fragdropped += fp->ipq_nfrags;
1229                         ip_freef(fragq, head, fp);
1230                 }
1231                 goto done;
1232         }
1233
1234         /*
1235          * Reassembly is complete.  Make sure the packet is a sane size.
1236          */
1237         q = fp->ipq_frags;
1238         ip = GETIP(q);
1239         if (next + (IP_VHL_HL(ip->ip_vhl) << 2) > IP_MAXPACKET) {
1240                 ipstat.ips_toolong++;
1241                 ipstat.ips_fragdropped += fp->ipq_nfrags;
1242                 ip_freef(fragq, head, fp);
1243                 goto done;
1244         }
1245
1246         /*
1247          * Concatenate fragments.
1248          */
1249         m = q;
1250         n = m->m_next;
1251         m->m_next = NULL;
1252         m_cat(m, n);
1253         nq = q->m_nextpkt;
1254         q->m_nextpkt = NULL;
1255         for (q = nq; q != NULL; q = nq) {
1256                 nq = q->m_nextpkt;
1257                 q->m_nextpkt = NULL;
1258                 m->m_pkthdr.csum_flags &= q->m_pkthdr.csum_flags;
1259                 m->m_pkthdr.csum_data += q->m_pkthdr.csum_data;
1260                 m_cat(m, q);
1261         }
1262
1263         /*
1264          * Clean up the 1's complement checksum.  Carry over 16 bits must
1265          * be added back.  This assumes no more then 65535 packet fragments
1266          * were reassembled.  A second carry can also occur (but not a third).
1267          */
1268         m->m_pkthdr.csum_data = (m->m_pkthdr.csum_data & 0xffff) +
1269                                 (m->m_pkthdr.csum_data >> 16);
1270         if (m->m_pkthdr.csum_data > 0xFFFF)
1271                 m->m_pkthdr.csum_data -= 0xFFFF;
1272
1273         /*
1274          * Create header for new ip packet by
1275          * modifying header of first packet;
1276          * dequeue and discard fragment reassembly header.
1277          * Make header visible.
1278          */
1279         ip->ip_len = next;
1280         ip->ip_src = fp->ipq_src;
1281         ip->ip_dst = fp->ipq_dst;
1282         TAILQ_REMOVE(head, fp, ipq_list);
1283         fragq->nipq--;
1284         mpipe_free(&ipq_mpipe, fp);
1285         m->m_len += (IP_VHL_HL(ip->ip_vhl) << 2);
1286         m->m_data -= (IP_VHL_HL(ip->ip_vhl) << 2);
1287         /* some debugging cruft by sklower, below, will go away soon */
1288         if (m->m_flags & M_PKTHDR) { /* XXX this should be done elsewhere */
1289                 int plen = 0;
1290
1291                 for (n = m; n; n = n->m_next)
1292                         plen += n->m_len;
1293                 m->m_pkthdr.len = plen;
1294         }
1295
1296         /*
1297          * Reassembly complete, return the next protocol.
1298          *
1299          * Be sure to clear M_HASH to force the packet
1300          * to be re-characterized.
1301          *
1302          * Clear M_FRAG, we are no longer a fragment.
1303          */
1304         m->m_flags &= ~(M_HASH | M_FRAG);
1305
1306         ipstat.ips_reassembled++;
1307         return (m);
1308
1309 dropfrag:
1310         ipstat.ips_fragdropped++;
1311         if (fp != NULL)
1312                 fp->ipq_nfrags--;
1313         m_freem(m);
1314 done:
1315         return (NULL);
1316
1317 #undef GETIP
1318 }
1319
1320 /*
1321  * Free a fragment reassembly header and all
1322  * associated datagrams.
1323  */
1324 static void
1325 ip_freef(struct ipfrag_queue *fragq, struct ipqhead *fhp, struct ipq *fp)
1326 {
1327         struct mbuf *q;
1328
1329         /*
1330          * Remove first to protect against blocking
1331          */
1332         TAILQ_REMOVE(fhp, fp, ipq_list);
1333
1334         /*
1335          * Clean out at our leisure
1336          */
1337         while (fp->ipq_frags) {
1338                 q = fp->ipq_frags;
1339                 fp->ipq_frags = q->m_nextpkt;
1340                 q->m_nextpkt = NULL;
1341                 m_freem(q);
1342         }
1343         mpipe_free(&ipq_mpipe, fp);
1344         fragq->nipq--;
1345 }
1346
1347 /*
1348  * If a timer expires on a reassembly queue, discard it.
1349  */
1350 static void
1351 ipfrag_timeo_dispatch(netmsg_t nmsg)
1352 {
1353         struct ipfrag_queue *fragq = &ipfrag_queue_pcpu[mycpuid];
1354         struct ipq *fp, *fp_temp;
1355         struct ipqhead *head;
1356         int i;
1357
1358         crit_enter();
1359         lwkt_replymsg(&nmsg->lmsg, 0);  /* reply ASAP */
1360         crit_exit();
1361
1362         for (i = 0; i < IPREASS_NHASH; i++) {
1363                 head = &fragq->ipq[i];
1364                 TAILQ_FOREACH_MUTABLE(fp, head, ipq_list, fp_temp) {
1365                         if (--fp->ipq_ttl == 0) {
1366                                 ipstat.ips_fragtimeout += fp->ipq_nfrags;
1367                                 ip_freef(fragq, head, fp);
1368                         }
1369                 }
1370         }
1371         /*
1372          * If we are over the maximum number of fragments
1373          * (due to the limit being lowered), drain off
1374          * enough to get down to the new limit.
1375          */
1376         if (maxnipq >= 0 && fragq->nipq > maxnipq) {
1377                 for (i = 0; i < IPREASS_NHASH; i++) {
1378                         head = &fragq->ipq[i];
1379                         while (fragq->nipq > maxnipq && !TAILQ_EMPTY(head)) {
1380                                 ipstat.ips_fragdropped +=
1381                                     TAILQ_FIRST(head)->ipq_nfrags;
1382                                 ip_freef(fragq, head, TAILQ_FIRST(head));
1383                         }
1384                 }
1385         }
1386 }
1387
1388 static void
1389 ipfrag_timeo_ipi(void *arg __unused)
1390 {
1391         int cpu = mycpuid;
1392         struct lwkt_msg *msg = &ipfrag_queue_pcpu[cpu].timeo_netmsg.lmsg;
1393
1394         crit_enter();
1395         if (msg->ms_flags & MSGF_DONE)
1396                 lwkt_sendmsg_oncpu(netisr_cpuport(cpu), msg);
1397         crit_exit();
1398 }
1399
1400 static void
1401 ipfrag_slowtimo(void)
1402 {
1403         cpumask_t mask;
1404
1405         CPUMASK_ASSBMASK(mask, ncpus);
1406         CPUMASK_ANDMASK(mask, smp_active_mask);
1407         if (CPUMASK_TESTNZERO(mask))
1408                 lwkt_send_ipiq_mask(mask, ipfrag_timeo_ipi, NULL);
1409 }
1410
1411 /*
1412  * IP timer processing
1413  */
1414 void
1415 ip_slowtimo(void)
1416 {
1417         ipfrag_slowtimo();
1418         ipflow_slowtimo();
1419 }
1420
1421 /*
1422  * Drain off all datagram fragments.
1423  */
1424 static void
1425 ipfrag_drain_dispatch(netmsg_t nmsg)
1426 {
1427         struct ipfrag_queue *fragq = &ipfrag_queue_pcpu[mycpuid];
1428         struct ipqhead *head;
1429         int i;
1430
1431         crit_enter();
1432         lwkt_replymsg(&nmsg->lmsg, 0);  /* reply ASAP */
1433         crit_exit();
1434
1435         for (i = 0; i < IPREASS_NHASH; i++) {
1436                 head = &fragq->ipq[i];
1437                 while (!TAILQ_EMPTY(head)) {
1438                         ipstat.ips_fragdropped += TAILQ_FIRST(head)->ipq_nfrags;
1439                         ip_freef(fragq, head, TAILQ_FIRST(head));
1440                 }
1441         }
1442 }
1443
1444 static void
1445 ipfrag_drain_ipi(void *arg __unused)
1446 {
1447         int cpu = mycpuid;
1448         struct lwkt_msg *msg = &ipfrag_queue_pcpu[cpu].drain_netmsg.lmsg;
1449
1450         crit_enter();
1451         if (msg->ms_flags & MSGF_DONE)
1452                 lwkt_sendmsg_oncpu(netisr_cpuport(cpu), msg);
1453         crit_exit();
1454 }
1455
1456 static void
1457 ipfrag_drain(void)
1458 {
1459         cpumask_t mask;
1460
1461         CPUMASK_ASSBMASK(mask, ncpus);
1462         CPUMASK_ANDMASK(mask, smp_active_mask);
1463         if (CPUMASK_TESTNZERO(mask))
1464                 lwkt_send_ipiq_mask(mask, ipfrag_drain_ipi, NULL);
1465 }
1466
1467 void
1468 ip_drain(void)
1469 {
1470         ipfrag_drain();
1471         in_rtqdrain();
1472 }
1473
1474 /*
1475  * Do option processing on a datagram,
1476  * possibly discarding it if bad options are encountered,
1477  * or forwarding it if source-routed.
1478  * The pass argument is used when operating in the IPSTEALTH
1479  * mode to tell what options to process:
1480  * [LS]SRR (pass 0) or the others (pass 1).
1481  * The reason for as many as two passes is that when doing IPSTEALTH,
1482  * non-routing options should be processed only if the packet is for us.
1483  * Returns 1 if packet has been forwarded/freed,
1484  * 0 if the packet should be processed further.
1485  */
1486 static int
1487 ip_dooptions(struct mbuf *m, int pass, struct sockaddr_in *next_hop)
1488 {
1489         struct sockaddr_in ipaddr = { sizeof ipaddr, AF_INET };
1490         struct ip *ip = mtod(m, struct ip *);
1491         u_char *cp;
1492         struct in_ifaddr *ia;
1493         int opt, optlen, cnt, off, code, type = ICMP_PARAMPROB;
1494         boolean_t forward = FALSE;
1495         struct in_addr *sin, dst;
1496         n_time ntime;
1497
1498         dst = ip->ip_dst;
1499         cp = (u_char *)(ip + 1);
1500         cnt = (IP_VHL_HL(ip->ip_vhl) << 2) - sizeof(struct ip);
1501         for (; cnt > 0; cnt -= optlen, cp += optlen) {
1502                 opt = cp[IPOPT_OPTVAL];
1503                 if (opt == IPOPT_EOL)
1504                         break;
1505                 if (opt == IPOPT_NOP)
1506                         optlen = 1;
1507                 else {
1508                         if (cnt < IPOPT_OLEN + sizeof(*cp)) {
1509                                 code = &cp[IPOPT_OLEN] - (u_char *)ip;
1510                                 goto bad;
1511                         }
1512                         optlen = cp[IPOPT_OLEN];
1513                         if (optlen < IPOPT_OLEN + sizeof(*cp) || optlen > cnt) {
1514                                 code = &cp[IPOPT_OLEN] - (u_char *)ip;
1515                                 goto bad;
1516                         }
1517                 }
1518                 switch (opt) {
1519
1520                 default:
1521                         break;
1522
1523                 /*
1524                  * Source routing with record.
1525                  * Find interface with current destination address.
1526                  * If none on this machine then drop if strictly routed,
1527                  * or do nothing if loosely routed.
1528                  * Record interface address and bring up next address
1529                  * component.  If strictly routed make sure next
1530                  * address is on directly accessible net.
1531                  */
1532                 case IPOPT_LSRR:
1533                 case IPOPT_SSRR:
1534                         if (ipstealth && pass > 0)
1535                                 break;
1536                         if (optlen < IPOPT_OFFSET + sizeof(*cp)) {
1537                                 code = &cp[IPOPT_OLEN] - (u_char *)ip;
1538                                 goto bad;
1539                         }
1540                         if ((off = cp[IPOPT_OFFSET]) < IPOPT_MINOFF) {
1541                                 code = &cp[IPOPT_OFFSET] - (u_char *)ip;
1542                                 goto bad;
1543                         }
1544                         ipaddr.sin_addr = ip->ip_dst;
1545                         ia = (struct in_ifaddr *)
1546                                 ifa_ifwithaddr((struct sockaddr *)&ipaddr);
1547                         if (ia == NULL) {
1548                                 if (opt == IPOPT_SSRR) {
1549                                         type = ICMP_UNREACH;
1550                                         code = ICMP_UNREACH_SRCFAIL;
1551                                         goto bad;
1552                                 }
1553                                 if (!ip_dosourceroute)
1554                                         goto nosourcerouting;
1555                                 /*
1556                                  * Loose routing, and not at next destination
1557                                  * yet; nothing to do except forward.
1558                                  */
1559                                 break;
1560                         }
1561                         off--;                  /* 0 origin */
1562                         if (off > optlen - (int)sizeof(struct in_addr)) {
1563                                 /*
1564                                  * End of source route.  Should be for us.
1565                                  */
1566                                 if (!ip_acceptsourceroute)
1567                                         goto nosourcerouting;
1568                                 save_rte(m, cp, ip->ip_src);
1569                                 break;
1570                         }
1571                         if (ipstealth)
1572                                 goto dropit;
1573                         if (!ip_dosourceroute) {
1574                                 if (ipforwarding) {
1575                                         char buf[sizeof "aaa.bbb.ccc.ddd"];
1576
1577                                         /*
1578                                          * Acting as a router, so generate ICMP
1579                                          */
1580 nosourcerouting:
1581                                         strcpy(buf, inet_ntoa(ip->ip_dst));
1582                                         log(LOG_WARNING,
1583                                             "attempted source route from %s to %s\n",
1584                                             inet_ntoa(ip->ip_src), buf);
1585                                         type = ICMP_UNREACH;
1586                                         code = ICMP_UNREACH_SRCFAIL;
1587                                         goto bad;
1588                                 } else {
1589                                         /*
1590                                          * Not acting as a router,
1591                                          * so silently drop.
1592                                          */
1593 dropit:
1594                                         ipstat.ips_cantforward++;
1595                                         m_freem(m);
1596                                         return (1);
1597                                 }
1598                         }
1599
1600                         /*
1601                          * locate outgoing interface
1602                          */
1603                         memcpy(&ipaddr.sin_addr, cp + off,
1604                             sizeof ipaddr.sin_addr);
1605
1606                         if (opt == IPOPT_SSRR) {
1607 #define INA     struct in_ifaddr *
1608 #define SA      struct sockaddr *
1609                                 if ((ia = (INA)ifa_ifwithdstaddr((SA)&ipaddr))
1610                                                                         == NULL)
1611                                         ia = (INA)ifa_ifwithnet((SA)&ipaddr);
1612                         } else {
1613                                 ia = ip_rtaddr(ipaddr.sin_addr, NULL);
1614                         }
1615                         if (ia == NULL) {
1616                                 type = ICMP_UNREACH;
1617                                 code = ICMP_UNREACH_SRCFAIL;
1618                                 goto bad;
1619                         }
1620                         ip->ip_dst = ipaddr.sin_addr;
1621                         memcpy(cp + off, &IA_SIN(ia)->sin_addr,
1622                             sizeof(struct in_addr));
1623                         cp[IPOPT_OFFSET] += sizeof(struct in_addr);
1624                         /*
1625                          * Let ip_intr's mcast routing check handle mcast pkts
1626                          */
1627                         forward = !IN_MULTICAST(ntohl(ip->ip_dst.s_addr));
1628                         break;
1629
1630                 case IPOPT_RR:
1631                         if (ipstealth && pass == 0)
1632                                 break;
1633                         if (optlen < IPOPT_OFFSET + sizeof(*cp)) {
1634                                 code = &cp[IPOPT_OFFSET] - (u_char *)ip;
1635                                 goto bad;
1636                         }
1637                         if ((off = cp[IPOPT_OFFSET]) < IPOPT_MINOFF) {
1638                                 code = &cp[IPOPT_OFFSET] - (u_char *)ip;
1639                                 goto bad;
1640                         }
1641                         /*
1642                          * If no space remains, ignore.
1643                          */
1644                         off--;                  /* 0 origin */
1645                         if (off > optlen - (int)sizeof(struct in_addr))
1646                                 break;
1647                         memcpy(&ipaddr.sin_addr, &ip->ip_dst,
1648                             sizeof ipaddr.sin_addr);
1649                         /*
1650                          * locate outgoing interface; if we're the destination,
1651                          * use the incoming interface (should be same).
1652                          */
1653                         if ((ia = (INA)ifa_ifwithaddr((SA)&ipaddr)) == NULL &&
1654                             (ia = ip_rtaddr(ipaddr.sin_addr, NULL)) == NULL) {
1655                                 type = ICMP_UNREACH;
1656                                 code = ICMP_UNREACH_HOST;
1657                                 goto bad;
1658                         }
1659                         memcpy(cp + off, &IA_SIN(ia)->sin_addr,
1660                             sizeof(struct in_addr));
1661                         cp[IPOPT_OFFSET] += sizeof(struct in_addr);
1662                         break;
1663
1664                 case IPOPT_TS:
1665                         if (ipstealth && pass == 0)
1666                                 break;
1667                         code = cp - (u_char *)ip;
1668                         if (optlen < 4 || optlen > 40) {
1669                                 code = &cp[IPOPT_OLEN] - (u_char *)ip;
1670                                 goto bad;
1671                         }
1672                         if ((off = cp[IPOPT_OFFSET]) < 5) {
1673                                 code = &cp[IPOPT_OLEN] - (u_char *)ip;
1674                                 goto bad;
1675                         }
1676                         if (off > optlen - (int)sizeof(int32_t)) {
1677                                 cp[IPOPT_OFFSET + 1] += (1 << 4);
1678                                 if ((cp[IPOPT_OFFSET + 1] & 0xf0) == 0) {
1679                                         code = &cp[IPOPT_OFFSET] - (u_char *)ip;
1680                                         goto bad;
1681                                 }
1682                                 break;
1683                         }
1684                         off--;                          /* 0 origin */
1685                         sin = (struct in_addr *)(cp + off);
1686                         switch (cp[IPOPT_OFFSET + 1] & 0x0f) {
1687
1688                         case IPOPT_TS_TSONLY:
1689                                 break;
1690
1691                         case IPOPT_TS_TSANDADDR:
1692                                 if (off + sizeof(n_time) +
1693                                     sizeof(struct in_addr) > optlen) {
1694                                         code = &cp[IPOPT_OFFSET] - (u_char *)ip;
1695                                         goto bad;
1696                                 }
1697                                 ipaddr.sin_addr = dst;
1698                                 ia = (INA)ifaof_ifpforaddr((SA)&ipaddr,
1699                                                             m->m_pkthdr.rcvif);
1700                                 if (ia == NULL)
1701                                         continue;
1702                                 memcpy(sin, &IA_SIN(ia)->sin_addr,
1703                                     sizeof(struct in_addr));
1704                                 cp[IPOPT_OFFSET] += sizeof(struct in_addr);
1705                                 off += sizeof(struct in_addr);
1706                                 break;
1707
1708                         case IPOPT_TS_PRESPEC:
1709                                 if (off + sizeof(n_time) +
1710                                     sizeof(struct in_addr) > optlen) {
1711                                         code = &cp[IPOPT_OFFSET] - (u_char *)ip;
1712                                         goto bad;
1713                                 }
1714                                 memcpy(&ipaddr.sin_addr, sin,
1715                                     sizeof(struct in_addr));
1716                                 if (ifa_ifwithaddr((SA)&ipaddr) == NULL)
1717                                         continue;
1718                                 cp[IPOPT_OFFSET] += sizeof(struct in_addr);
1719                                 off += sizeof(struct in_addr);
1720                                 break;
1721
1722                         default:
1723                                 code = &cp[IPOPT_OFFSET + 1] - (u_char *)ip;
1724                                 goto bad;
1725                         }
1726                         ntime = iptime();
1727                         memcpy(cp + off, &ntime, sizeof(n_time));
1728                         cp[IPOPT_OFFSET] += sizeof(n_time);
1729                 }
1730         }
1731         if (forward && ipforwarding) {
1732                 ip_forward(m, TRUE, next_hop);
1733                 return (1);
1734         }
1735         return (0);
1736 bad:
1737         icmp_error(m, type, code, 0, 0);
1738         ipstat.ips_badoptions++;
1739         return (1);
1740 }
1741
1742 /*
1743  * Given address of next destination (final or next hop),
1744  * return internet address info of interface to be used to get there.
1745  */
1746 struct in_ifaddr *
1747 ip_rtaddr(struct in_addr dst, struct route *ro0)
1748 {
1749         struct route sro, *ro;
1750         struct sockaddr_in *sin;
1751         struct in_ifaddr *ia;
1752
1753         if (ro0 != NULL) {
1754                 ro = ro0;
1755         } else {
1756                 bzero(&sro, sizeof(sro));
1757                 ro = &sro;
1758         }
1759
1760         sin = (struct sockaddr_in *)&ro->ro_dst;
1761
1762         if (ro->ro_rt == NULL || dst.s_addr != sin->sin_addr.s_addr) {
1763                 if (ro->ro_rt != NULL) {
1764                         RTFREE(ro->ro_rt);
1765                         ro->ro_rt = NULL;
1766                 }
1767                 sin->sin_family = AF_INET;
1768                 sin->sin_len = sizeof *sin;
1769                 sin->sin_addr = dst;
1770                 rtalloc_ign(ro, RTF_PRCLONING);
1771         }
1772
1773         if (ro->ro_rt == NULL)
1774                 return (NULL);
1775
1776         ia = ifatoia(ro->ro_rt->rt_ifa);
1777
1778         if (ro == &sro)
1779                 RTFREE(ro->ro_rt);
1780         return ia;
1781 }
1782
1783 /*
1784  * Save incoming source route for use in replies,
1785  * to be picked up later by ip_srcroute if the receiver is interested.
1786  */
1787 static void
1788 save_rte(struct mbuf *m, u_char *option, struct in_addr dst)
1789 {
1790         struct m_tag *mtag;
1791         struct ip_srcrt_opt *opt;
1792         unsigned olen;
1793
1794         mtag = m_tag_get(PACKET_TAG_IPSRCRT, sizeof(*opt), MB_DONTWAIT);
1795         if (mtag == NULL)
1796                 return;
1797         opt = m_tag_data(mtag);
1798
1799         olen = option[IPOPT_OLEN];
1800 #ifdef DIAGNOSTIC
1801         if (ipprintfs)
1802                 kprintf("save_rte: olen %d\n", olen);
1803 #endif
1804         if (olen > sizeof(opt->ip_srcrt) - (1 + sizeof(dst))) {
1805                 m_tag_free(mtag);
1806                 return;
1807         }
1808         bcopy(option, opt->ip_srcrt.srcopt, olen);
1809         opt->ip_nhops = (olen - IPOPT_OFFSET - 1) / sizeof(struct in_addr);
1810         opt->ip_srcrt.dst = dst;
1811         m_tag_prepend(m, mtag);
1812 }
1813
1814 /*
1815  * Retrieve incoming source route for use in replies,
1816  * in the same form used by setsockopt.
1817  * The first hop is placed before the options, will be removed later.
1818  */
1819 struct mbuf *
1820 ip_srcroute(struct mbuf *m0)
1821 {
1822         struct in_addr *p, *q;
1823         struct mbuf *m;
1824         struct m_tag *mtag;
1825         struct ip_srcrt_opt *opt;
1826
1827         if (m0 == NULL)
1828                 return NULL;
1829
1830         mtag = m_tag_find(m0, PACKET_TAG_IPSRCRT, NULL);
1831         if (mtag == NULL)
1832                 return NULL;
1833         opt = m_tag_data(mtag);
1834
1835         if (opt->ip_nhops == 0)
1836                 return (NULL);
1837         m = m_get(MB_DONTWAIT, MT_HEADER);
1838         if (m == NULL)
1839                 return (NULL);
1840
1841 #define OPTSIZ  (sizeof(opt->ip_srcrt.nop) + sizeof(opt->ip_srcrt.srcopt))
1842
1843         /* length is (nhops+1)*sizeof(addr) + sizeof(nop + srcrt header) */
1844         m->m_len = opt->ip_nhops * sizeof(struct in_addr) +
1845                    sizeof(struct in_addr) + OPTSIZ;
1846 #ifdef DIAGNOSTIC
1847         if (ipprintfs) {
1848                 kprintf("ip_srcroute: nhops %d mlen %d",
1849                         opt->ip_nhops, m->m_len);
1850         }
1851 #endif
1852
1853         /*
1854          * First save first hop for return route
1855          */
1856         p = &opt->ip_srcrt.route[opt->ip_nhops - 1];
1857         *(mtod(m, struct in_addr *)) = *p--;
1858 #ifdef DIAGNOSTIC
1859         if (ipprintfs)
1860                 kprintf(" hops %x", ntohl(mtod(m, struct in_addr *)->s_addr));
1861 #endif
1862
1863         /*
1864          * Copy option fields and padding (nop) to mbuf.
1865          */
1866         opt->ip_srcrt.nop = IPOPT_NOP;
1867         opt->ip_srcrt.srcopt[IPOPT_OFFSET] = IPOPT_MINOFF;
1868         memcpy(mtod(m, caddr_t) + sizeof(struct in_addr), &opt->ip_srcrt.nop,
1869             OPTSIZ);
1870         q = (struct in_addr *)(mtod(m, caddr_t) +
1871             sizeof(struct in_addr) + OPTSIZ);
1872 #undef OPTSIZ
1873         /*
1874          * Record return path as an IP source route,
1875          * reversing the path (pointers are now aligned).
1876          */
1877         while (p >= opt->ip_srcrt.route) {
1878 #ifdef DIAGNOSTIC
1879                 if (ipprintfs)
1880                         kprintf(" %x", ntohl(q->s_addr));
1881 #endif
1882                 *q++ = *p--;
1883         }
1884         /*
1885          * Last hop goes to final destination.
1886          */
1887         *q = opt->ip_srcrt.dst;
1888         m_tag_delete(m0, mtag);
1889 #ifdef DIAGNOSTIC
1890         if (ipprintfs)
1891                 kprintf(" %x\n", ntohl(q->s_addr));
1892 #endif
1893         return (m);
1894 }
1895
1896 /*
1897  * Strip out IP options.
1898  */
1899 void
1900 ip_stripoptions(struct mbuf *m)
1901 {
1902         int datalen;
1903         struct ip *ip = mtod(m, struct ip *);
1904         caddr_t opts;
1905         int optlen;
1906
1907         optlen = (IP_VHL_HL(ip->ip_vhl) << 2) - sizeof(struct ip);
1908         opts = (caddr_t)(ip + 1);
1909         datalen = m->m_len - (sizeof(struct ip) + optlen);
1910         bcopy(opts + optlen, opts, datalen);
1911         m->m_len -= optlen;
1912         if (m->m_flags & M_PKTHDR)
1913                 m->m_pkthdr.len -= optlen;
1914         ip->ip_vhl = IP_MAKE_VHL(IPVERSION, sizeof(struct ip) >> 2);
1915 }
1916
1917 u_char inetctlerrmap[PRC_NCMDS] = {
1918         0,              0,              0,              0,
1919         0,              EMSGSIZE,       EHOSTDOWN,      EHOSTUNREACH,
1920         EHOSTUNREACH,   EHOSTUNREACH,   ECONNREFUSED,   ECONNREFUSED,
1921         EMSGSIZE,       EHOSTUNREACH,   0,              0,
1922         0,              0,              0,              0,
1923         ENOPROTOOPT,    ECONNREFUSED
1924 };
1925
1926 /*
1927  * Forward a packet.  If some error occurs return the sender
1928  * an icmp packet.  Note we can't always generate a meaningful
1929  * icmp message because icmp doesn't have a large enough repertoire
1930  * of codes and types.
1931  *
1932  * If not forwarding, just drop the packet.  This could be confusing
1933  * if ipforwarding was zero but some routing protocol was advancing
1934  * us as a gateway to somewhere.  However, we must let the routing
1935  * protocol deal with that.
1936  *
1937  * The using_srcrt parameter indicates whether the packet is being forwarded
1938  * via a source route.
1939  */
1940 void
1941 ip_forward(struct mbuf *m, boolean_t using_srcrt, struct sockaddr_in *next_hop)
1942 {
1943         struct ip *ip = mtod(m, struct ip *);
1944         struct rtentry *rt;
1945         struct route fwd_ro;
1946         int error, type = 0, code = 0, destmtu = 0;
1947         struct mbuf *mcopy, *mtemp = NULL;
1948         n_long dest;
1949         struct in_addr pkt_dst;
1950
1951         dest = INADDR_ANY;
1952         /*
1953          * Cache the destination address of the packet; this may be
1954          * changed by use of 'ipfw fwd'.
1955          */
1956         pkt_dst = (next_hop != NULL) ? next_hop->sin_addr : ip->ip_dst;
1957
1958 #ifdef DIAGNOSTIC
1959         if (ipprintfs)
1960                 kprintf("forward: src %x dst %x ttl %x\n",
1961                        ip->ip_src.s_addr, pkt_dst.s_addr, ip->ip_ttl);
1962 #endif
1963
1964         if (m->m_flags & (M_BCAST | M_MCAST) || !in_canforward(pkt_dst)) {
1965                 ipstat.ips_cantforward++;
1966                 m_freem(m);
1967                 return;
1968         }
1969         if (!ipstealth && ip->ip_ttl <= IPTTLDEC) {
1970                 icmp_error(m, ICMP_TIMXCEED, ICMP_TIMXCEED_INTRANS, dest, 0);
1971                 return;
1972         }
1973
1974         bzero(&fwd_ro, sizeof(fwd_ro));
1975         ip_rtaddr(pkt_dst, &fwd_ro);
1976         if (fwd_ro.ro_rt == NULL) {
1977                 icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_HOST, dest, 0);
1978                 return;
1979         }
1980         rt = fwd_ro.ro_rt;
1981
1982         if (curthread->td_type == TD_TYPE_NETISR) {
1983                 /*
1984                  * Save the IP header and at most 8 bytes of the payload,
1985                  * in case we need to generate an ICMP message to the src.
1986                  */
1987                 mtemp = ipforward_mtemp[mycpuid];
1988                 KASSERT((mtemp->m_flags & M_EXT) == 0 &&
1989                     mtemp->m_data == mtemp->m_pktdat &&
1990                     m_tag_first(mtemp) == NULL,
1991                     ("ip_forward invalid mtemp1"));
1992
1993                 if (!m_dup_pkthdr(mtemp, m, MB_DONTWAIT)) {
1994                         /*
1995                          * It's probably ok if the pkthdr dup fails (because
1996                          * the deep copy of the tag chain failed), but for now
1997                          * be conservative and just discard the copy since
1998                          * code below may some day want the tags.
1999                          */
2000                         mtemp = NULL;
2001                 } else {
2002                         mtemp->m_type = m->m_type;
2003                         mtemp->m_len = imin((IP_VHL_HL(ip->ip_vhl) << 2) + 8,
2004                             (int)ip->ip_len);
2005                         mtemp->m_pkthdr.len = mtemp->m_len;
2006                         m_copydata(m, 0, mtemp->m_len, mtod(mtemp, caddr_t));
2007                 }
2008         }
2009
2010         if (!ipstealth)
2011                 ip->ip_ttl -= IPTTLDEC;
2012
2013         /*
2014          * If forwarding packet using same interface that it came in on,
2015          * perhaps should send a redirect to sender to shortcut a hop.
2016          * Only send redirect if source is sending directly to us,
2017          * and if packet was not source routed (or has any options).
2018          * Also, don't send redirect if forwarding using a default route
2019          * or a route modified by a redirect.
2020          */
2021         if (rt->rt_ifp == m->m_pkthdr.rcvif &&
2022             !(rt->rt_flags & (RTF_DYNAMIC | RTF_MODIFIED)) &&
2023             satosin(rt_key(rt))->sin_addr.s_addr != INADDR_ANY &&
2024             ipsendredirects && !using_srcrt && next_hop == NULL) {
2025                 u_long src = ntohl(ip->ip_src.s_addr);
2026                 struct in_ifaddr *rt_ifa = (struct in_ifaddr *)rt->rt_ifa;
2027
2028                 if (rt_ifa != NULL &&
2029                     (src & rt_ifa->ia_subnetmask) == rt_ifa->ia_subnet) {
2030                         if (rt->rt_flags & RTF_GATEWAY)
2031                                 dest = satosin(rt->rt_gateway)->sin_addr.s_addr;
2032                         else
2033                                 dest = pkt_dst.s_addr;
2034                         /*
2035                          * Router requirements says to only send
2036                          * host redirects.
2037                          */
2038                         type = ICMP_REDIRECT;
2039                         code = ICMP_REDIRECT_HOST;
2040 #ifdef DIAGNOSTIC
2041                         if (ipprintfs)
2042                                 kprintf("redirect (%d) to %x\n", code, dest);
2043 #endif
2044                 }
2045         }
2046
2047         error = ip_output(m, NULL, &fwd_ro, IP_FORWARDING, NULL, NULL);
2048         if (error == 0) {
2049                 ipstat.ips_forward++;
2050                 if (type == 0) {
2051                         if (mtemp)
2052                                 ipflow_create(&fwd_ro, mtemp);
2053                         goto done;
2054                 } else {
2055                         ipstat.ips_redirectsent++;
2056                 }
2057         } else {
2058                 ipstat.ips_cantforward++;
2059         }
2060
2061         if (mtemp == NULL)
2062                 goto done;
2063
2064         /*
2065          * Errors that do not require generating ICMP message
2066          */
2067         switch (error) {
2068         case ENOBUFS:
2069                 /*
2070                  * A router should not generate ICMP_SOURCEQUENCH as
2071                  * required in RFC1812 Requirements for IP Version 4 Routers.
2072                  * Source quench could be a big problem under DoS attacks,
2073                  * or if the underlying interface is rate-limited.
2074                  * Those who need source quench packets may re-enable them
2075                  * via the net.inet.ip.sendsourcequench sysctl.
2076                  */
2077                 if (!ip_sendsourcequench)
2078                         goto done;
2079                 break;
2080
2081         case EACCES:                    /* ipfw denied packet */
2082                 goto done;
2083         }
2084
2085         KASSERT((mtemp->m_flags & M_EXT) == 0 &&
2086             mtemp->m_data == mtemp->m_pktdat,
2087             ("ip_forward invalid mtemp2"));
2088         mcopy = m_copym(mtemp, 0, mtemp->m_len, MB_DONTWAIT);
2089         if (mcopy == NULL)
2090                 goto done;
2091
2092         /*
2093          * Send ICMP message.
2094          */
2095         switch (error) {
2096         case 0:                         /* forwarded, but need redirect */
2097                 /* type, code set above */
2098                 break;
2099
2100         case ENETUNREACH:               /* shouldn't happen, checked above */
2101         case EHOSTUNREACH:
2102         case ENETDOWN:
2103         case EHOSTDOWN:
2104         default:
2105                 type = ICMP_UNREACH;
2106                 code = ICMP_UNREACH_HOST;
2107                 break;
2108
2109         case EMSGSIZE:
2110                 type = ICMP_UNREACH;
2111                 code = ICMP_UNREACH_NEEDFRAG;
2112 #ifdef IPSEC
2113                 /*
2114                  * If the packet is routed over IPsec tunnel, tell the
2115                  * originator the tunnel MTU.
2116                  *      tunnel MTU = if MTU - sizeof(IP) - ESP/AH hdrsiz
2117                  * XXX quickhack!!!
2118                  */
2119                 if (fwd_ro.ro_rt != NULL) {
2120                         struct secpolicy *sp = NULL;
2121                         int ipsecerror;
2122                         int ipsechdr;
2123                         struct route *ro;
2124
2125                         sp = ipsec4_getpolicybyaddr(mcopy,
2126                                                     IPSEC_DIR_OUTBOUND,
2127                                                     IP_FORWARDING,
2128                                                     &ipsecerror);
2129
2130                         if (sp == NULL)
2131                                 destmtu = fwd_ro.ro_rt->rt_ifp->if_mtu;
2132                         else {
2133                                 /* count IPsec header size */
2134                                 ipsechdr = ipsec4_hdrsiz(mcopy,
2135                                                          IPSEC_DIR_OUTBOUND,
2136                                                          NULL);
2137
2138                                 /*
2139                                  * find the correct route for outer IPv4
2140                                  * header, compute tunnel MTU.
2141                                  *
2142                                  */
2143                                 if (sp->req != NULL && sp->req->sav != NULL &&
2144                                     sp->req->sav->sah != NULL) {
2145                                         ro = &sp->req->sav->sah->sa_route;
2146                                         if (ro->ro_rt != NULL &&
2147                                             ro->ro_rt->rt_ifp != NULL) {
2148                                                 destmtu =
2149                                                     ro->ro_rt->rt_ifp->if_mtu;
2150                                                 destmtu -= ipsechdr;
2151                                         }
2152                                 }
2153
2154                                 key_freesp(sp);
2155                         }
2156                 }
2157 #elif defined(FAST_IPSEC)
2158                 /*
2159                  * If the packet is routed over IPsec tunnel, tell the
2160                  * originator the tunnel MTU.
2161                  *      tunnel MTU = if MTU - sizeof(IP) - ESP/AH hdrsiz
2162                  * XXX quickhack!!!
2163                  */
2164                 if (fwd_ro.ro_rt != NULL) {
2165                         struct secpolicy *sp = NULL;
2166                         int ipsecerror;
2167                         int ipsechdr;
2168                         struct route *ro;
2169
2170                         sp = ipsec_getpolicybyaddr(mcopy,
2171                                                    IPSEC_DIR_OUTBOUND,
2172                                                    IP_FORWARDING,
2173                                                    &ipsecerror);
2174
2175                         if (sp == NULL)
2176                                 destmtu = fwd_ro.ro_rt->rt_ifp->if_mtu;
2177                         else {
2178                                 /* count IPsec header size */
2179                                 ipsechdr = ipsec4_hdrsiz(mcopy,
2180                                                          IPSEC_DIR_OUTBOUND,
2181                                                          NULL);
2182
2183                                 /*
2184                                  * find the correct route for outer IPv4
2185                                  * header, compute tunnel MTU.
2186                                  */
2187
2188                                 if (sp->req != NULL &&
2189                                     sp->req->sav != NULL &&
2190                                     sp->req->sav->sah != NULL) {
2191                                         ro = &sp->req->sav->sah->sa_route;
2192                                         if (ro->ro_rt != NULL &&
2193                                             ro->ro_rt->rt_ifp != NULL) {
2194                                                 destmtu =
2195                                                     ro->ro_rt->rt_ifp->if_mtu;
2196                                                 destmtu -= ipsechdr;
2197                                         }
2198                                 }
2199
2200                                 KEY_FREESP(&sp);
2201                         }
2202                 }
2203 #else /* !IPSEC && !FAST_IPSEC */
2204                 if (fwd_ro.ro_rt != NULL)
2205                         destmtu = fwd_ro.ro_rt->rt_ifp->if_mtu;
2206 #endif /*IPSEC*/
2207                 ipstat.ips_cantfrag++;
2208                 break;
2209
2210         case ENOBUFS:
2211                 type = ICMP_SOURCEQUENCH;
2212                 code = 0;
2213                 break;
2214
2215         case EACCES:                    /* ipfw denied packet */
2216                 panic("ip_forward EACCES should not reach");
2217         }
2218         icmp_error(mcopy, type, code, dest, destmtu);
2219 done:
2220         if (mtemp != NULL)
2221                 m_tag_delete_chain(mtemp);
2222         if (fwd_ro.ro_rt != NULL)
2223                 RTFREE(fwd_ro.ro_rt);
2224 }
2225
2226 void
2227 ip_savecontrol(struct inpcb *inp, struct mbuf **mp, struct ip *ip,
2228                struct mbuf *m)
2229 {
2230         if (inp->inp_socket->so_options & SO_TIMESTAMP) {
2231                 struct timeval tv;
2232
2233                 microtime(&tv);
2234                 *mp = sbcreatecontrol((caddr_t) &tv, sizeof(tv),
2235                     SCM_TIMESTAMP, SOL_SOCKET);
2236                 if (*mp)
2237                         mp = &(*mp)->m_next;
2238         }
2239         if (inp->inp_flags & INP_RECVDSTADDR) {
2240                 *mp = sbcreatecontrol((caddr_t) &ip->ip_dst,
2241                     sizeof(struct in_addr), IP_RECVDSTADDR, IPPROTO_IP);
2242                 if (*mp)
2243                         mp = &(*mp)->m_next;
2244         }
2245         if (inp->inp_flags & INP_RECVTTL) {
2246                 *mp = sbcreatecontrol((caddr_t) &ip->ip_ttl,
2247                     sizeof(u_char), IP_RECVTTL, IPPROTO_IP);
2248                 if (*mp)
2249                         mp = &(*mp)->m_next;
2250         }
2251 #ifdef notyet
2252         /* XXX
2253          * Moving these out of udp_input() made them even more broken
2254          * than they already were.
2255          */
2256         /* options were tossed already */
2257         if (inp->inp_flags & INP_RECVOPTS) {
2258                 *mp = sbcreatecontrol((caddr_t) opts_deleted_above,
2259                     sizeof(struct in_addr), IP_RECVOPTS, IPPROTO_IP);
2260                 if (*mp)
2261                         mp = &(*mp)->m_next;
2262         }
2263         /* ip_srcroute doesn't do what we want here, need to fix */
2264         if (inp->inp_flags & INP_RECVRETOPTS) {
2265                 *mp = sbcreatecontrol((caddr_t) ip_srcroute(m),
2266                     sizeof(struct in_addr), IP_RECVRETOPTS, IPPROTO_IP);
2267                 if (*mp)
2268                         mp = &(*mp)->m_next;
2269         }
2270 #endif
2271         if (inp->inp_flags & INP_RECVIF) {
2272                 struct ifnet *ifp;
2273                 struct sdlbuf {
2274                         struct sockaddr_dl sdl;
2275                         u_char  pad[32];
2276                 } sdlbuf;
2277                 struct sockaddr_dl *sdp;
2278                 struct sockaddr_dl *sdl2 = &sdlbuf.sdl;
2279
2280                 if (((ifp = m->m_pkthdr.rcvif)) &&
2281                     ((ifp->if_index != 0) && (ifp->if_index <= if_index))) {
2282                         sdp = IF_LLSOCKADDR(ifp);
2283                         /*
2284                          * Change our mind and don't try copy.
2285                          */
2286                         if ((sdp->sdl_family != AF_LINK) ||
2287                             (sdp->sdl_len > sizeof(sdlbuf))) {
2288                                 goto makedummy;
2289                         }
2290                         bcopy(sdp, sdl2, sdp->sdl_len);
2291                 } else {
2292 makedummy:
2293                         sdl2->sdl_len =
2294                             offsetof(struct sockaddr_dl, sdl_data[0]);
2295                         sdl2->sdl_family = AF_LINK;
2296                         sdl2->sdl_index = 0;
2297                         sdl2->sdl_nlen = sdl2->sdl_alen = sdl2->sdl_slen = 0;
2298                 }
2299                 *mp = sbcreatecontrol((caddr_t) sdl2, sdl2->sdl_len,
2300                         IP_RECVIF, IPPROTO_IP);
2301                 if (*mp)
2302                         mp = &(*mp)->m_next;
2303         }
2304 }
2305
2306 /*
2307  * XXX these routines are called from the upper part of the kernel.
2308  *
2309  * They could also be moved to ip_mroute.c, since all the RSVP
2310  *  handling is done there already.
2311  */
2312 int
2313 ip_rsvp_init(struct socket *so)
2314 {
2315         if (so->so_type != SOCK_RAW ||
2316             so->so_proto->pr_protocol != IPPROTO_RSVP)
2317                 return EOPNOTSUPP;
2318
2319         if (ip_rsvpd != NULL)
2320                 return EADDRINUSE;
2321
2322         ip_rsvpd = so;
2323         /*
2324          * This may seem silly, but we need to be sure we don't over-increment
2325          * the RSVP counter, in case something slips up.
2326          */
2327         if (!ip_rsvp_on) {
2328                 ip_rsvp_on = 1;
2329                 rsvp_on++;
2330         }
2331
2332         return 0;
2333 }
2334
2335 int
2336 ip_rsvp_done(void)
2337 {
2338         ip_rsvpd = NULL;
2339         /*
2340          * This may seem silly, but we need to be sure we don't over-decrement
2341          * the RSVP counter, in case something slips up.
2342          */
2343         if (ip_rsvp_on) {
2344                 ip_rsvp_on = 0;
2345                 rsvp_on--;
2346         }
2347         return 0;
2348 }
2349
2350 int
2351 rsvp_input(struct mbuf **mp, int *offp, int proto)
2352 {
2353         struct mbuf *m = *mp;
2354
2355         *mp = NULL;
2356
2357         if (rsvp_input_p) { /* call the real one if loaded */
2358                 *mp = m;
2359                 rsvp_input_p(mp, offp, proto);
2360                 return(IPPROTO_DONE);
2361         }
2362
2363         /* Can still get packets with rsvp_on = 0 if there is a local member
2364          * of the group to which the RSVP packet is addressed.  But in this
2365          * case we want to throw the packet away.
2366          */
2367
2368         if (!rsvp_on) {
2369                 m_freem(m);
2370                 return(IPPROTO_DONE);
2371         }
2372
2373         if (ip_rsvpd != NULL) {
2374                 *mp = m;
2375                 rip_input(mp, offp, proto);
2376                 return(IPPROTO_DONE);
2377         }
2378         /* Drop the packet */
2379         m_freem(m);
2380         return(IPPROTO_DONE);
2381 }