Merge branch 'vendor/LIBRESSL'

[dragonfly.git] / sys / net / ipfw / ip_fw2.c

/*
 * Copyright (c) 2002 Luigi Rizzo, Universita` di Pisa
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $FreeBSD: src/sys/netinet/ip_fw2.c,v 1.6.2.12 2003/04/08 10:42:32 maxim Exp $
 */

/*
 * Implement IP packet firewall (new version)
 */

#include "opt_ipfw.h"
#include "opt_inet.h"
#ifndef INET
#error IPFIREWALL requires INET.
#endif /* INET */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/kernel.h>
#include <sys/proc.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/sysctl.h>
#include <sys/syslog.h>
#include <sys/ucred.h>
#include <sys/in_cksum.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/tree.h>

#include <net/if.h>
#include <net/route.h>
#include <net/pfil.h>
#include <net/dummynet/ip_dummynet.h>

#include <sys/thread2.h>
#include <sys/mplock2.h>
#include <net/netmsg2.h>

#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/in_var.h>
#include <netinet/in_pcb.h>
#include <netinet/ip.h>
#include <netinet/ip_var.h>
#include <netinet/ip_icmp.h>
#include <netinet/tcp.h>
#include <netinet/tcp_seq.h>
#include <netinet/tcp_timer.h>
#include <netinet/tcp_var.h>
#include <netinet/tcpip.h>
#include <netinet/udp.h>
#include <netinet/udp_var.h>
#include <netinet/ip_divert.h>
#include <netinet/if_ether.h> /* XXX for ETHERTYPE_IP */

#include <net/ipfw/ip_fw2.h>

#ifdef IPFIREWALL_DEBUG
#define DPRINTF(fmt, ...) \
do { \
        if (fw_debug > 0) \
                kprintf(fmt, __VA_ARGS__); \
} while (0)
#else
#define DPRINTF(fmt, ...)       ((void)0)
#endif

/*
 * Description about per-CPU rule duplication:
 *
 * Module loading/unloading and all ioctl operations are serialized
 * by netisr0, so we don't have any ordering or locking problems.
 *
 * Following graph shows how operation on per-CPU rule list is
 * performed [2 CPU case]:
 *
 *   CPU0                 CPU1
 *
 * netisr0 <------------------------------------+
 *  domsg                                       |
 *    :                                         |
 *    :(delete/add...)                          |
 *    :                                         |
 *    :         netmsg                          | netmsg
 *  forwardmsg---------->netisr1                |
 *                          :                   |
 *                          :(delete/add...)    |
 *                          :                   |
 *                          :                   |
 *                        replymsg--------------+
 *
 *
 *
 * Rule structure [2 CPU case]
 *
 *    CPU0               CPU1
 *
 * layer3_chain       layer3_chain
 *     |                  |
 *     V                  V
 * +-------+ sibling  +-------+ sibling
 * | rule1 |--------->| rule1 |--------->NULL
 * +-------+          +-------+
 *     |                  |
 *     |next              |next
 *     V                  V
 * +-------+ sibling  +-------+ sibling
 * | rule2 |--------->| rule2 |--------->NULL
 * +-------+          +-------+
 *
 * ip_fw.sibling:
 * 1) Ease statistics calculation during IP_FW_GET.  We only need to
 *    iterate layer3_chain in netisr0; the current rule's duplication
 *    to the other CPUs could safely be read-only accessed through
 *    ip_fw.sibling.
 * 2) Accelerate rule insertion and deletion, e.g. rule insertion:
 *    a) In netisr0 rule3 is determined to be inserted between rule1
 *       and rule2.  To make this decision we need to iterate the
 *       layer3_chain in netisr0.  The netmsg, which is used to insert
 *       the rule, will contain rule1 in netisr0 as prev_rule and rule2
 *       in netisr0 as next_rule.
 *    b) After the insertion in netisr0 is done, we will move on to
 *       netisr1.  But instead of relocating the rule3's position in
 *       netisr1 by iterating the layer3_chain in netisr1, we set the
 *       netmsg's prev_rule to rule1->sibling and next_rule to
 *       rule2->sibling before the netmsg is forwarded to netisr1 from
 *       netisr0.
 */

/*
 * Description of states and tracks.
 *
 * Both states and tracks are stored in per-cpu RB trees instead of
 * per-cpu hash tables to avoid the worst case hash degeneration.
 *
 * The lifetimes of states and tracks are regulated by dyn_*_lifetime,
 * measured in seconds and depending on the flags.
 *
 * When a packet is received, its address fields are first masked with
 * the mask defined for the rule, then matched against the entries in
 * the per-cpu state RB tree.  States are generated by 'keep-state'
 * and 'limit' options.
 *
 * The max number of states is ipfw_state_max.  When we reach the
 * maximum number of states we do not create anymore.  This is done to
 * avoid consuming too much memory, but also too much time when
 * searching on each packet.
 *
 * Each state holds a pointer to the parent ipfw rule of the current
 * CPU so we know what action to perform.  States are removed when the
 * parent rule is deleted.  XXX we should make them survive.
 *
 * There are some limitations with states -- we do not obey the
 * 'randomized match', and we do not do multiple passes through the
 * firewall.  XXX check the latter!!!
 *
 * States grow independently on each CPU, e.g. 2 CPU case:
 *
 *        CPU0                     CPU1
 * ...................      ...................
 * :  state RB tree  :      :  state RB tree  :
 * :                 :      :                 :
 * : state1   state2 :      :      state3     :
 * :     |    |      :      :        |        :
 * :.....|....|......:      :........|........:
 *       |    |                      |
 *       |    |                      |st_rule
 *       |    |                      |
 *       V    V                      V
 *     +-------+                 +-------+
 *     | rule1 |                 | rule1 |
 *     +-------+                 +-------+
 *
 * Tracks are used to enforce limits on the number of sessions.  Tracks
 * are generated by 'limit' option.
 *
 * The max number of tracks is ipfw_track_max.  When we reach the
 * maximum number of tracks we do not create anymore.  This is done to
 * avoid consuming too much memory.
 *
 * Tracks are organized into two layers, track counter RB tree is
 * shared between CPUs, track RB tree is per-cpu.  States generated by
 * 'limit' option are linked to the track in addition to the per-cpu
 * state RB tree; mainly to ease expiration.  e.g. 2 CPU case:
 *
 *             ..............................
 *             :    track counter RB tree   :
 *             :                            :
 *             :        +-----------+       :
 *             :        |  trkcnt1  |       :
 *             :        |           |       :
 *             :      +--->counter<----+    :
 *             :      | |           |  |    :
 *             :      | +-----------+  |    :
 *             :......|................|....:
 *                    |                |
 *        CPU0        |                |         CPU1
 * .................  |t_count         |  .................
 * : track RB tree :  |                |  : track RB tree :
 * :               :  |                |  :               :
 * : +-->track1-------+                +--------track2    :
 * : |     A       :                      :               :
 * : |     |       :                      :               :
 * :.|.....|.......:                      :...............:
 *   |     +----------------+
 *   | .................... |
 *   | :   state RB tree  : |st_track
 *   | :                  : |
 *   +---state1    state2---+
 *     :     |       |    :
 *     :.....|.......|....:
 *           |       |
 *           |       |st_rule
 *           V       V
 *         +----------+
 *         |   rule1  |
 *         +----------+
 */

#define IPFW_AUTOINC_STEP_MIN   1
#define IPFW_AUTOINC_STEP_MAX   1000
#define IPFW_AUTOINC_STEP_DEF   100

#define IPFW_TABLE_MAX_DEF      64

#define IPFW_DEFAULT_RULE       65535   /* rulenum for the default rule */
#define IPFW_DEFAULT_SET        31      /* set number for the default rule */

#define MATCH_REVERSE           0
#define MATCH_FORWARD           1
#define MATCH_NONE              2
#define MATCH_UNKNOWN           3

#define TIME_LEQ(a, b)          ((a) - (b) <= 0)

#define IPFW_STATE_TCPFLAGS     (TH_SYN | TH_FIN | TH_RST)
#define IPFW_STATE_TCPSTATES    (IPFW_STATE_TCPFLAGS |  \
                                 (IPFW_STATE_TCPFLAGS << 8))

#define BOTH_SYN                (TH_SYN | (TH_SYN << 8))
#define BOTH_FIN                (TH_FIN | (TH_FIN << 8))
#define BOTH_RST                (TH_RST | (TH_RST << 8))
/* TH_ACK here means FIN was ACKed. */
#define BOTH_FINACK             (TH_ACK | (TH_ACK << 8))

#define IPFW_STATE_TCPCLOSED(s) ((s)->st_proto == IPPROTO_TCP &&        \
                                 (((s)->st_state & BOTH_RST) ||         \
                                  ((s)->st_state & BOTH_FINACK) == BOTH_FINACK))

#define O_ANCHOR                O_NOP

#define IPFW_ISXLAT(type)       ((type) == O_REDIRECT)
#define IPFW_XLAT_INVALID(s)    (IPFW_ISXLAT((s)->st_type) &&   \
                                 ((struct ipfw_xlat *)(s))->xlat_invalid)

#define IPFW_MBUF_XLATINS       FW_MBUF_PRIVATE1
#define IPFW_MBUF_XLATFWD       FW_MBUF_PRIVATE2

#define IPFW_XLATE_INSERT       0x0001
#define IPFW_XLATE_FORWARD      0x0002
#define IPFW_XLATE_OUTPUT       0x0004

struct netmsg_ipfw {
        struct netmsg_base      base;
        const struct ipfw_ioc_rule *ioc_rule;
        struct ip_fw            *next_rule;
        struct ip_fw            *prev_rule;
        struct ip_fw            *sibling;
        uint32_t                rule_flags;
        struct ip_fw            **cross_rules;
};

struct netmsg_del {
        struct netmsg_base      base;
        struct ip_fw            *start_rule;
        struct ip_fw            *prev_rule;
        uint16_t                rulenum;
        uint8_t                 from_set;
        uint8_t                 to_set;
};

struct netmsg_zent {
        struct netmsg_base      base;
        struct ip_fw            *start_rule;
        uint16_t                rulenum;
        uint16_t                log_only;
};

struct netmsg_cpstate {
        struct netmsg_base      base;
        struct ipfw_ioc_state   *ioc_state;
        int                     state_cntmax;
        int                     state_cnt;
};

struct netmsg_tblent {
        struct netmsg_base      base;
        struct sockaddr         *key;
        struct sockaddr         *netmask;
        struct ipfw_tblent      *sibling;
        int                     tableid;
};

struct netmsg_tblflush {
        struct netmsg_base      base;
        int                     tableid;
        int                     destroy;
};

struct netmsg_tblexp {
        struct netmsg_base      base;
        time_t                  expire;
        int                     tableid;
        int                     cnt;
        int                     expcnt;
        struct radix_node_head  *rnh;
};

struct ipfw_table_cp {
        struct ipfw_ioc_tblent  *te;
        int                     te_idx;
        int                     te_cnt;
};

struct ip_fw_local {
        /*
         * offset       The offset of a fragment. offset != 0 means that
         *      we have a fragment at this offset of an IPv4 packet.
         *      offset == 0 means that (if this is an IPv4 packet)
         *      this is the first or only fragment.
         */
        u_short                 offset;

        /*
         * Local copies of addresses. They are only valid if we have
         * an IP packet.
         *
         * proto        The protocol. Set to 0 for non-ip packets,
         *      or to the protocol read from the packet otherwise.
         *      proto != 0 means that we have an IPv4 packet.
         *
         * src_port, dst_port   port numbers, in HOST format. Only
         *      valid for TCP and UDP packets.
         *
         * src_ip, dst_ip       ip addresses, in NETWORK format.
         *      Only valid for IPv4 packets.
         */
        uint8_t                 proto;
        uint16_t                src_port;       /* NOTE: host format    */
        uint16_t                dst_port;       /* NOTE: host format    */
        struct in_addr          src_ip;         /* NOTE: network format */
        struct in_addr          dst_ip;         /* NOTE: network format */
        uint16_t                ip_len;
        struct tcphdr           *tcp;
};

struct ipfw_addrs {
        uint32_t                addr1;  /* host byte order */
        uint32_t                addr2;  /* host byte order */
};

struct ipfw_ports {
        uint16_t                port1;  /* host byte order */
        uint16_t                port2;  /* host byte order */
};

struct ipfw_key {
        union {
                struct ipfw_addrs addrs;
                uint64_t        value;
        } addr_u;
        union {
                struct ipfw_ports ports;
                uint32_t        value;
        } port_u;
        uint8_t                 proto;
        uint8_t                 swap;   /* IPFW_KEY_SWAP_ */
        uint16_t                rsvd2;
};

#define IPFW_KEY_SWAP_ADDRS     0x1
#define IPFW_KEY_SWAP_PORTS     0x2
#define IPFW_KEY_SWAP_ALL       (IPFW_KEY_SWAP_ADDRS | IPFW_KEY_SWAP_PORTS)

struct ipfw_trkcnt {
        RB_ENTRY(ipfw_trkcnt)   tc_rblink;
        struct ipfw_key         tc_key;
        uintptr_t               tc_ruleid;
        int                     tc_refs;
        int                     tc_count;
        time_t                  tc_expire;      /* userland get-only */
        uint16_t                tc_rulenum;     /* userland get-only */
} __cachealign;

#define tc_addrs                tc_key.addr_u.value
#define tc_ports                tc_key.port_u.value
#define tc_proto                tc_key.proto
#define tc_saddr                tc_key.addr_u.addrs.addr1
#define tc_daddr                tc_key.addr_u.addrs.addr2
#define tc_sport                tc_key.port_u.ports.port1
#define tc_dport                tc_key.port_u.ports.port2

RB_HEAD(ipfw_trkcnt_tree, ipfw_trkcnt);

struct ipfw_state;

struct ipfw_track {
        RB_ENTRY(ipfw_track)    t_rblink;
        struct ipfw_key         t_key;
        struct ip_fw            *t_rule;
        time_t                  t_lastexp;
        LIST_HEAD(, ipfw_state) t_state_list;
        time_t                  t_expire;
        volatile int            *t_count;
        struct ipfw_trkcnt      *t_trkcnt;
        TAILQ_ENTRY(ipfw_track) t_link;
};

#define t_addrs                 t_key.addr_u.value
#define t_ports                 t_key.port_u.value
#define t_proto                 t_key.proto
#define t_saddr                 t_key.addr_u.addrs.addr1
#define t_daddr                 t_key.addr_u.addrs.addr2
#define t_sport                 t_key.port_u.ports.port1
#define t_dport                 t_key.port_u.ports.port2

RB_HEAD(ipfw_track_tree, ipfw_track);
TAILQ_HEAD(ipfw_track_list, ipfw_track);

struct ipfw_state {
        RB_ENTRY(ipfw_state)    st_rblink;
        struct ipfw_key         st_key;

        time_t                  st_expire;      /* expire time */
        struct ip_fw            *st_rule;

        uint64_t                st_pcnt;        /* packets */
        uint64_t                st_bcnt;        /* bytes */

        /*
         * st_state:
         * State of this rule, typically a combination of TCP flags.
         *
         * st_ack_fwd/st_ack_rev:
         * Most recent ACKs in forward and reverse direction.  They
         * are used to generate keepalives.
         */
        uint32_t                st_state;
        uint32_t                st_ack_fwd;     /* host byte order */
        uint32_t                st_seq_fwd;     /* host byte order */
        uint32_t                st_ack_rev;     /* host byte order */
        uint32_t                st_seq_rev;     /* host byte order */

        uint16_t                st_flags;       /* IPFW_STATE_F_ */
        uint16_t                st_type;        /* KEEP_STATE/LIMIT/RDR */
        struct ipfw_track       *st_track;

        LIST_ENTRY(ipfw_state)  st_trklink;
        TAILQ_ENTRY(ipfw_state) st_link;
};

#define st_addrs                st_key.addr_u.value
#define st_ports                st_key.port_u.value
#define st_proto                st_key.proto
#define st_swap                 st_key.swap

#define IPFW_STATE_F_ACKFWD     0x0001
#define IPFW_STATE_F_SEQFWD     0x0002
#define IPFW_STATE_F_ACKREV     0x0004
#define IPFW_STATE_F_SEQREV     0x0008
#define IPFW_STATE_F_XLATSRC    0x0010
#define IPFW_STATE_F_XLATSLAVE  0x0020
#define IPFW_STATE_F_LINKED     0x0040

#define IPFW_STATE_SCANSKIP(s)  ((s)->st_type == O_ANCHOR ||    \
                                 ((s)->st_flags & IPFW_STATE_F_XLATSLAVE))

/* Expired or being deleted. */
#define IPFW_STATE_ISDEAD(s)    (TIME_LEQ((s)->st_expire, time_uptime) || \
                                 IPFW_XLAT_INVALID((s)))

TAILQ_HEAD(ipfw_state_list, ipfw_state);
RB_HEAD(ipfw_state_tree, ipfw_state);

struct ipfw_xlat {
        struct ipfw_state       xlat_st;        /* MUST be the first field */
        uint32_t                xlat_addr;      /* network byte order */
        uint16_t                xlat_port;      /* network byte order */
        uint16_t                xlat_dir;       /* MATCH_ */
        struct ifnet            *xlat_ifp;      /* matching ifnet */
        struct ipfw_xlat        *xlat_pair;     /* paired state */
        int                     xlat_pcpu;      /* paired cpu */
        volatile int            xlat_invalid;   /* invalid, but not dtor yet */
        volatile uint64_t       xlat_crefs;     /* cross references */
        struct netmsg_base      xlat_freenm;    /* for remote free */
};

#define xlat_type               xlat_st.st_type
#define xlat_flags              xlat_st.st_flags
#define xlat_rule               xlat_st.st_rule
#define xlat_bcnt               xlat_st.st_bcnt
#define xlat_pcnt               xlat_st.st_pcnt

struct ipfw_tblent {
        struct radix_node       te_nodes[2];
        struct sockaddr_in      te_key;
        u_long                  te_use;
        time_t                  te_lastuse;
        struct ipfw_tblent      *te_sibling;
        volatile int            te_expired;
};

struct ipfw_context {
        struct ip_fw            *ipfw_layer3_chain;     /* rules for layer3 */
        struct ip_fw            *ipfw_default_rule;     /* default rule */
        uint64_t                ipfw_norule_counter;    /* ipfw_log(NULL) stat*/

        /*
         * ipfw_set_disable contains one bit per set value (0..31).
         * If the bit is set, all rules with the corresponding set
         * are disabled.  Set IPDW_DEFAULT_SET is reserved for the
         * default rule and CANNOT be disabled.
         */
        uint32_t                ipfw_set_disable;

        uint8_t                 ipfw_flags;     /* IPFW_FLAG_ */

        struct ip_fw            *ipfw_cont_rule;
        struct ipfw_xlat        *ipfw_cont_xlat;

        struct ipfw_state_tree  ipfw_state_tree;
        struct ipfw_state_list  ipfw_state_list;
        int                     ipfw_state_loosecnt;
        int                     ipfw_state_cnt;

        union {
                struct ipfw_state state;
                struct ipfw_track track;
                struct ipfw_trkcnt trkcnt;
        } ipfw_tmpkey;

        struct ipfw_track_tree  ipfw_track_tree;
        struct ipfw_track_list  ipfw_track_list;
        struct ipfw_trkcnt      *ipfw_trkcnt_spare;

        struct callout          ipfw_stateto_ch;
        time_t                  ipfw_state_lastexp;
        struct netmsg_base      ipfw_stateexp_nm;
        struct netmsg_base      ipfw_stateexp_more;
        struct ipfw_state       ipfw_stateexp_anch;

        struct callout          ipfw_trackto_ch;
        time_t                  ipfw_track_lastexp;
        struct netmsg_base      ipfw_trackexp_nm;
        struct netmsg_base      ipfw_trackexp_more;
        struct ipfw_track       ipfw_trackexp_anch;

        struct callout          ipfw_keepalive_ch;
        struct netmsg_base      ipfw_keepalive_nm;
        struct netmsg_base      ipfw_keepalive_more;
        struct ipfw_state       ipfw_keepalive_anch;

        struct callout          ipfw_xlatreap_ch;
        struct netmsg_base      ipfw_xlatreap_nm;
        struct ipfw_state_list  ipfw_xlatreap;

        /*
         * Statistics
         */
        u_long                  ipfw_sts_reap;
        u_long                  ipfw_sts_reapfailed;
        u_long                  ipfw_sts_overflow;
        u_long                  ipfw_sts_nomem;
        u_long                  ipfw_sts_tcprecycled;

        u_long                  ipfw_tks_nomem;
        u_long                  ipfw_tks_reap;
        u_long                  ipfw_tks_reapfailed;
        u_long                  ipfw_tks_overflow;
        u_long                  ipfw_tks_cntnomem;

        u_long                  ipfw_frags;
        u_long                  ipfw_defraged;
        u_long                  ipfw_defrag_remote;

        u_long                  ipfw_xlated;
        u_long                  ipfw_xlate_split;
        u_long                  ipfw_xlate_conflicts;
        u_long                  ipfw_xlate_cresolved;

        /* Last field */
        struct radix_node_head  *ipfw_tables[];
};

#define IPFW_FLAG_KEEPALIVE     0x01
#define IPFW_FLAG_STATEEXP      0x02
#define IPFW_FLAG_TRACKEXP      0x04
#define IPFW_FLAG_STATEREAP     0x08
#define IPFW_FLAG_TRACKREAP     0x10

#define ipfw_state_tmpkey       ipfw_tmpkey.state
#define ipfw_track_tmpkey       ipfw_tmpkey.track
#define ipfw_trkcnt_tmpkey      ipfw_tmpkey.trkcnt

struct ipfw_global {
        int                     ipfw_state_loosecnt;    /* cache aligned */
        time_t                  ipfw_state_globexp __cachealign;

        struct lwkt_token       ipfw_trkcnt_token __cachealign;
        struct ipfw_trkcnt_tree ipfw_trkcnt_tree;
        int                     ipfw_trkcnt_cnt;
        time_t                  ipfw_track_globexp;

        /* Accessed in netisr0. */
        struct ip_fw            *ipfw_crossref_free __cachealign;
        struct callout          ipfw_crossref_ch;
        struct netmsg_base      ipfw_crossref_nm;

#ifdef KLD_MODULE
        /*
         * Module can not be unloaded, if there are references to
         * certains rules of ipfw(4), e.g. dummynet(4)
         */
        int                     ipfw_refcnt __cachealign;
#endif
} __cachealign;

static struct ipfw_context      *ipfw_ctx[MAXCPU];

MALLOC_DEFINE(M_IPFW, "IpFw/IpAcct", "IpFw/IpAcct chain's");

/*
 * Following two global variables are accessed and updated only
 * in netisr0.
 */
static uint32_t static_count;   /* # of static rules */
static uint32_t static_ioc_len; /* bytes of static rules */

/*
 * If 1, then ipfw static rules are being flushed,
 * ipfw_chk() will skip to the default rule.
 */
static int ipfw_flushing;

static int fw_verbose;
static int verbose_limit;

static int fw_debug;
static int autoinc_step = IPFW_AUTOINC_STEP_DEF;

static int      ipfw_table_max = IPFW_TABLE_MAX_DEF;

static int      ipfw_sysctl_enable(SYSCTL_HANDLER_ARGS);
static int      ipfw_sysctl_autoinc_step(SYSCTL_HANDLER_ARGS);

TUNABLE_INT("net.inet.ip.fw.table_max", &ipfw_table_max);

SYSCTL_NODE(_net_inet_ip, OID_AUTO, fw, CTLFLAG_RW, 0, "Firewall");
SYSCTL_NODE(_net_inet_ip_fw, OID_AUTO, stats, CTLFLAG_RW, 0,
    "Firewall statistics");

SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, enable, CTLTYPE_INT | CTLFLAG_RW,
    &fw_enable, 0, ipfw_sysctl_enable, "I", "Enable ipfw");
SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, autoinc_step, CTLTYPE_INT | CTLFLAG_RW,
    &autoinc_step, 0, ipfw_sysctl_autoinc_step, "I",
    "Rule number autincrement step");
SYSCTL_INT(_net_inet_ip_fw, OID_AUTO,one_pass,CTLFLAG_RW,
    &fw_one_pass, 0,
    "Only do a single pass through ipfw when using dummynet(4)");
SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, debug, CTLFLAG_RW,
    &fw_debug, 0, "Enable printing of debug ip_fw statements");
SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, verbose, CTLFLAG_RW,
    &fw_verbose, 0, "Log matches to ipfw rules");
SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, verbose_limit, CTLFLAG_RW,
    &verbose_limit, 0, "Set upper limit of matches of ipfw rules logged");
SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, table_max, CTLFLAG_RD,
    &ipfw_table_max, 0, "Max # of tables");

static int      ipfw_sysctl_dyncnt(SYSCTL_HANDLER_ARGS);
static int      ipfw_sysctl_dynmax(SYSCTL_HANDLER_ARGS);
static int      ipfw_sysctl_statecnt(SYSCTL_HANDLER_ARGS);
static int      ipfw_sysctl_statemax(SYSCTL_HANDLER_ARGS);
static int      ipfw_sysctl_scancnt(SYSCTL_HANDLER_ARGS);
static int      ipfw_sysctl_stat(SYSCTL_HANDLER_ARGS);

/*
 * Timeouts for various events in handing states.
 *
 * NOTE:
 * 1 == 0~1 second.
 * 2 == 1~2 second(s).
 *
 * We use 2 seconds for FIN lifetime, so that the states will not be
 * ripped prematurely.
 */
static uint32_t dyn_ack_lifetime = 300;
static uint32_t dyn_syn_lifetime = 20;
static uint32_t dyn_finwait_lifetime = 20;
static uint32_t dyn_fin_lifetime = 2;
static uint32_t dyn_rst_lifetime = 2;
static uint32_t dyn_udp_lifetime = 10;
static uint32_t dyn_short_lifetime = 5; /* used by tracks too */

/*
 * Keepalives are sent if dyn_keepalive is set. They are sent every
 * dyn_keepalive_period seconds, in the last dyn_keepalive_interval
 * seconds of lifetime of a rule.
 */
static uint32_t dyn_keepalive_interval = 20;
static uint32_t dyn_keepalive_period = 5;
static uint32_t dyn_keepalive = 1;      /* do send keepalives */

static struct ipfw_global       ipfw_gd;
static int      ipfw_state_loosecnt_updthr;
static int      ipfw_state_max = 4096;  /* max # of states */
static int      ipfw_track_max = 4096;  /* max # of tracks */

static int      ipfw_state_headroom;    /* setup at module load time */
static int      ipfw_state_reap_min = 8;
static int      ipfw_state_expire_max = 32;
static int      ipfw_state_scan_max = 256;
static int      ipfw_keepalive_max = 8;
static int      ipfw_track_reap_max = 4;
static int      ipfw_track_expire_max = 16;
static int      ipfw_track_scan_max = 128;

static eventhandler_tag ipfw_ifaddr_event;

/* Compat */
SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, dyn_count,
    CTLTYPE_INT | CTLFLAG_RD, NULL, 0, ipfw_sysctl_dyncnt, "I",
    "Number of states and tracks");
SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, dyn_max,
    CTLTYPE_INT | CTLFLAG_RW, NULL, 0, ipfw_sysctl_dynmax, "I",
    "Max number of states and tracks");

SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, state_cnt,
    CTLTYPE_INT | CTLFLAG_RD, NULL, 0, ipfw_sysctl_statecnt, "I",
    "Number of states");
SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, state_max,
    CTLTYPE_INT | CTLFLAG_RW, NULL, 0, ipfw_sysctl_statemax, "I",
    "Max number of states");
SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, state_headroom, CTLFLAG_RW,
    &ipfw_state_headroom, 0, "headroom for state reap");
SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, track_cnt, CTLFLAG_RD,
    &ipfw_gd.ipfw_trkcnt_cnt, 0, "Number of tracks");
SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, track_max, CTLFLAG_RW,
    &ipfw_track_max, 0, "Max number of tracks");
SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, static_count, CTLFLAG_RD,
    &static_count, 0, "Number of static rules");
SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, dyn_ack_lifetime, CTLFLAG_RW,
    &dyn_ack_lifetime, 0, "Lifetime of dyn. rules for acks");
SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, dyn_syn_lifetime, CTLFLAG_RW,
    &dyn_syn_lifetime, 0, "Lifetime of dyn. rules for syn");
SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, dyn_fin_lifetime, CTLFLAG_RW,
    &dyn_fin_lifetime, 0, "Lifetime of dyn. rules for fin");
SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, dyn_finwait_lifetime, CTLFLAG_RW,
    &dyn_finwait_lifetime, 0, "Lifetime of dyn. rules for fin wait");
SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, dyn_rst_lifetime, CTLFLAG_RW,
    &dyn_rst_lifetime, 0, "Lifetime of dyn. rules for rst");
SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, dyn_udp_lifetime, CTLFLAG_RW,
    &dyn_udp_lifetime, 0, "Lifetime of dyn. rules for UDP");
SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, dyn_short_lifetime, CTLFLAG_RW,
    &dyn_short_lifetime, 0, "Lifetime of dyn. rules for other situations");
SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, dyn_keepalive, CTLFLAG_RW,
    &dyn_keepalive, 0, "Enable keepalives for dyn. rules");
SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, state_scan_max,
    CTLTYPE_INT | CTLFLAG_RW, &ipfw_state_scan_max, 0, ipfw_sysctl_scancnt,
    "I", "# of states to scan for each expire iteration");
SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, state_expire_max,
    CTLTYPE_INT | CTLFLAG_RW, &ipfw_state_expire_max, 0, ipfw_sysctl_scancnt,
    "I", "# of states to expire for each expire iteration");
SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, keepalive_max,
    CTLTYPE_INT | CTLFLAG_RW, &ipfw_keepalive_max, 0, ipfw_sysctl_scancnt,
    "I", "# of states to expire for each expire iteration");
SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, state_reap_min,
    CTLTYPE_INT | CTLFLAG_RW, &ipfw_state_reap_min, 0, ipfw_sysctl_scancnt,
    "I", "# of states to reap for state shortage");
SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, track_scan_max,
    CTLTYPE_INT | CTLFLAG_RW, &ipfw_track_scan_max, 0, ipfw_sysctl_scancnt,
    "I", "# of tracks to scan for each expire iteration");
SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, track_expire_max,
    CTLTYPE_INT | CTLFLAG_RW, &ipfw_track_expire_max, 0, ipfw_sysctl_scancnt,
    "I", "# of tracks to expire for each expire iteration");
SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, track_reap_max,
    CTLTYPE_INT | CTLFLAG_RW, &ipfw_track_reap_max, 0, ipfw_sysctl_scancnt,
    "I", "# of tracks to reap for track shortage");

SYSCTL_PROC(_net_inet_ip_fw_stats, OID_AUTO, state_reap,
    CTLTYPE_ULONG | CTLFLAG_RW, NULL,
    __offsetof(struct ipfw_context, ipfw_sts_reap), ipfw_sysctl_stat,
    "LU", "# of state reaps due to states shortage");
SYSCTL_PROC(_net_inet_ip_fw_stats, OID_AUTO, state_reapfailed,
    CTLTYPE_ULONG | CTLFLAG_RW, NULL,
    __offsetof(struct ipfw_context, ipfw_sts_reapfailed), ipfw_sysctl_stat,
    "LU", "# of state reap failure");
SYSCTL_PROC(_net_inet_ip_fw_stats, OID_AUTO, state_overflow,
    CTLTYPE_ULONG | CTLFLAG_RW, NULL,
    __offsetof(struct ipfw_context, ipfw_sts_overflow), ipfw_sysctl_stat,
    "LU", "# of state overflow");
SYSCTL_PROC(_net_inet_ip_fw_stats, OID_AUTO, state_nomem,
    CTLTYPE_ULONG | CTLFLAG_RW, NULL,
    __offsetof(struct ipfw_context, ipfw_sts_nomem), ipfw_sysctl_stat,
    "LU", "# of state allocation failure");
SYSCTL_PROC(_net_inet_ip_fw_stats, OID_AUTO, state_tcprecycled,
    CTLTYPE_ULONG | CTLFLAG_RW, NULL,
    __offsetof(struct ipfw_context, ipfw_sts_tcprecycled), ipfw_sysctl_stat,
    "LU", "# of state deleted due to fast TCP port recycling");

SYSCTL_PROC(_net_inet_ip_fw_stats, OID_AUTO, track_nomem,
    CTLTYPE_ULONG | CTLFLAG_RW, NULL,
    __offsetof(struct ipfw_context, ipfw_tks_nomem), ipfw_sysctl_stat,
    "LU", "# of track allocation failure");
SYSCTL_PROC(_net_inet_ip_fw_stats, OID_AUTO, track_reap,
    CTLTYPE_ULONG | CTLFLAG_RW, NULL,
    __offsetof(struct ipfw_context, ipfw_tks_reap), ipfw_sysctl_stat,
    "LU", "# of track reap due to tracks shortage");
SYSCTL_PROC(_net_inet_ip_fw_stats, OID_AUTO, track_reapfailed,
    CTLTYPE_ULONG | CTLFLAG_RW, NULL,
    __offsetof(struct ipfw_context, ipfw_tks_reapfailed), ipfw_sysctl_stat,
    "LU", "# of track reap failure");
SYSCTL_PROC(_net_inet_ip_fw_stats, OID_AUTO, track_overflow,
    CTLTYPE_ULONG | CTLFLAG_RW, NULL,
    __offsetof(struct ipfw_context, ipfw_tks_overflow), ipfw_sysctl_stat,
    "LU", "# of track overflow");
SYSCTL_PROC(_net_inet_ip_fw_stats, OID_AUTO, track_cntnomem,
    CTLTYPE_ULONG | CTLFLAG_RW, NULL,
    __offsetof(struct ipfw_context, ipfw_tks_cntnomem), ipfw_sysctl_stat,
    "LU", "# of track counter allocation failure");
SYSCTL_PROC(_net_inet_ip_fw_stats, OID_AUTO, frags,
    CTLTYPE_ULONG | CTLFLAG_RW, NULL,
    __offsetof(struct ipfw_context, ipfw_frags), ipfw_sysctl_stat,
    "LU", "# of IP fragements defraged");
SYSCTL_PROC(_net_inet_ip_fw_stats, OID_AUTO, defraged,
    CTLTYPE_ULONG | CTLFLAG_RW, NULL,
    __offsetof(struct ipfw_context, ipfw_defraged), ipfw_sysctl_stat,
    "LU", "# of IP packets after defrag");
SYSCTL_PROC(_net_inet_ip_fw_stats, OID_AUTO, defrag_remote,
    CTLTYPE_ULONG | CTLFLAG_RW, NULL,
    __offsetof(struct ipfw_context, ipfw_defrag_remote), ipfw_sysctl_stat,
    "LU", "# of IP packets after defrag dispatched to remote cpus");
SYSCTL_PROC(_net_inet_ip_fw_stats, OID_AUTO, xlated,
    CTLTYPE_ULONG | CTLFLAG_RW, NULL,
    __offsetof(struct ipfw_context, ipfw_xlated), ipfw_sysctl_stat,
    "LU", "# address/port translations");
SYSCTL_PROC(_net_inet_ip_fw_stats, OID_AUTO, xlate_split,
    CTLTYPE_ULONG | CTLFLAG_RW, NULL,
    __offsetof(struct ipfw_context, ipfw_xlate_split), ipfw_sysctl_stat,
    "LU", "# address/port translations split between different cpus");
SYSCTL_PROC(_net_inet_ip_fw_stats, OID_AUTO, xlate_conflicts,
    CTLTYPE_ULONG | CTLFLAG_RW, NULL,
    __offsetof(struct ipfw_context, ipfw_xlate_conflicts), ipfw_sysctl_stat,
    "LU", "# address/port translations conflicts on remote cpu");
SYSCTL_PROC(_net_inet_ip_fw_stats, OID_AUTO, xlate_cresolved,
    CTLTYPE_ULONG | CTLFLAG_RW, NULL,
    __offsetof(struct ipfw_context, ipfw_xlate_cresolved), ipfw_sysctl_stat,
    "LU", "# address/port translations conflicts resolved on remote cpu");

static int              ipfw_state_cmp(struct ipfw_state *,
                            struct ipfw_state *);
static int              ipfw_trkcnt_cmp(struct ipfw_trkcnt *,
                            struct ipfw_trkcnt *);
static int              ipfw_track_cmp(struct ipfw_track *,
                            struct ipfw_track *);

RB_PROTOTYPE(ipfw_state_tree, ipfw_state, st_rblink, ipfw_state_cmp);
RB_GENERATE(ipfw_state_tree, ipfw_state, st_rblink, ipfw_state_cmp);

RB_PROTOTYPE(ipfw_trkcnt_tree, ipfw_trkcnt, tc_rblink, ipfw_trkcnt_cmp);
RB_GENERATE(ipfw_trkcnt_tree, ipfw_trkcnt, tc_rblink, ipfw_trkcnt_cmp);

RB_PROTOTYPE(ipfw_track_tree, ipfw_track, t_rblink, ipfw_track_cmp);
RB_GENERATE(ipfw_track_tree, ipfw_track, t_rblink, ipfw_track_cmp);

static int              ipfw_chk(struct ip_fw_args *);
static void             ipfw_track_expire_ipifunc(void *);
static void             ipfw_state_expire_ipifunc(void *);
static void             ipfw_keepalive(void *);
static int              ipfw_state_expire_start(struct ipfw_context *,
                            int, int);
static void             ipfw_crossref_timeo(void *);
static void             ipfw_state_remove(struct ipfw_context *,
                            struct ipfw_state *);
static void             ipfw_xlat_reap_timeo(void *);
static void             ipfw_defrag_redispatch(struct mbuf *, int,
                            struct ip_fw *);

#define IPFW_TRKCNT_TOKGET      lwkt_gettoken(&ipfw_gd.ipfw_trkcnt_token)
#define IPFW_TRKCNT_TOKREL      lwkt_reltoken(&ipfw_gd.ipfw_trkcnt_token)
#define IPFW_TRKCNT_TOKINIT     \
        lwkt_token_init(&ipfw_gd.ipfw_trkcnt_token, "ipfw_trkcnt");

static void
sa_maskedcopy(const struct sockaddr *src, struct sockaddr *dst,
    const struct sockaddr *netmask)
{
        const u_char *cp1 = (const u_char *)src;
        u_char *cp2 = (u_char *)dst;
        const u_char *cp3 = (const u_char *)netmask;
        u_char *cplim = cp2 + *cp3;
        u_char *cplim2 = cp2 + *cp1;

        *cp2++ = *cp1++; *cp2++ = *cp1++; /* copies sa_len & sa_family */
        cp3 += 2;
        if (cplim > cplim2)
                cplim = cplim2;
        while (cp2 < cplim)
                *cp2++ = *cp1++ & *cp3++;
        if (cp2 < cplim2)
                bzero(cp2, cplim2 - cp2);
}

static __inline uint16_t
pfil_cksum_fixup(uint16_t cksum, uint16_t old, uint16_t new, uint8_t udp)
{
        uint32_t l;

        if (udp && !cksum)
                return (0x0000);
        l = cksum + old - new;
        l = (l >> 16) + (l & 65535);
        l = l & 65535;
        if (udp && !l)
                return (0xFFFF);
        return (l);
}

static __inline void
ipfw_key_build(struct ipfw_key *key, in_addr_t saddr, uint16_t sport,
    in_addr_t daddr, uint16_t dport, uint8_t proto)
{

        key->proto = proto;
        key->swap = 0;

        if (saddr < daddr) {
                key->addr_u.addrs.addr1 = daddr;
                key->addr_u.addrs.addr2 = saddr;
                key->swap |= IPFW_KEY_SWAP_ADDRS;
        } else {
                key->addr_u.addrs.addr1 = saddr;
                key->addr_u.addrs.addr2 = daddr;
        }

        if (sport < dport) {
                key->port_u.ports.port1 = dport;
                key->port_u.ports.port2 = sport;
                key->swap |= IPFW_KEY_SWAP_PORTS;
        } else {
                key->port_u.ports.port1 = sport;
                key->port_u.ports.port2 = dport;
        }

        if (sport == dport && (key->swap & IPFW_KEY_SWAP_ADDRS))
                key->swap |= IPFW_KEY_SWAP_PORTS;
        if (saddr == daddr && (key->swap & IPFW_KEY_SWAP_PORTS))
                key->swap |= IPFW_KEY_SWAP_ADDRS;
}

static __inline void
ipfw_key_4tuple(const struct ipfw_key *key, in_addr_t *saddr, uint16_t *sport,
    in_addr_t *daddr, uint16_t *dport)
{

        if (key->swap & IPFW_KEY_SWAP_ADDRS) {
                *saddr = key->addr_u.addrs.addr2;
                *daddr = key->addr_u.addrs.addr1;
        } else {
                *saddr = key->addr_u.addrs.addr1;
                *daddr = key->addr_u.addrs.addr2;
        }

        if (key->swap & IPFW_KEY_SWAP_PORTS) {
                *sport = key->port_u.ports.port2;
                *dport = key->port_u.ports.port1;
        } else {
                *sport = key->port_u.ports.port1;
                *dport = key->port_u.ports.port2;
        }
}

static int
ipfw_state_cmp(struct ipfw_state *s1, struct ipfw_state *s2)
{

        if (s1->st_proto > s2->st_proto)
                return (1);
        if (s1->st_proto < s2->st_proto)
                return (-1);

        if (s1->st_addrs > s2->st_addrs)
                return (1);
        if (s1->st_addrs < s2->st_addrs)
                return (-1);

        if (s1->st_ports > s2->st_ports)
                return (1);
        if (s1->st_ports < s2->st_ports)
                return (-1);

        if (s1->st_swap == s2->st_swap ||
            (s1->st_swap ^ s2->st_swap) == IPFW_KEY_SWAP_ALL)
                return (0);

        if (s1->st_swap > s2->st_swap)
                return (1);
        else
                return (-1);
}

static int
ipfw_trkcnt_cmp(struct ipfw_trkcnt *t1, struct ipfw_trkcnt *t2)
{

        if (t1->tc_proto > t2->tc_proto)
                return (1);
        if (t1->tc_proto < t2->tc_proto)
                return (-1);

        if (t1->tc_addrs > t2->tc_addrs)
                return (1);
        if (t1->tc_addrs < t2->tc_addrs)
                return (-1);

        if (t1->tc_ports > t2->tc_ports)
                return (1);
        if (t1->tc_ports < t2->tc_ports)
                return (-1);

        if (t1->tc_ruleid > t2->tc_ruleid)
                return (1);
        if (t1->tc_ruleid < t2->tc_ruleid)
                return (-1);

        return (0);
}

static int
ipfw_track_cmp(struct ipfw_track *t1, struct ipfw_track *t2)
{

        if (t1->t_proto > t2->t_proto)
                return (1);
        if (t1->t_proto < t2->t_proto)
                return (-1);

        if (t1->t_addrs > t2->t_addrs)
                return (1);
        if (t1->t_addrs < t2->t_addrs)
                return (-1);

        if (t1->t_ports > t2->t_ports)
                return (1);
        if (t1->t_ports < t2->t_ports)
                return (-1);

        if ((uintptr_t)t1->t_rule > (uintptr_t)t2->t_rule)
                return (1);
        if ((uintptr_t)t1->t_rule < (uintptr_t)t2->t_rule)
                return (-1);

        return (0);
}

static __inline struct ipfw_state *
ipfw_state_link(struct ipfw_context *ctx, struct ipfw_state *s)
{
        struct ipfw_state *dup;

        KASSERT((s->st_flags & IPFW_STATE_F_LINKED) == 0,
            ("state %p was linked", s));
        dup = RB_INSERT(ipfw_state_tree, &ctx->ipfw_state_tree, s);
        if (dup == NULL) {
                TAILQ_INSERT_TAIL(&ctx->ipfw_state_list, s, st_link);
                s->st_flags |= IPFW_STATE_F_LINKED;
        }
        return (dup);
}

static __inline void
ipfw_state_unlink(struct ipfw_context *ctx, struct ipfw_state *s)
{

        KASSERT(s->st_flags & IPFW_STATE_F_LINKED,
            ("state %p was not linked", s));
        RB_REMOVE(ipfw_state_tree, &ctx->ipfw_state_tree, s);
        TAILQ_REMOVE(&ctx->ipfw_state_list, s, st_link);
        s->st_flags &= ~IPFW_STATE_F_LINKED;
}

static void
ipfw_state_max_set(int state_max)
{

        ipfw_state_max = state_max;
        /* Allow 5% states over-allocation. */
        ipfw_state_loosecnt_updthr = (state_max / 20) / netisr_ncpus;
}

static __inline int
ipfw_state_cntcoll(void)
{
        int cpu, state_cnt = 0;

        for (cpu = 0; cpu < netisr_ncpus; ++cpu)
                state_cnt += ipfw_ctx[cpu]->ipfw_state_cnt;
        return (state_cnt);
}

static __inline int
ipfw_state_cntsync(void)
{
        int state_cnt;

        state_cnt = ipfw_state_cntcoll();
        ipfw_gd.ipfw_state_loosecnt = state_cnt;
        return (state_cnt);
}

static __inline int
ipfw_free_rule(struct ip_fw *rule)
{
        KASSERT(rule->cpuid == mycpuid, ("rule freed on cpu%d", mycpuid));
        KASSERT(rule->refcnt > 0, ("invalid refcnt %u", rule->refcnt));
        rule->refcnt--;
        if (rule->refcnt == 0) {
                if (rule->cross_rules != NULL)
                        kfree(rule->cross_rules, M_IPFW);
                kfree(rule, M_IPFW);
                return 1;
        }
        return 0;
}

static void
ipfw_unref_rule(void *priv)
{
        ipfw_free_rule(priv);
#ifdef KLD_MODULE
        KASSERT(ipfw_gd.ipfw_refcnt > 0,
            ("invalid ipfw_refcnt %d", ipfw_gd.ipfw_refcnt));
        atomic_subtract_int(&ipfw_gd.ipfw_refcnt, 1);
#endif
}

static __inline void
ipfw_ref_rule(struct ip_fw *rule)
{
        KASSERT(rule->cpuid == mycpuid, ("rule used on cpu%d", mycpuid));
#ifdef KLD_MODULE
        atomic_add_int(&ipfw_gd.ipfw_refcnt, 1);
#endif
        rule->refcnt++;
}

/*
 * This macro maps an ip pointer into a layer3 header pointer of type T
 */
#define L3HDR(T, ip) ((T *)((uint32_t *)(ip) + (ip)->ip_hl))

static __inline int
icmptype_match(struct ip *ip, ipfw_insn_u32 *cmd)
{
        int type = L3HDR(struct icmp,ip)->icmp_type;
        int idx_max = F_LEN(&cmd->o) - F_INSN_SIZE(ipfw_insn);
        int idx = type / 32;

        if (idx >= idx_max)
                return (0);
        return (cmd->d[idx] & (1 << (type % 32)));
}

static __inline int
icmpcode_match(struct ip *ip, ipfw_insn_u32 *cmd)
{
        int code = L3HDR(struct icmp,ip)->icmp_code;
        int idx_max = F_LEN(&cmd->o) - F_INSN_SIZE(ipfw_insn);
        int idx = code / 32;

        if (idx >= idx_max)
                return (0);
        return (cmd->d[idx] & (1 << (code % 32)));
}

#define TT      ((1 << ICMP_ECHO) | \
                 (1 << ICMP_ROUTERSOLICIT) | \
                 (1 << ICMP_TSTAMP) | \
                 (1 << ICMP_IREQ) | \
                 (1 << ICMP_MASKREQ))

static int
is_icmp_query(struct ip *ip)
{
        int type = L3HDR(struct icmp, ip)->icmp_type;

        return (type < 32 && (TT & (1 << type)));
}

#undef TT

/*
 * The following checks use two arrays of 8 or 16 bits to store the
 * bits that we want set or clear, respectively. They are in the
 * low and high half of cmd->arg1 or cmd->d[0].
 *
 * We scan options and store the bits we find set. We succeed if
 *
 *      (want_set & ~bits) == 0 && (want_clear & ~bits) == want_clear
 *
 * The code is sometimes optimized not to store additional variables.
 */
static int
flags_match(ipfw_insn *cmd, uint8_t bits)
{
        u_char want_clear;
        bits = ~bits;

        if (((cmd->arg1 & 0xff) & bits) != 0)
                return 0; /* some bits we want set were clear */

        want_clear = (cmd->arg1 >> 8) & 0xff;
        if ((want_clear & bits) != want_clear)
                return 0; /* some bits we want clear were set */
        return 1;
}

static int
ipopts_match(struct ip *ip, ipfw_insn *cmd)
{
        int optlen, bits = 0;
        u_char *cp = (u_char *)(ip + 1);
        int x = (ip->ip_hl << 2) - sizeof(struct ip);

        for (; x > 0; x -= optlen, cp += optlen) {
                int opt = cp[IPOPT_OPTVAL];

                if (opt == IPOPT_EOL)
                        break;

                if (opt == IPOPT_NOP) {
                        optlen = 1;
                } else {
                        optlen = cp[IPOPT_OLEN];
                        if (optlen <= 0 || optlen > x)
                                return 0; /* invalid or truncated */
                }

                switch (opt) {
                case IPOPT_LSRR:
                        bits |= IP_FW_IPOPT_LSRR;
                        break;

                case IPOPT_SSRR:
                        bits |= IP_FW_IPOPT_SSRR;
                        break;

                case IPOPT_RR:
                        bits |= IP_FW_IPOPT_RR;
                        break;

                case IPOPT_TS:
                        bits |= IP_FW_IPOPT_TS;
                        break;

                default:
                        break;
                }
        }
        return (flags_match(cmd, bits));
}

static int
tcpopts_match(struct ip *ip, ipfw_insn *cmd)
{
        int optlen, bits = 0;
        struct tcphdr *tcp = L3HDR(struct tcphdr,ip);
        u_char *cp = (u_char *)(tcp + 1);
        int x = (tcp->th_off << 2) - sizeof(struct tcphdr);

        for (; x > 0; x -= optlen, cp += optlen) {
                int opt = cp[0];

                if (opt == TCPOPT_EOL)
                        break;

                if (opt == TCPOPT_NOP) {
                        optlen = 1;
                } else {
                        optlen = cp[1];
                        if (optlen <= 0)
                                break;
                }

                switch (opt) {
                case TCPOPT_MAXSEG:
                        bits |= IP_FW_TCPOPT_MSS;
                        break;

                case TCPOPT_WINDOW:
                        bits |= IP_FW_TCPOPT_WINDOW;
                        break;

                case TCPOPT_SACK_PERMITTED:
                case TCPOPT_SACK:
                        bits |= IP_FW_TCPOPT_SACK;
                        break;

                case TCPOPT_TIMESTAMP:
                        bits |= IP_FW_TCPOPT_TS;
                        break;

                case TCPOPT_CC:
                case TCPOPT_CCNEW:
                case TCPOPT_CCECHO:
                        bits |= IP_FW_TCPOPT_CC;
                        break;

                default:
                        break;
                }
        }
        return (flags_match(cmd, bits));
}

static int
iface_match(struct ifnet *ifp, ipfw_insn_if *cmd)
{
        if (ifp == NULL)        /* no iface with this packet, match fails */
                return 0;

        /* Check by name or by IP address */
        if (cmd->name[0] != '\0') { /* match by name */
                /* Check name */
                if (cmd->p.glob) {
                        if (kfnmatch(cmd->name, ifp->if_xname, 0) == 0)
                                return(1);
                } else {
                        if (strncmp(ifp->if_xname, cmd->name, IFNAMSIZ) == 0)
                                return(1);
                }
        } else {
                struct ifaddr_container *ifac;

                TAILQ_FOREACH(ifac, &ifp->if_addrheads[mycpuid], ifa_link) {
                        struct ifaddr *ia = ifac->ifa;

                        if (ia->ifa_addr == NULL)
                                continue;
                        if (ia->ifa_addr->sa_family != AF_INET)
                                continue;
                        if (cmd->p.ip.s_addr == ((struct sockaddr_in *)
                            (ia->ifa_addr))->sin_addr.s_addr)
                                return(1);      /* match */
                }
        }
        return(0);      /* no match, fail ... */
}

#define SNPARGS(buf, len) buf + len, sizeof(buf) > len ? sizeof(buf) - len : 0

/*
 * We enter here when we have a rule with O_LOG.
 * XXX this function alone takes about 2Kbytes of code!
 */
static void
ipfw_log(struct ipfw_context *ctx, struct ip_fw *f, u_int hlen,
    struct ether_header *eh, struct mbuf *m, struct ifnet *oif)
{
        char *action;
        int limit_reached = 0;
        char action2[40], proto[48], fragment[28], abuf[INET_ADDRSTRLEN];

        fragment[0] = '\0';
        proto[0] = '\0';

        if (f == NULL) {        /* bogus pkt */
                if (verbose_limit != 0 &&
                    ctx->ipfw_norule_counter >= verbose_limit)
                        return;
                ctx->ipfw_norule_counter++;
                if (ctx->ipfw_norule_counter == verbose_limit)
                        limit_reached = verbose_limit;
                action = "Refuse";
        } else {        /* O_LOG is the first action, find the real one */
                ipfw_insn *cmd = ACTION_PTR(f);
                ipfw_insn_log *l = (ipfw_insn_log *)cmd;

                if (l->max_log != 0 && l->log_left == 0)
                        return;
                l->log_left--;
                if (l->log_left == 0)
                        limit_reached = l->max_log;
                cmd += F_LEN(cmd);      /* point to first action */
                if (cmd->opcode == O_PROB)
                        cmd += F_LEN(cmd);

                action = action2;
                switch (cmd->opcode) {
                case O_DENY:
                        action = "Deny";
                        break;

                case O_REJECT:
                        if (cmd->arg1==ICMP_REJECT_RST) {
                                action = "Reset";
                        } else if (cmd->arg1==ICMP_UNREACH_HOST) {
                                action = "Reject";
                        } else {
                                ksnprintf(SNPARGS(action2, 0), "Unreach %d",
                                          cmd->arg1);
                        }
                        break;

                case O_ACCEPT:
                        action = "Accept";
                        break;

                case O_COUNT:
                        action = "Count";
                        break;

                case O_DIVERT:
                        ksnprintf(SNPARGS(action2, 0), "Divert %d", cmd->arg1);
                        break;

                case O_TEE:
                        ksnprintf(SNPARGS(action2, 0), "Tee %d", cmd->arg1);
                        break;

                case O_SKIPTO:
                        ksnprintf(SNPARGS(action2, 0), "SkipTo %d", cmd->arg1);
                        break;

                case O_PIPE:
                        ksnprintf(SNPARGS(action2, 0), "Pipe %d", cmd->arg1);
                        break;

                case O_QUEUE:
                        ksnprintf(SNPARGS(action2, 0), "Queue %d", cmd->arg1);
                        break;

                case O_FORWARD_IP:
                        {
                                ipfw_insn_sa *sa = (ipfw_insn_sa *)cmd;
                                int len;

                                len = ksnprintf(SNPARGS(action2, 0),
                                    "Forward to %s",
                                    kinet_ntoa(sa->sa.sin_addr, abuf));
                                if (sa->sa.sin_port) {
                                        ksnprintf(SNPARGS(action2, len), ":%d",
                                                  sa->sa.sin_port);
                                }
                        }
                        break;

                default:
                        action = "UNKNOWN";
                        break;
                }
        }

        if (hlen == 0) {        /* non-ip */
                ksnprintf(SNPARGS(proto, 0), "MAC");
        } else {
                struct ip *ip = mtod(m, struct ip *);
                /* these three are all aliases to the same thing */
                struct icmp *const icmp = L3HDR(struct icmp, ip);
                struct tcphdr *const tcp = (struct tcphdr *)icmp;
                struct udphdr *const udp = (struct udphdr *)icmp;

                int ip_off, offset, ip_len;
                int len;

                if (eh != NULL) { /* layer 2 packets are as on the wire */
                        ip_off = ntohs(ip->ip_off);
                        ip_len = ntohs(ip->ip_len);
                } else {
                        ip_off = ip->ip_off;
                        ip_len = ip->ip_len;
                }
                offset = ip_off & IP_OFFMASK;
                switch (ip->ip_p) {
                case IPPROTO_TCP:
                        len = ksnprintf(SNPARGS(proto, 0), "TCP %s",
                                        kinet_ntoa(ip->ip_src, abuf));
                        if (offset == 0) {
                                ksnprintf(SNPARGS(proto, len), ":%d %s:%d",
                                          ntohs(tcp->th_sport),
                                          kinet_ntoa(ip->ip_dst, abuf),
                                          ntohs(tcp->th_dport));
                        } else {
                                ksnprintf(SNPARGS(proto, len), " %s",
                                          kinet_ntoa(ip->ip_dst, abuf));
                        }
                        break;

                case IPPROTO_UDP:
                        len = ksnprintf(SNPARGS(proto, 0), "UDP %s",
                                        kinet_ntoa(ip->ip_src, abuf));
                        if (offset == 0) {
                                ksnprintf(SNPARGS(proto, len), ":%d %s:%d",
                                          ntohs(udp->uh_sport),
                                          kinet_ntoa(ip->ip_dst, abuf),
                                          ntohs(udp->uh_dport));
                        } else {
                                ksnprintf(SNPARGS(proto, len), " %s",
                                          kinet_ntoa(ip->ip_dst, abuf));
                        }
                        break;

                case IPPROTO_ICMP:
                        if (offset == 0) {
                                len = ksnprintf(SNPARGS(proto, 0),
                                                "ICMP:%u.%u ",
                                                icmp->icmp_type,
                                                icmp->icmp_code);
                        } else {
                                len = ksnprintf(SNPARGS(proto, 0), "ICMP ");
                        }
                        len += ksnprintf(SNPARGS(proto, len), "%s",
                                         kinet_ntoa(ip->ip_src, abuf));
                        ksnprintf(SNPARGS(proto, len), " %s",
                                  kinet_ntoa(ip->ip_dst, abuf));
                        break;

                default:
                        len = ksnprintf(SNPARGS(proto, 0), "P:%d %s", ip->ip_p,
                                        kinet_ntoa(ip->ip_src, abuf));
                        ksnprintf(SNPARGS(proto, len), " %s",
                                  kinet_ntoa(ip->ip_dst, abuf));
                        break;
                }

                if (ip_off & (IP_MF | IP_OFFMASK)) {
                        ksnprintf(SNPARGS(fragment, 0), " (frag %d:%d@%d%s)",
                                  ntohs(ip->ip_id), ip_len - (ip->ip_hl << 2),
                                  offset << 3, (ip_off & IP_MF) ? "+" : "");
                }
        }

        if (oif || m->m_pkthdr.rcvif) {
                log(LOG_SECURITY | LOG_INFO,
                    "ipfw: %d %s %s %s via %s%s\n",
                    f ? f->rulenum : -1,
                    action, proto, oif ? "out" : "in",
                    oif ? oif->if_xname : m->m_pkthdr.rcvif->if_xname,
                    fragment);
        } else {
                log(LOG_SECURITY | LOG_INFO,
                    "ipfw: %d %s %s [no if info]%s\n",
                    f ? f->rulenum : -1,
                    action, proto, fragment);
        }

        if (limit_reached) {
                log(LOG_SECURITY | LOG_NOTICE,
                    "ipfw: limit %d reached on entry %d\n",
                    limit_reached, f ? f->rulenum : -1);
        }
}

#undef SNPARGS

static void
ipfw_xlat_reap(struct ipfw_xlat *x, struct ipfw_xlat *slave_x)
{
        struct ip_fw *rule = slave_x->xlat_rule;

        KKASSERT(rule->cpuid == mycpuid);

        /* No more cross references; free this pair now. */
        kfree(x, M_IPFW);
        kfree(slave_x, M_IPFW);

        /* See the comment in ipfw_ip_xlate_dispatch(). */
        rule->cross_refs--;
}

static void
ipfw_xlat_reap_dispatch(netmsg_t nm)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct ipfw_state *s, *ns;

        ASSERT_NETISR_NCPUS(mycpuid);

        crit_enter();
        /* Reply ASAP. */
        netisr_replymsg(&ctx->ipfw_xlatreap_nm, 0);
        crit_exit();

        /* TODO: limit scanning depth */
        TAILQ_FOREACH_MUTABLE(s, &ctx->ipfw_xlatreap, st_link, ns) {
                struct ipfw_xlat *x = (struct ipfw_xlat *)s;
                struct ipfw_xlat *slave_x = x->xlat_pair;
                uint64_t crefs;

                crefs = slave_x->xlat_crefs + x->xlat_crefs;
                if (crefs == 0) {
                        TAILQ_REMOVE(&ctx->ipfw_xlatreap, &x->xlat_st, st_link);
                        ipfw_xlat_reap(x, slave_x);
                }
        }
        if (!TAILQ_EMPTY(&ctx->ipfw_xlatreap)) {
                callout_reset(&ctx->ipfw_xlatreap_ch, 2, ipfw_xlat_reap_timeo,
                    &ctx->ipfw_xlatreap_nm);
        }
}

static void
ipfw_xlat_reap_timeo(void *xnm)
{
        struct netmsg_base *nm = xnm;

        KKASSERT(mycpuid < netisr_ncpus);

        crit_enter();
        if (nm->lmsg.ms_flags & MSGF_DONE)
                netisr_sendmsg_oncpu(nm);
        crit_exit();
}

static void
ipfw_xlat_free_dispatch(netmsg_t nmsg)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct ipfw_xlat *x = nmsg->lmsg.u.ms_resultp;
        struct ipfw_xlat *slave_x = x->xlat_pair;
        uint64_t crefs;

        ASSERT_NETISR_NCPUS(mycpuid);

        KKASSERT(slave_x != NULL);
        KKASSERT(slave_x->xlat_invalid && x->xlat_invalid);

        KASSERT((x->xlat_flags & IPFW_STATE_F_LINKED) == 0,
            ("master xlat is still linked"));
        if (slave_x->xlat_flags & IPFW_STATE_F_LINKED)
                ipfw_state_unlink(ctx, &slave_x->xlat_st);

        /* See the comment in ipfw_ip_xlate_dispatch(). */
        slave_x->xlat_crefs--;

        crefs = slave_x->xlat_crefs + x->xlat_crefs;
        if (crefs == 0) {
                ipfw_xlat_reap(x, slave_x);
                return;
        }

        if (TAILQ_EMPTY(&ctx->ipfw_xlatreap)) {
                callout_reset(&ctx->ipfw_xlatreap_ch, 2, ipfw_xlat_reap_timeo,
                    &ctx->ipfw_xlatreap_nm);
        }

        /*
         * This pair is still referenced; defer its destruction.
         * YYY reuse st_link.
         */
        TAILQ_INSERT_TAIL(&ctx->ipfw_xlatreap, &x->xlat_st, st_link);
}

static __inline void
ipfw_xlat_invalidate(struct ipfw_xlat *x)
{

        x->xlat_invalid = 1;
        x->xlat_pair->xlat_invalid = 1;
}

static void
ipfw_state_del(struct ipfw_context *ctx, struct ipfw_state *s)
{
        struct ipfw_xlat *x, *slave_x;
        struct netmsg_base *nm;

        KASSERT(s->st_type == O_KEEP_STATE || s->st_type == O_LIMIT ||
            IPFW_ISXLAT(s->st_type), ("invalid state type %u", s->st_type));
        KASSERT((s->st_flags & IPFW_STATE_F_XLATSLAVE) == 0,
            ("delete slave xlat"));

        KASSERT(ctx->ipfw_state_cnt > 0,
            ("invalid state count %d", ctx->ipfw_state_cnt));
        ctx->ipfw_state_cnt--;
        if (ctx->ipfw_state_loosecnt > 0)
                ctx->ipfw_state_loosecnt--;

        /*
         * Unhook this state.
         */
        if (s->st_track != NULL) {
                struct ipfw_track *t = s->st_track;

                KASSERT(!LIST_EMPTY(&t->t_state_list),
                    ("track state list is empty"));
                LIST_REMOVE(s, st_trklink);

                KASSERT(*t->t_count > 0,
                    ("invalid track count %d", *t->t_count));
                atomic_subtract_int(t->t_count, 1);
        }
        ipfw_state_unlink(ctx, s);

        /*
         * Free this state.  Xlat requires special processing,
         * since xlat are paired state and they could be on
         * different cpus.
         */

        if (!IPFW_ISXLAT(s->st_type)) {
                /* Not xlat; free now. */
                kfree(s, M_IPFW);
                /* Done! */
                return;
        }
        x = (struct ipfw_xlat *)s;

        if (x->xlat_pair == NULL) {
                /* Not setup yet; free now. */
                kfree(x, M_IPFW);
                /* Done! */
                return;
        }
        slave_x = x->xlat_pair;
        KKASSERT(slave_x->xlat_flags & IPFW_STATE_F_XLATSLAVE);

        if (x->xlat_pcpu == mycpuid) {
                /*
                 * Paired states are on the same cpu; delete this
                 * pair now.
                 */
                KKASSERT(x->xlat_crefs == 0);
                KKASSERT(slave_x->xlat_crefs == 0);
                if (slave_x->xlat_flags & IPFW_STATE_F_LINKED)
                        ipfw_state_unlink(ctx, &slave_x->xlat_st);
                kfree(x, M_IPFW);
                kfree(slave_x, M_IPFW);
                return;
        }

        /*
         * Free the paired states on the cpu owning the slave xlat.
         */

        /* 
         * Mark the state pair invalid; completely deleting them
         * may take some time.
         */
        ipfw_xlat_invalidate(x);

        nm = &x->xlat_freenm;
        netmsg_init(nm, NULL, &netisr_apanic_rport, MSGF_PRIORITY,
            ipfw_xlat_free_dispatch);
        nm->lmsg.u.ms_resultp = x;

        /* See the comment in ipfw_xlate_redispatch(). */
        x->xlat_rule->cross_refs++;
        x->xlat_crefs++;

        netisr_sendmsg(nm, x->xlat_pcpu);
}

static void
ipfw_state_remove(struct ipfw_context *ctx, struct ipfw_state *s)
{

        if (s->st_flags & IPFW_STATE_F_XLATSLAVE) {
                KKASSERT(IPFW_ISXLAT(s->st_type));
                ipfw_xlat_invalidate((struct ipfw_xlat *)s);
                ipfw_state_unlink(ctx, s);
                return;
        }
        ipfw_state_del(ctx, s);
}

static int
ipfw_state_reap(struct ipfw_context *ctx, int reap_max)
{
        struct ipfw_state *s, *anchor;
        int expired;

        if (reap_max < ipfw_state_reap_min)
                reap_max = ipfw_state_reap_min;

        if ((ctx->ipfw_flags & IPFW_FLAG_STATEEXP) == 0) {
                /*
                 * Kick start state expiring.  Ignore scan limit,
                 * we are short of states.
                 */
                ctx->ipfw_flags |= IPFW_FLAG_STATEREAP;
                expired = ipfw_state_expire_start(ctx, INT_MAX, reap_max);
                ctx->ipfw_flags &= ~IPFW_FLAG_STATEREAP;
                return (expired);
        }

        /*
         * States are being expired.
         */

        if (ctx->ipfw_state_cnt == 0)
                return (0);

        expired = 0;
        anchor = &ctx->ipfw_stateexp_anch;
        while ((s = TAILQ_NEXT(anchor, st_link)) != NULL) {
                /*
                 * Ignore scan limit; we are short of states.
                 */

                TAILQ_REMOVE(&ctx->ipfw_state_list, anchor, st_link);
                TAILQ_INSERT_AFTER(&ctx->ipfw_state_list, s, anchor, st_link);

                if (IPFW_STATE_SCANSKIP(s))
                        continue;

                if (IPFW_STATE_ISDEAD(s) || IPFW_STATE_TCPCLOSED(s)) {
                        ipfw_state_del(ctx, s);
                        if (++expired >= reap_max)
                                break;
                        if ((expired & 0xff) == 0 && 
                            ipfw_state_cntcoll() + ipfw_state_headroom <=
                            ipfw_state_max)
                                break;
                }
        }
        /*
         * NOTE:
         * Leave the anchor on the list, even if the end of the list has
         * been reached.  ipfw_state_expire_more_dispatch() will handle
         * the removal.
         */
        return (expired);
}

static void
ipfw_state_flush(struct ipfw_context *ctx, const struct ip_fw *rule)
{
        struct ipfw_state *s, *sn;

        TAILQ_FOREACH_MUTABLE(s, &ctx->ipfw_state_list, st_link, sn) {
                if (IPFW_STATE_SCANSKIP(s))
                        continue;
                if (rule != NULL && s->st_rule != rule)
                        continue;
                ipfw_state_del(ctx, s);
        }
}

static void
ipfw_state_expire_done(struct ipfw_context *ctx)
{

        KASSERT(ctx->ipfw_flags & IPFW_FLAG_STATEEXP,
            ("stateexp is not in progress"));
        ctx->ipfw_flags &= ~IPFW_FLAG_STATEEXP;
        callout_reset(&ctx->ipfw_stateto_ch, hz,
            ipfw_state_expire_ipifunc, NULL);
}

static void
ipfw_state_expire_more(struct ipfw_context *ctx)
{
        struct netmsg_base *nm = &ctx->ipfw_stateexp_more;

        KASSERT(ctx->ipfw_flags & IPFW_FLAG_STATEEXP,
            ("stateexp is not in progress"));
        KASSERT(nm->lmsg.ms_flags & MSGF_DONE,
            ("stateexp more did not finish"));
        netisr_sendmsg_oncpu(nm);
}

static int
ipfw_state_expire_loop(struct ipfw_context *ctx, struct ipfw_state *anchor,
    int scan_max, int expire_max)
{
        struct ipfw_state *s;
        int scanned = 0, expired = 0;

        KASSERT(ctx->ipfw_flags & IPFW_FLAG_STATEEXP,
            ("stateexp is not in progress"));

        while ((s = TAILQ_NEXT(anchor, st_link)) != NULL) {
                if (scanned++ >= scan_max) {
                        ipfw_state_expire_more(ctx);
                        return (expired);
                }

                TAILQ_REMOVE(&ctx->ipfw_state_list, anchor, st_link);
                TAILQ_INSERT_AFTER(&ctx->ipfw_state_list, s, anchor, st_link);

                if (IPFW_STATE_SCANSKIP(s))
                        continue;

                if (IPFW_STATE_ISDEAD(s) ||
                    ((ctx->ipfw_flags & IPFW_FLAG_STATEREAP) &&
                     IPFW_STATE_TCPCLOSED(s))) {
                        ipfw_state_del(ctx, s);
                        if (++expired >= expire_max) {
                                ipfw_state_expire_more(ctx);
                                return (expired);
                        }
                        if ((ctx->ipfw_flags & IPFW_FLAG_STATEREAP) &&
                            (expired & 0xff) == 0 &&
                            ipfw_state_cntcoll() + ipfw_state_headroom <=
                            ipfw_state_max) {
                                ipfw_state_expire_more(ctx);
                                return (expired);
                        }
                }
        }
        TAILQ_REMOVE(&ctx->ipfw_state_list, anchor, st_link);
        ipfw_state_expire_done(ctx);
        return (expired);
}

static void
ipfw_state_expire_more_dispatch(netmsg_t nm)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct ipfw_state *anchor;

        ASSERT_NETISR_NCPUS(mycpuid);
        KASSERT(ctx->ipfw_flags & IPFW_FLAG_STATEEXP,
            ("statexp is not in progress"));

        /* Reply ASAP */
        netisr_replymsg(&nm->base, 0);

        anchor = &ctx->ipfw_stateexp_anch;
        if (ctx->ipfw_state_cnt == 0) {
                TAILQ_REMOVE(&ctx->ipfw_state_list, anchor, st_link);
                ipfw_state_expire_done(ctx);
                return;
        }
        ipfw_state_expire_loop(ctx, anchor,
            ipfw_state_scan_max, ipfw_state_expire_max);
}

static int
ipfw_state_expire_start(struct ipfw_context *ctx, int scan_max, int expire_max)
{
        struct ipfw_state *anchor;

        KASSERT((ctx->ipfw_flags & IPFW_FLAG_STATEEXP) == 0,
            ("stateexp is in progress"));
        ctx->ipfw_flags |= IPFW_FLAG_STATEEXP;

        if (ctx->ipfw_state_cnt == 0) {
                ipfw_state_expire_done(ctx);
                return (0);
        }

        /*
         * Do not expire more than once per second, it is useless.
         */
        if ((ctx->ipfw_flags & IPFW_FLAG_STATEREAP) == 0 &&
            ctx->ipfw_state_lastexp == time_uptime) {
                ipfw_state_expire_done(ctx);
                return (0);
        }
        ctx->ipfw_state_lastexp = time_uptime;

        anchor = &ctx->ipfw_stateexp_anch;
        TAILQ_INSERT_HEAD(&ctx->ipfw_state_list, anchor, st_link);
        return (ipfw_state_expire_loop(ctx, anchor, scan_max, expire_max));
}

static void
ipfw_state_expire_dispatch(netmsg_t nm)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];

        ASSERT_NETISR_NCPUS(mycpuid);

        /* Reply ASAP */
        crit_enter();
        netisr_replymsg(&nm->base, 0);
        crit_exit();

        if (ctx->ipfw_flags & IPFW_FLAG_STATEEXP) {
                /* Running; done. */
                return;
        }
        ipfw_state_expire_start(ctx,
            ipfw_state_scan_max, ipfw_state_expire_max);
}

static void
ipfw_state_expire_ipifunc(void *dummy __unused)
{
        struct netmsg_base *msg;

        KKASSERT(mycpuid < netisr_ncpus);
        msg = &ipfw_ctx[mycpuid]->ipfw_stateexp_nm;

        crit_enter();
        if (msg->lmsg.ms_flags & MSGF_DONE)
                netisr_sendmsg_oncpu(msg);
        crit_exit();
}

static boolean_t
ipfw_state_update_tcp(struct ipfw_state *s, int dir, const struct tcphdr *tcp)
{
        uint32_t seq = ntohl(tcp->th_seq);
        uint32_t ack = ntohl(tcp->th_ack);

        if (tcp->th_flags & TH_RST)
                return (TRUE);

        if (dir == MATCH_FORWARD) {
                if ((s->st_flags & IPFW_STATE_F_SEQFWD) == 0) {
                        s->st_flags |= IPFW_STATE_F_SEQFWD;
                        s->st_seq_fwd = seq;
                } else if (SEQ_GEQ(seq, s->st_seq_fwd)) {
                        s->st_seq_fwd = seq;
                } else {
                        /* Out-of-sequence; done. */
                        return (FALSE);
                }
                if (tcp->th_flags & TH_ACK) {
                        if ((s->st_flags & IPFW_STATE_F_ACKFWD) == 0) {
                                s->st_flags |= IPFW_STATE_F_ACKFWD;
                                s->st_ack_fwd = ack;
                        } else if (SEQ_GEQ(ack, s->st_ack_fwd)) {
                                s->st_ack_fwd = ack;
                        } else {
                                /* Out-of-sequence; done. */
                                return (FALSE);
                        }

                        if ((s->st_state & ((TH_FIN | TH_ACK) << 8)) ==
                            (TH_FIN << 8) && s->st_ack_fwd == s->st_seq_rev + 1)
                                s->st_state |= (TH_ACK << 8);
                }
        } else {
                if ((s->st_flags & IPFW_STATE_F_SEQREV) == 0) {
                        s->st_flags |= IPFW_STATE_F_SEQREV;
                        s->st_seq_rev = seq;
                } else if (SEQ_GEQ(seq, s->st_seq_rev)) {
                        s->st_seq_rev = seq;
                } else {
                        /* Out-of-sequence; done. */
                        return (FALSE);
                }
                if (tcp->th_flags & TH_ACK) {
                        if ((s->st_flags & IPFW_STATE_F_ACKREV) == 0) {
                                s->st_flags |= IPFW_STATE_F_ACKREV;
                                s->st_ack_rev= ack;
                        } else if (SEQ_GEQ(ack, s->st_ack_rev)) {
                                s->st_ack_rev = ack;
                        } else {
                                /* Out-of-sequence; done. */
                                return (FALSE);
                        }

                        if ((s->st_state & (TH_FIN | TH_ACK)) == TH_FIN &&
                            s->st_ack_rev == s->st_seq_fwd + 1)
                                s->st_state |= TH_ACK;
                }
        }
        return (TRUE);
}

static void
ipfw_state_update(const struct ipfw_flow_id *pkt, int dir,
    const struct tcphdr *tcp, struct ipfw_state *s)
{

        if (pkt->proto == IPPROTO_TCP) { /* update state according to flags */
                u_char flags = pkt->flags & IPFW_STATE_TCPFLAGS;

                if (tcp != NULL && !ipfw_state_update_tcp(s, dir, tcp))
                        return;

                s->st_state |= (dir == MATCH_FORWARD) ? flags : (flags << 8);
                switch (s->st_state & IPFW_STATE_TCPSTATES) {
                case TH_SYN:                            /* opening */
                        s->st_expire = time_uptime + dyn_syn_lifetime;
                        break;

                case BOTH_SYN:                  /* move to established */
                case BOTH_SYN | TH_FIN:         /* one side tries to close */
                case BOTH_SYN | (TH_FIN << 8):
                        s->st_expire = time_uptime + dyn_ack_lifetime;
                        break;

                case BOTH_SYN | BOTH_FIN:       /* both sides closed */
                        if ((s->st_state & BOTH_FINACK) == BOTH_FINACK) {
                                /* And both FINs were ACKed. */
                                s->st_expire = time_uptime + dyn_fin_lifetime;
                        } else {
                                s->st_expire = time_uptime +
                                    dyn_finwait_lifetime;
                        }
                        break;

                default:
#if 0
                        /*
                         * reset or some invalid combination, but can also
                         * occur if we use keep-state the wrong way.
                         */
                        if ((s->st_state & ((TH_RST << 8) | TH_RST)) == 0)
                                kprintf("invalid state: 0x%x\n", s->st_state);
#endif
                        s->st_expire = time_uptime + dyn_rst_lifetime;
                        break;
                }
        } else if (pkt->proto == IPPROTO_UDP) {
                s->st_expire = time_uptime + dyn_udp_lifetime;
        } else {
                /* other protocols */
                s->st_expire = time_uptime + dyn_short_lifetime;
        }
}

/*
 * Lookup a state.
 */
static struct ipfw_state *
ipfw_state_lookup(struct ipfw_context *ctx, const struct ipfw_flow_id *pkt,
    int *match_direction, const struct tcphdr *tcp)
{
        struct ipfw_state *key, *s;
        int dir = MATCH_NONE;

        key = &ctx->ipfw_state_tmpkey;
        ipfw_key_build(&key->st_key, pkt->src_ip, pkt->src_port,
            pkt->dst_ip, pkt->dst_port, pkt->proto);
        s = RB_FIND(ipfw_state_tree, &ctx->ipfw_state_tree, key);
        if (s == NULL)
                goto done; /* not found. */
        if (IPFW_STATE_ISDEAD(s)) {
                ipfw_state_remove(ctx, s);
                s = NULL;
                goto done;
        }
        if ((pkt->flags & TH_SYN) && IPFW_STATE_TCPCLOSED(s)) {
                /* TCP ports recycling is too fast. */
                ctx->ipfw_sts_tcprecycled++;
                ipfw_state_remove(ctx, s);
                s = NULL;
                goto done;
        }

        if (s->st_swap == key->st_swap) {
                dir = MATCH_FORWARD;
        } else {
                KASSERT((s->st_swap & key->st_swap) == 0,
                    ("found mismatch state"));
                dir = MATCH_REVERSE;
        }

        /* Update this state. */
        ipfw_state_update(pkt, dir, tcp, s);

        if (s->st_track != NULL) {
                /* This track has been used. */
                s->st_track->t_expire = time_uptime + dyn_short_lifetime;
        }
done:
        if (match_direction)
                *match_direction = dir;
        return (s);
}

static struct ipfw_state *
ipfw_state_alloc(struct ipfw_context *ctx, const struct ipfw_flow_id *id,
    uint16_t type, struct ip_fw *rule, const struct tcphdr *tcp)
{
        struct ipfw_state *s;
        size_t sz;

        KASSERT(type == O_KEEP_STATE || type == O_LIMIT || IPFW_ISXLAT(type),
            ("invalid state type %u", type));

        sz = sizeof(struct ipfw_state);
        if (IPFW_ISXLAT(type))
                sz = sizeof(struct ipfw_xlat);

        s = kmalloc(sz, M_IPFW, M_INTWAIT | M_NULLOK | M_ZERO);
        if (s == NULL) {
                ctx->ipfw_sts_nomem++;
                return (NULL);
        }

        ipfw_key_build(&s->st_key, id->src_ip, id->src_port,
            id->dst_ip, id->dst_port, id->proto);

        s->st_rule = rule;
        s->st_type = type;
        if (IPFW_ISXLAT(type)) {
                struct ipfw_xlat *x = (struct ipfw_xlat *)s;

                x->xlat_dir = MATCH_NONE;
                x->xlat_pcpu = -1;
        }

        /*
         * Update this state:
         * Set st_expire and st_state.
         */
        ipfw_state_update(id, MATCH_FORWARD, tcp, s);

        return (s);
}

static struct ipfw_state *
ipfw_state_add(struct ipfw_context *ctx, const struct ipfw_flow_id *id,
    uint16_t type, struct ip_fw *rule, struct ipfw_track *t,
    const struct tcphdr *tcp)
{
        struct ipfw_state *s, *dup;

        s = ipfw_state_alloc(ctx, id, type, rule, tcp);
        if (s == NULL)
                return (NULL);

        ctx->ipfw_state_cnt++;
        ctx->ipfw_state_loosecnt++;
        if (ctx->ipfw_state_loosecnt >= ipfw_state_loosecnt_updthr) {
                ipfw_gd.ipfw_state_loosecnt += ctx->ipfw_state_loosecnt;
                ctx->ipfw_state_loosecnt = 0;
        }

        dup = ipfw_state_link(ctx, s);
        if (dup != NULL)
                panic("ipfw: %u state exists %p", type, dup);

        if (t != NULL) {
                /* Keep the track referenced. */
                LIST_INSERT_HEAD(&t->t_state_list, s, st_trklink);
                s->st_track = t;
        }
        return (s);
}

static boolean_t
ipfw_track_free(struct ipfw_context *ctx, struct ipfw_track *t)
{
        struct ipfw_trkcnt *trk;
        boolean_t trk_freed = FALSE;

        KASSERT(t->t_count != NULL, ("track anchor"));
        KASSERT(LIST_EMPTY(&t->t_state_list),
            ("invalid track is still referenced"));

        trk = t->t_trkcnt;
        KASSERT(trk != NULL, ("track has no trkcnt"));

        RB_REMOVE(ipfw_track_tree, &ctx->ipfw_track_tree, t);
        TAILQ_REMOVE(&ctx->ipfw_track_list, t, t_link);
        kfree(t, M_IPFW);

        /*
         * fdrop() style reference counting.
         * See kern/kern_descrip.c fdrop().
         */
        for (;;) {
                int refs = trk->tc_refs;

                cpu_ccfence();
                KASSERT(refs > 0, ("invalid trkcnt refs %d", refs));
                if (refs == 1) {
                        IPFW_TRKCNT_TOKGET;
                        if (atomic_cmpset_int(&trk->tc_refs, refs, 0)) {
                                KASSERT(trk->tc_count == 0,
                                    ("%d states reference this trkcnt",
                                     trk->tc_count));
                                RB_REMOVE(ipfw_trkcnt_tree,
                                    &ipfw_gd.ipfw_trkcnt_tree, trk);

                                KASSERT(ipfw_gd.ipfw_trkcnt_cnt > 0,
                                    ("invalid trkcnt cnt %d",
                                     ipfw_gd.ipfw_trkcnt_cnt));
                                ipfw_gd.ipfw_trkcnt_cnt--;
                                IPFW_TRKCNT_TOKREL;

                                if (ctx->ipfw_trkcnt_spare == NULL)
                                        ctx->ipfw_trkcnt_spare = trk;
                                else
                                        kfree(trk, M_IPFW);
                                trk_freed = TRUE;
                                break; /* done! */
                        }
                        IPFW_TRKCNT_TOKREL;
                        /* retry */
                } else if (atomic_cmpset_int(&trk->tc_refs, refs, refs - 1)) {
                        break; /* done! */
                }
                /* retry */
        }
        return (trk_freed);
}

static void
ipfw_track_flush(struct ipfw_context *ctx, struct ip_fw *rule)
{
        struct ipfw_track *t, *tn;

        TAILQ_FOREACH_MUTABLE(t, &ctx->ipfw_track_list, t_link, tn) {
                if (t->t_count == NULL) /* anchor */
                        continue;
                if (rule != NULL && t->t_rule != rule)
                        continue;
                ipfw_track_free(ctx, t);
        }
}

static boolean_t
ipfw_track_state_expire(struct ipfw_context *ctx, struct ipfw_track *t,
    boolean_t reap)
{
        struct ipfw_state *s, *sn;
        boolean_t ret = FALSE;

        KASSERT(t->t_count != NULL, ("track anchor"));

        if (LIST_EMPTY(&t->t_state_list))
                return (FALSE);

        /*
         * Do not expire more than once per second, it is useless.
         */
        if (t->t_lastexp == time_uptime)
                return (FALSE);
        t->t_lastexp = time_uptime;

        LIST_FOREACH_MUTABLE(s, &t->t_state_list, st_trklink, sn) {
                if (IPFW_STATE_ISDEAD(s) || (reap && IPFW_STATE_TCPCLOSED(s))) {
                        KASSERT(s->st_track == t,
                            ("state track %p does not match %p",
                             s->st_track, t));
                        ipfw_state_del(ctx, s);
                        ret = TRUE;
                }
        }
        return (ret);
}

static __inline struct ipfw_trkcnt *
ipfw_trkcnt_alloc(struct ipfw_context *ctx)
{
        struct ipfw_trkcnt *trk;

        if (ctx->ipfw_trkcnt_spare != NULL) {
                trk = ctx->ipfw_trkcnt_spare;
                ctx->ipfw_trkcnt_spare = NULL;
        } else {
                trk = kmalloc_cachealign(sizeof(*trk), M_IPFW,
                    M_INTWAIT | M_NULLOK);
        }
        return (trk);
}

static void
ipfw_track_expire_done(struct ipfw_context *ctx)
{

        KASSERT(ctx->ipfw_flags & IPFW_FLAG_TRACKEXP,
            ("trackexp is not in progress"));
        ctx->ipfw_flags &= ~IPFW_FLAG_TRACKEXP;
        callout_reset(&ctx->ipfw_trackto_ch, hz,
            ipfw_track_expire_ipifunc, NULL);
}

static void
ipfw_track_expire_more(struct ipfw_context *ctx)
{
        struct netmsg_base *nm = &ctx->ipfw_trackexp_more;

        KASSERT(ctx->ipfw_flags & IPFW_FLAG_TRACKEXP,
            ("trackexp is not in progress"));
        KASSERT(nm->lmsg.ms_flags & MSGF_DONE,
            ("trackexp more did not finish"));
        netisr_sendmsg_oncpu(nm);
}

static int
ipfw_track_expire_loop(struct ipfw_context *ctx, struct ipfw_track *anchor,
    int scan_max, int expire_max)
{
        struct ipfw_track *t;
        int scanned = 0, expired = 0;
        boolean_t reap = FALSE;

        KASSERT(ctx->ipfw_flags & IPFW_FLAG_TRACKEXP,
            ("trackexp is not in progress"));

        if (ctx->ipfw_flags & IPFW_FLAG_TRACKREAP)
                reap = TRUE;

        while ((t = TAILQ_NEXT(anchor, t_link)) != NULL) {
                if (scanned++ >= scan_max) {
                        ipfw_track_expire_more(ctx);
                        return (expired);
                }

                TAILQ_REMOVE(&ctx->ipfw_track_list, anchor, t_link);
                TAILQ_INSERT_AFTER(&ctx->ipfw_track_list, t, anchor, t_link);

                if (t->t_count == NULL) /* anchor */
                        continue;

                ipfw_track_state_expire(ctx, t, reap);
                if (!LIST_EMPTY(&t->t_state_list)) {
                        /* There are states referencing this track. */
                        continue;
                }

                if (TIME_LEQ(t->t_expire, time_uptime) || reap) {
                        /* Expired. */
                        if (ipfw_track_free(ctx, t)) {
                                if (++expired >= expire_max) {
                                        ipfw_track_expire_more(ctx);
                                        return (expired);
                                }
                        }
                }
        }
        TAILQ_REMOVE(&ctx->ipfw_track_list, anchor, t_link);
        ipfw_track_expire_done(ctx);
        return (expired);
}

static int
ipfw_track_expire_start(struct ipfw_context *ctx, int scan_max, int expire_max)
{
        struct ipfw_track *anchor;

        KASSERT((ctx->ipfw_flags & IPFW_FLAG_TRACKEXP) == 0,
            ("trackexp is in progress"));
        ctx->ipfw_flags |= IPFW_FLAG_TRACKEXP;

        if (RB_EMPTY(&ctx->ipfw_track_tree)) {
                ipfw_track_expire_done(ctx);
                return (0);
        }

        /*
         * Do not expire more than once per second, it is useless.
         */
        if ((ctx->ipfw_flags & IPFW_FLAG_TRACKREAP) == 0 &&
            ctx->ipfw_track_lastexp == time_uptime) {
                ipfw_track_expire_done(ctx);
                return (0);
        }
        ctx->ipfw_track_lastexp = time_uptime;

        anchor = &ctx->ipfw_trackexp_anch;
        TAILQ_INSERT_HEAD(&ctx->ipfw_track_list, anchor, t_link);
        return (ipfw_track_expire_loop(ctx, anchor, scan_max, expire_max));
}

static void
ipfw_track_expire_more_dispatch(netmsg_t nm)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct ipfw_track *anchor;

        ASSERT_NETISR_NCPUS(mycpuid);
        KASSERT(ctx->ipfw_flags & IPFW_FLAG_TRACKEXP,
            ("trackexp is not in progress"));

        /* Reply ASAP */
        netisr_replymsg(&nm->base, 0);

        anchor = &ctx->ipfw_trackexp_anch;
        if (RB_EMPTY(&ctx->ipfw_track_tree)) {
                TAILQ_REMOVE(&ctx->ipfw_track_list, anchor, t_link);
                ipfw_track_expire_done(ctx);
                return;
        }
        ipfw_track_expire_loop(ctx, anchor,
            ipfw_track_scan_max, ipfw_track_expire_max);
}

static void
ipfw_track_expire_dispatch(netmsg_t nm)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];

        ASSERT_NETISR_NCPUS(mycpuid);

        /* Reply ASAP */
        crit_enter();
        netisr_replymsg(&nm->base, 0);
        crit_exit();

        if (ctx->ipfw_flags & IPFW_FLAG_TRACKEXP) {
                /* Running; done. */
                return;
        }
        ipfw_track_expire_start(ctx,
            ipfw_track_scan_max, ipfw_track_expire_max);
}

static void
ipfw_track_expire_ipifunc(void *dummy __unused)
{
        struct netmsg_base *msg;

        KKASSERT(mycpuid < netisr_ncpus);
        msg = &ipfw_ctx[mycpuid]->ipfw_trackexp_nm;

        crit_enter();
        if (msg->lmsg.ms_flags & MSGF_DONE)
                netisr_sendmsg_oncpu(msg);
        crit_exit();
}

static int
ipfw_track_reap(struct ipfw_context *ctx)
{
        struct ipfw_track *t, *anchor;
        int expired;

        if ((ctx->ipfw_flags & IPFW_FLAG_TRACKEXP) == 0) {
                /*
                 * Kick start track expiring.  Ignore scan limit,
                 * we are short of tracks.
                 */
                ctx->ipfw_flags |= IPFW_FLAG_TRACKREAP;
                expired = ipfw_track_expire_start(ctx, INT_MAX,
                    ipfw_track_reap_max);
                ctx->ipfw_flags &= ~IPFW_FLAG_TRACKREAP;
                return (expired);
        }

        /*
         * Tracks are being expired.
         */

        if (RB_EMPTY(&ctx->ipfw_track_tree))
                return (0);

        expired = 0;
        anchor = &ctx->ipfw_trackexp_anch;
        while ((t = TAILQ_NEXT(anchor, t_link)) != NULL) {
                /*
                 * Ignore scan limit; we are short of tracks.
                 */

                TAILQ_REMOVE(&ctx->ipfw_track_list, anchor, t_link);
                TAILQ_INSERT_AFTER(&ctx->ipfw_track_list, t, anchor, t_link);

                if (t->t_count == NULL) /* anchor */
                        continue;

                ipfw_track_state_expire(ctx, t, TRUE);
                if (!LIST_EMPTY(&t->t_state_list)) {
                        /* There are states referencing this track. */
                        continue;
                }

                if (ipfw_track_free(ctx, t)) {
                        if (++expired >= ipfw_track_reap_max) {
                                ipfw_track_expire_more(ctx);
                                break;
                        }
                }
        }
        /*
         * NOTE:
         * Leave the anchor on the list, even if the end of the list has
         * been reached.  ipfw_track_expire_more_dispatch() will handle
         * the removal.
         */
        return (expired);
}

static struct ipfw_track *
ipfw_track_alloc(struct ipfw_context *ctx, const struct ipfw_flow_id *id,
    uint16_t limit_mask, struct ip_fw *rule)
{
        struct ipfw_track *key, *t, *dup;
        struct ipfw_trkcnt *trk, *ret;
        boolean_t do_expire = FALSE;

        KASSERT(rule->track_ruleid != 0,
            ("rule %u has no track ruleid", rule->rulenum));

        key = &ctx->ipfw_track_tmpkey;
        key->t_proto = id->proto;
        key->t_addrs = 0;
        key->t_ports = 0;
        key->t_rule = rule;
        if (limit_mask & DYN_SRC_ADDR)
                key->t_saddr = id->src_ip;
        if (limit_mask & DYN_DST_ADDR)
                key->t_daddr = id->dst_ip;
        if (limit_mask & DYN_SRC_PORT)
                key->t_sport = id->src_port;
        if (limit_mask & DYN_DST_PORT)
                key->t_dport = id->dst_port;

        t = RB_FIND(ipfw_track_tree, &ctx->ipfw_track_tree, key);
        if (t != NULL)
                goto done;

        t = kmalloc(sizeof(*t), M_IPFW, M_INTWAIT | M_NULLOK);
        if (t == NULL) {
                ctx->ipfw_tks_nomem++;
                return (NULL);
        }

        t->t_key = key->t_key;
        t->t_rule = rule;
        t->t_lastexp = 0;
        LIST_INIT(&t->t_state_list);

        if (ipfw_gd.ipfw_trkcnt_cnt >= ipfw_track_max) {
                time_t globexp, uptime;

                trk = NULL;
                do_expire = TRUE;

                /*
                 * Do not expire globally more than once per second,
                 * it is useless.
                 */
                uptime = time_uptime;
                globexp = ipfw_gd.ipfw_track_globexp;
                if (globexp != uptime &&
                    atomic_cmpset_long(&ipfw_gd.ipfw_track_globexp,
                    globexp, uptime)) {
                        int cpu;

                        /* Expire tracks on other CPUs. */
                        for (cpu = 0; cpu < netisr_ncpus; ++cpu) {
                                if (cpu == mycpuid)
                                        continue;
                                lwkt_send_ipiq(globaldata_find(cpu),
                                    ipfw_track_expire_ipifunc, NULL);
                        }
                }
        } else {
                trk = ipfw_trkcnt_alloc(ctx);
        }
        if (trk == NULL) {
                struct ipfw_trkcnt *tkey;

                tkey = &ctx->ipfw_trkcnt_tmpkey;
                key = NULL; /* tkey overlaps key */

                tkey->tc_key = t->t_key;
                tkey->tc_ruleid = rule->track_ruleid;

                IPFW_TRKCNT_TOKGET;
                trk = RB_FIND(ipfw_trkcnt_tree, &ipfw_gd.ipfw_trkcnt_tree,
                    tkey);
                if (trk == NULL) {
                        IPFW_TRKCNT_TOKREL;
                        if (do_expire) {
                                ctx->ipfw_tks_reap++;
                                if (ipfw_track_reap(ctx) > 0) {
                                        if (ipfw_gd.ipfw_trkcnt_cnt <
                                            ipfw_track_max) {
                                                trk = ipfw_trkcnt_alloc(ctx);
                                                if (trk != NULL)
                                                        goto install;
                                                ctx->ipfw_tks_cntnomem++;
                                        } else {
                                                ctx->ipfw_tks_overflow++;
                                        }
                                } else {
                                        ctx->ipfw_tks_reapfailed++;
                                        ctx->ipfw_tks_overflow++;
                                }
                        } else {
                                ctx->ipfw_tks_cntnomem++;
                        }
                        kfree(t, M_IPFW);
                        return (NULL);
                }
                KASSERT(trk->tc_refs > 0 && trk->tc_refs < netisr_ncpus,
                    ("invalid trkcnt refs %d", trk->tc_refs));
                atomic_add_int(&trk->tc_refs, 1);
                IPFW_TRKCNT_TOKREL;
        } else {
install:
                trk->tc_key = t->t_key;
                trk->tc_ruleid = rule->track_ruleid;
                trk->tc_refs = 0;
                trk->tc_count = 0;
                trk->tc_expire = 0;
                trk->tc_rulenum = rule->rulenum;

                IPFW_TRKCNT_TOKGET;
                ret = RB_INSERT(ipfw_trkcnt_tree, &ipfw_gd.ipfw_trkcnt_tree,
                    trk);
                if (ret != NULL) {
                        KASSERT(ret->tc_refs > 0 &&
                            ret->tc_refs < netisr_ncpus,
                            ("invalid trkcnt refs %d", ret->tc_refs));
                        KASSERT(ctx->ipfw_trkcnt_spare == NULL,
                            ("trkcnt spare was installed"));
                        ctx->ipfw_trkcnt_spare = trk;
                        trk = ret;
                } else {
                        ipfw_gd.ipfw_trkcnt_cnt++;
                }
                atomic_add_int(&trk->tc_refs, 1);
                IPFW_TRKCNT_TOKREL;
        }
        t->t_count = &trk->tc_count;
        t->t_trkcnt = trk;

        dup = RB_INSERT(ipfw_track_tree, &ctx->ipfw_track_tree, t);
        if (dup != NULL)
                panic("ipfw: track exists");
        TAILQ_INSERT_TAIL(&ctx->ipfw_track_list, t, t_link);
done:
        t->t_expire = time_uptime + dyn_short_lifetime;
        return (t);
}

/*
 * Install state for rule type cmd->o.opcode
 *
 * Returns NULL if state is not installed because of errors or because
 * states limitations are enforced.
 */
static struct ipfw_state *
ipfw_state_install(struct ipfw_context *ctx, struct ip_fw *rule,
    ipfw_insn_limit *cmd, struct ip_fw_args *args, const struct tcphdr *tcp)
{
        struct ipfw_state *s;
        struct ipfw_track *t;
        int count, diff;

        if (ipfw_gd.ipfw_state_loosecnt >= ipfw_state_max &&
            (diff = (ipfw_state_cntsync() - ipfw_state_max)) >= 0) {
                boolean_t overflow = TRUE;

                ctx->ipfw_sts_reap++;
                if (ipfw_state_reap(ctx, diff) == 0)
                        ctx->ipfw_sts_reapfailed++;
                if (ipfw_state_cntsync() < ipfw_state_max)
                        overflow = FALSE;

                if (overflow) {
                        time_t globexp, uptime;
                        int cpu;

                        /*
                         * Do not expire globally more than once per second,
                         * it is useless.
                         */
                        uptime = time_uptime;
                        globexp = ipfw_gd.ipfw_state_globexp;
                        if (globexp == uptime ||
                            !atomic_cmpset_long(&ipfw_gd.ipfw_state_globexp,
                            globexp, uptime)) {
                                ctx->ipfw_sts_overflow++;
                                return (NULL);
                        }

                        /* Expire states on other CPUs. */
                        for (cpu = 0; cpu < netisr_ncpus; ++cpu) {
                                if (cpu == mycpuid)
                                        continue;
                                lwkt_send_ipiq(globaldata_find(cpu),
                                    ipfw_state_expire_ipifunc, NULL);
                        }
                        ctx->ipfw_sts_overflow++;
                        return (NULL);
                }
        }

        switch (cmd->o.opcode) {
        case O_KEEP_STATE: /* bidir rule */
        case O_REDIRECT:
                s = ipfw_state_add(ctx, &args->f_id, cmd->o.opcode, rule, NULL,
                    tcp);
                if (s == NULL)
                        return (NULL);
                break;

        case O_LIMIT: /* limit number of sessions */
                t = ipfw_track_alloc(ctx, &args->f_id, cmd->limit_mask, rule);
                if (t == NULL)
                        return (NULL);

                if (*t->t_count >= cmd->conn_limit) {
                        if (!ipfw_track_state_expire(ctx, t, TRUE))
                                return (NULL);
                }
                for (;;) {
                        count = *t->t_count;
                        if (count >= cmd->conn_limit)
                                return (NULL);
                        if (atomic_cmpset_int(t->t_count, count, count + 1))
                                break;
                }

                s = ipfw_state_add(ctx, &args->f_id, O_LIMIT, rule, t, tcp);
                if (s == NULL) {
                        /* Undo damage. */
                        atomic_subtract_int(t->t_count, 1);
                        return (NULL);
                }
                break;

        default:
                panic("unknown state type %u\n", cmd->o.opcode);
        }

        if (s->st_type == O_REDIRECT) {
                struct ipfw_xlat *x = (struct ipfw_xlat *)s;
                ipfw_insn_rdr *r = (ipfw_insn_rdr *)cmd;

                x->xlat_addr = r->addr.s_addr;
                x->xlat_port = r->port;
                x->xlat_ifp = args->m->m_pkthdr.rcvif;
                x->xlat_dir = MATCH_FORWARD;
                KKASSERT(x->xlat_ifp != NULL);
        }
        return (s);
}

static int
ipfw_table_lookup(struct ipfw_context *ctx, uint16_t tableid,
    const struct in_addr *in)
{
        struct radix_node_head *rnh;
        struct sockaddr_in sin;
        struct ipfw_tblent *te;

        KASSERT(tableid < ipfw_table_max, ("invalid tableid %u", tableid));
        rnh = ctx->ipfw_tables[tableid];
        if (rnh == NULL)
                return (0); /* no match */

        memset(&sin, 0, sizeof(sin));
        sin.sin_family = AF_INET;
        sin.sin_len = sizeof(sin);
        sin.sin_addr = *in;

        te = (struct ipfw_tblent *)rnh->rnh_matchaddr((char *)&sin, rnh);
        if (te == NULL)
                return (0); /* no match */

        te->te_use++;
        te->te_lastuse = time_second;
        return (1); /* match */
}

/*
 * Transmit a TCP packet, containing either a RST or a keepalive.
 * When flags & TH_RST, we are sending a RST packet, because of a
 * "reset" action matched the packet.
 * Otherwise we are sending a keepalive, and flags & TH_
 *
 * Only {src,dst}_{ip,port} of "id" are used.
 */
static void
send_pkt(const struct ipfw_flow_id *id, uint32_t seq, uint32_t ack, int flags)
{
        struct mbuf *m;
        struct ip *ip;
        struct tcphdr *tcp;
        struct route sro;       /* fake route */

        MGETHDR(m, M_NOWAIT, MT_HEADER);
        if (m == NULL)
                return;
        m->m_pkthdr.rcvif = NULL;
        m->m_pkthdr.len = m->m_len = sizeof(struct ip) + sizeof(struct tcphdr);
        m->m_data += max_linkhdr;

        ip = mtod(m, struct ip *);
        bzero(ip, m->m_len);
        tcp = (struct tcphdr *)(ip + 1); /* no IP options */
        ip->ip_p = IPPROTO_TCP;
        tcp->th_off = 5;

        /*
         * Assume we are sending a RST (or a keepalive in the reverse
         * direction), swap src and destination addresses and ports.
         */
        ip->ip_src.s_addr = htonl(id->dst_ip);
        ip->ip_dst.s_addr = htonl(id->src_ip);
        tcp->th_sport = htons(id->dst_port);
        tcp->th_dport = htons(id->src_port);
        if (flags & TH_RST) {   /* we are sending a RST */
                if (flags & TH_ACK) {
                        tcp->th_seq = htonl(ack);
                        tcp->th_ack = htonl(0);
                        tcp->th_flags = TH_RST;
                } else {
                        if (flags & TH_SYN)
                                seq++;
                        tcp->th_seq = htonl(0);
                        tcp->th_ack = htonl(seq);
                        tcp->th_flags = TH_RST | TH_ACK;
                }
        } else {
                /*
                 * We are sending a keepalive. flags & TH_SYN determines
                 * the direction, forward if set, reverse if clear.
                 * NOTE: seq and ack are always assumed to be correct
                 * as set by the caller. This may be confusing...
                 */
                if (flags & TH_SYN) {
                        /*
                         * we have to rewrite the correct addresses!
                         */
                        ip->ip_dst.s_addr = htonl(id->dst_ip);
                        ip->ip_src.s_addr = htonl(id->src_ip);
                        tcp->th_dport = htons(id->dst_port);
                        tcp->th_sport = htons(id->src_port);
                }
                tcp->th_seq = htonl(seq);
                tcp->th_ack = htonl(ack);
                tcp->th_flags = TH_ACK;
        }

        /*
         * set ip_len to the payload size so we can compute
         * the tcp checksum on the pseudoheader
         * XXX check this, could save a couple of words ?
         */
        ip->ip_len = htons(sizeof(struct tcphdr));
        tcp->th_sum = in_cksum(m, m->m_pkthdr.len);

        /*
         * now fill fields left out earlier
         */
        ip->ip_ttl = ip_defttl;
        ip->ip_len = m->m_pkthdr.len;

        bzero(&sro, sizeof(sro));
        ip_rtaddr(ip->ip_dst, &sro);

        m->m_pkthdr.fw_flags |= IPFW_MBUF_GENERATED;
        ip_output(m, NULL, &sro, 0, NULL, NULL);
        if (sro.ro_rt)
                RTFREE(sro.ro_rt);
}

/*
 * Send a reject message, consuming the mbuf passed as an argument.
 */
static void
send_reject(struct ip_fw_args *args, int code, int offset, int ip_len)
{
        if (code != ICMP_REJECT_RST) { /* Send an ICMP unreach */
                /* We need the IP header in host order for icmp_error(). */
                if (args->eh != NULL) {
                        struct ip *ip = mtod(args->m, struct ip *);

                        ip->ip_len = ntohs(ip->ip_len);
                        ip->ip_off = ntohs(ip->ip_off);
                }
                icmp_error(args->m, ICMP_UNREACH, code, 0L, 0);
        } else if (offset == 0 && args->f_id.proto == IPPROTO_TCP) {
                struct tcphdr *const tcp =
                    L3HDR(struct tcphdr, mtod(args->m, struct ip *));

                if ((tcp->th_flags & TH_RST) == 0) {
                        send_pkt(&args->f_id, ntohl(tcp->th_seq),
                                 ntohl(tcp->th_ack), tcp->th_flags | TH_RST);
                }
                m_freem(args->m);
        } else {
                m_freem(args->m);
        }
        args->m = NULL;
}

/*
 * Given an ip_fw *, lookup_next_rule will return a pointer
 * to the next rule, which can be either the jump
 * target (for skipto instructions) or the next one in the list (in
 * all other cases including a missing jump target).
 * The result is also written in the "next_rule" field of the rule.
 * Backward jumps are not allowed, so start looking from the next
 * rule...
 *
 * This never returns NULL -- in case we do not have an exact match,
 * the next rule is returned. When the ruleset is changed,
 * pointers are flushed so we are always correct.
 */
static struct ip_fw *
lookup_next_rule(struct ip_fw *me)
{
        struct ip_fw *rule = NULL;
        ipfw_insn *cmd;

        /* look for action, in case it is a skipto */
        cmd = ACTION_PTR(me);
        if (cmd->opcode == O_LOG)
                cmd += F_LEN(cmd);
        if (cmd->opcode == O_SKIPTO) {
                for (rule = me->next; rule; rule = rule->next) {
                        if (rule->rulenum >= cmd->arg1)
                                break;
                }
        }
        if (rule == NULL)                       /* failure or not a skipto */
                rule = me->next;
        me->next_rule = rule;
        return rule;
}

static int
ipfw_match_uid(const struct ipfw_flow_id *fid, struct ifnet *oif,
                enum ipfw_opcodes opcode, uid_t uid)
{
        struct in_addr src_ip, dst_ip;
        struct inpcbinfo *pi;
        boolean_t wildcard;
        struct inpcb *pcb;

        if (fid->proto == IPPROTO_TCP) {
                wildcard = FALSE;
                pi = &tcbinfo[mycpuid];
        } else if (fid->proto == IPPROTO_UDP) {
                wildcard = TRUE;
                pi = &udbinfo[mycpuid];
        } else {
                return 0;
        }

        /*
         * Values in 'fid' are in host byte order
         */
        dst_ip.s_addr = htonl(fid->dst_ip);
        src_ip.s_addr = htonl(fid->src_ip);
        if (oif) {
                pcb = in_pcblookup_hash(pi,
                        dst_ip, htons(fid->dst_port),
                        src_ip, htons(fid->src_port),
                        wildcard, oif);
        } else {
                pcb = in_pcblookup_hash(pi,
                        src_ip, htons(fid->src_port),
                        dst_ip, htons(fid->dst_port),
                        wildcard, NULL);
        }
        if (pcb == NULL || pcb->inp_socket == NULL)
                return 0;

        if (opcode == O_UID) {
#define socheckuid(a,b) ((a)->so_cred->cr_uid != (b))
                return !socheckuid(pcb->inp_socket, uid);
#undef socheckuid
        } else  {
                return groupmember(uid, pcb->inp_socket->so_cred);
        }
}

static int
ipfw_match_ifip(ipfw_insn_ifip *cmd, const struct in_addr *ip)
{

        if (__predict_false((cmd->o.arg1 & IPFW_IFIP_VALID) == 0)) {
                struct ifaddr_container *ifac;
                struct ifnet *ifp;

                ifp = ifunit_netisr(cmd->ifname);
                if (ifp == NULL)
                        return (0);

                TAILQ_FOREACH(ifac, &ifp->if_addrheads[mycpuid], ifa_link) {
                        struct ifaddr *ia = ifac->ifa;

                        if (ia->ifa_addr == NULL)
                                continue;
                        if (ia->ifa_addr->sa_family != AF_INET)
                                continue;

                        cmd->mask.s_addr = INADDR_ANY;
                        if (cmd->o.arg1 & IPFW_IFIP_NET) {
                                cmd->mask = ((struct sockaddr_in *)
                                    ia->ifa_netmask)->sin_addr;
                        }
                        if (cmd->mask.s_addr == INADDR_ANY)
                                cmd->mask.s_addr = INADDR_BROADCAST;

                        cmd->addr =
                            ((struct sockaddr_in *)ia->ifa_addr)->sin_addr;
                        cmd->addr.s_addr &= cmd->mask.s_addr;

                        cmd->o.arg1 |= IPFW_IFIP_VALID;
                        break;
                }
                if ((cmd->o.arg1 & IPFW_IFIP_VALID) == 0)
                        return (0);
        }
        return ((ip->s_addr & cmd->mask.s_addr) == cmd->addr.s_addr);
}

static void
ipfw_xlate(const struct ipfw_xlat *x, struct mbuf *m,
    struct in_addr *old_addr, uint16_t *old_port)
{
        struct ip *ip = mtod(m, struct ip *);
        struct in_addr *addr;
        uint16_t *port, *csum, dlen = 0;
        uint8_t udp = 0;
        boolean_t pseudo = FALSE;

        if (x->xlat_flags & IPFW_STATE_F_XLATSRC) {
                addr = &ip->ip_src;
                switch (ip->ip_p) {
                case IPPROTO_TCP:
                        port = &L3HDR(struct tcphdr, ip)->th_sport;
                        csum = &L3HDR(struct tcphdr, ip)->th_sum;
                        break;
                case IPPROTO_UDP:
                        port = &L3HDR(struct udphdr, ip)->uh_sport;
                        csum = &L3HDR(struct udphdr, ip)->uh_sum;
                        udp = 1;
                        break;
                default:
                        panic("ipfw: unsupported src xlate proto %u", ip->ip_p);
                }
        } else {
                addr = &ip->ip_dst;
                switch (ip->ip_p) {
                case IPPROTO_TCP:
                        port = &L3HDR(struct tcphdr, ip)->th_dport;
                        csum = &L3HDR(struct tcphdr, ip)->th_sum;
                        break;
                case IPPROTO_UDP:
                        port = &L3HDR(struct udphdr, ip)->uh_dport;
                        csum = &L3HDR(struct udphdr, ip)->uh_sum;
                        udp = 1;
                        break;
                default:
                        panic("ipfw: unsupported dst xlate proto %u", ip->ip_p);
                }
        }
        if (old_addr != NULL)
                *old_addr = *addr;
        if (old_port != NULL) {
                if (x->xlat_port != 0)
                        *old_port = *port;
                else
                        *old_port = 0;
        }

        if (m->m_pkthdr.csum_flags & (CSUM_UDP | CSUM_TCP | CSUM_TSO)) {
                if ((m->m_pkthdr.csum_flags & CSUM_TSO) == 0)
                        dlen = ip->ip_len - (ip->ip_hl << 2);
                pseudo = TRUE;
        }

        if (!pseudo) {
                const uint16_t *oaddr, *naddr;

                oaddr = (const uint16_t *)&addr->s_addr;
                naddr = (const uint16_t *)&x->xlat_addr;

                ip->ip_sum = pfil_cksum_fixup(pfil_cksum_fixup(ip->ip_sum,
                    oaddr[0], naddr[0], 0), oaddr[1], naddr[1], 0);
                *csum = pfil_cksum_fixup(pfil_cksum_fixup(*csum,
                    oaddr[0], naddr[0], udp), oaddr[1], naddr[1], udp);
        }
        addr->s_addr = x->xlat_addr;

        if (x->xlat_port != 0) {
                if (!pseudo) {
                        *csum = pfil_cksum_fixup(*csum, *port, x->xlat_port,
                            udp);
                }
                *port = x->xlat_port;
        }

        if (pseudo) {
                *csum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
                    htons(dlen + ip->ip_p));
        }
}

static void
ipfw_ip_xlate_dispatch(netmsg_t nmsg)
{
        struct netmsg_genpkt *nm = (struct netmsg_genpkt *)nmsg;
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct mbuf *m = nm->m;
        struct ipfw_xlat *x = nm->arg1;
        struct ip_fw *rule = x->xlat_rule;

        ASSERT_NETISR_NCPUS(mycpuid);
        KASSERT(rule->cpuid == mycpuid,
            ("rule does not belong to cpu%d", mycpuid));
        KASSERT(m->m_pkthdr.fw_flags & IPFW_MBUF_CONTINUE,
            ("mbuf does not have ipfw continue rule"));

        KASSERT(ctx->ipfw_cont_rule == NULL,
            ("pending ipfw continue rule"));
        KASSERT(ctx->ipfw_cont_xlat == NULL,
            ("pending ipfw continue xlat"));
        ctx->ipfw_cont_rule = rule;
        ctx->ipfw_cont_xlat = x;

        if (nm->arg2 == 0)
                ip_input(m);
        else
                ip_output(m, NULL, NULL, IP_FORWARDING, NULL, NULL);

        /* May not be cleared, if ipfw was unload/disabled. */
        ctx->ipfw_cont_rule = NULL;
        ctx->ipfw_cont_xlat = NULL;

        /*
         * This state is no longer used; decrement its xlat_crefs,
         * so this state can be deleted.
         */
        x->xlat_crefs--;
        /*
         * This rule is no longer used; decrement its cross_refs,
         * so this rule can be deleted.
         *
         * NOTE:
         * Decrement cross_refs in the last step of this function,
         * so that the module could be unloaded safely.
         */
        rule->cross_refs--;
}

static void
ipfw_xlate_redispatch(struct mbuf *m, int cpuid, struct ipfw_xlat *x,
    uint32_t flags)
{
        struct netmsg_genpkt *nm;

        KASSERT(x->xlat_pcpu == cpuid, ("xlat paired cpu%d, target cpu%d",
            x->xlat_pcpu, cpuid));

        /*
         * Bump cross_refs to prevent this rule and its siblings
         * from being deleted, while this mbuf is inflight.  The
         * cross_refs of the sibling rule on the target cpu will
         * be decremented, once this mbuf is going to be filtered
         * on the target cpu.
         */
        x->xlat_rule->cross_refs++;
        /*
         * Bump xlat_crefs to prevent this state and its paired
         * state from being deleted, while this mbuf is inflight.
         * The xlat_crefs of the paired state on the target cpu
         * will be decremented, once this mbuf is going to be
         * filtered on the target cpu.
         */
        x->xlat_crefs++;

        m->m_pkthdr.fw_flags |= IPFW_MBUF_CONTINUE;
        if (flags & IPFW_XLATE_INSERT)
                m->m_pkthdr.fw_flags |= IPFW_MBUF_XLATINS;
        if (flags & IPFW_XLATE_FORWARD)
                m->m_pkthdr.fw_flags |= IPFW_MBUF_XLATFWD;

        if ((flags & IPFW_XLATE_OUTPUT) == 0) {
                struct ip *ip = mtod(m, struct ip *);

                /*
                 * NOTE:
                 * ip_input() expects ip_len/ip_off are in network
                 * byte order.
                 */
                ip->ip_len = htons(ip->ip_len);
                ip->ip_off = htons(ip->ip_off);
        }

        nm = &m->m_hdr.mh_genmsg;
        netmsg_init(&nm->base, NULL, &netisr_apanic_rport, 0,
            ipfw_ip_xlate_dispatch);
        nm->m = m;
        nm->arg1 = x->xlat_pair;
        nm->arg2 = 0;
        if (flags & IPFW_XLATE_OUTPUT)
                nm->arg2 = 1;
        netisr_sendmsg(&nm->base, cpuid);
}

static struct mbuf *
ipfw_setup_local(struct mbuf *m, const int hlen, struct ip_fw_args *args,
    struct ip_fw_local *local, struct ip **ip0)
{
        struct ip *ip = mtod(m, struct ip *);
        struct tcphdr *tcp;
        struct udphdr *udp;

        /*
         * Collect parameters into local variables for faster matching.
         */
        if (hlen == 0) {        /* do not grab addresses for non-ip pkts */
                local->proto = args->f_id.proto = 0;    /* mark f_id invalid */
                goto done;
        }

        local->proto = args->f_id.proto = ip->ip_p;
        local->src_ip = ip->ip_src;
        local->dst_ip = ip->ip_dst;
        if (args->eh != NULL) { /* layer 2 packets are as on the wire */
                local->offset = ntohs(ip->ip_off) & IP_OFFMASK;
                local->ip_len = ntohs(ip->ip_len);
        } else {
                local->offset = ip->ip_off & IP_OFFMASK;
                local->ip_len = ip->ip_len;
        }

#define PULLUP_TO(len)                                  \
do {                                                    \
        if (m->m_len < (len)) {                         \
                args->m = m = m_pullup(m, (len));       \
                if (m == NULL) {                        \
                        ip = NULL;                      \
                        goto done;                      \
                }                                       \
                ip = mtod(m, struct ip *);              \
        }                                               \
} while (0)

        if (local->offset == 0) {
                switch (local->proto) {
                case IPPROTO_TCP:
                        PULLUP_TO(hlen + sizeof(struct tcphdr));
                        local->tcp = tcp = L3HDR(struct tcphdr, ip);
                        local->dst_port = tcp->th_dport;
                        local->src_port = tcp->th_sport;
                        args->f_id.flags = tcp->th_flags;
                        break;

                case IPPROTO_UDP:
                        PULLUP_TO(hlen + sizeof(struct udphdr));
                        udp = L3HDR(struct udphdr, ip);
                        local->dst_port = udp->uh_dport;
                        local->src_port = udp->uh_sport;
                        break;

                case IPPROTO_ICMP:
                        PULLUP_TO(hlen + 4);    /* type, code and checksum. */
                        args->f_id.flags = L3HDR(struct icmp, ip)->icmp_type;
                        break;

                default:
                        break;
                }
        }

#undef PULLUP_TO

        args->f_id.src_ip = ntohl(local->src_ip.s_addr);
        args->f_id.dst_ip = ntohl(local->dst_ip.s_addr);
        args->f_id.src_port = local->src_port = ntohs(local->src_port);
        args->f_id.dst_port = local->dst_port = ntohs(local->dst_port);
done:
        *ip0 = ip;
        return (m);
}

static struct mbuf *
ipfw_rehashm(struct mbuf *m, const int hlen, struct ip_fw_args *args,
    struct ip_fw_local *local, struct ip **ip0)
{
        struct ip *ip = mtod(m, struct ip *);

        ip->ip_len = htons(ip->ip_len);
        ip->ip_off = htons(ip->ip_off);

        m->m_flags &= ~M_HASH;
        ip_hashfn(&m, 0);
        args->m = m;
        if (m == NULL) {
                *ip0 = NULL;
                return (NULL);
        }
        KASSERT(m->m_flags & M_HASH, ("no hash"));

        /* 'm' might be changed by ip_hashfn(). */
        ip = mtod(m, struct ip *);
        ip->ip_len = ntohs(ip->ip_len);
        ip->ip_off = ntohs(ip->ip_off);

        return (ipfw_setup_local(m, hlen, args, local, ip0));
}

/*
 * The main check routine for the firewall.
 *
 * All arguments are in args so we can modify them and return them
 * back to the caller.
 *
 * Parameters:
 *
 *      args->m (in/out) The packet; we set to NULL when/if we nuke it.
 *              Starts with the IP header.
 *      args->eh (in)   Mac header if present, or NULL for layer3 packet.
 *      args->oif       Outgoing interface, or NULL if packet is incoming.
 *              The incoming interface is in the mbuf. (in)
 *
 *      args->rule      Pointer to the last matching rule (in/out)
 *      args->f_id      Addresses grabbed from the packet (out)
 *
 * Return value:
 *
 *      If the packet was denied/rejected and has been dropped, *m is equal
 *      to NULL upon return.
 *
 *      IP_FW_DENY      the packet must be dropped.
 *      IP_FW_PASS      The packet is to be accepted and routed normally.
 *      IP_FW_DIVERT    Divert the packet to port (args->cookie)
 *      IP_FW_TEE       Tee the packet to port (args->cookie)
 *      IP_FW_DUMMYNET  Send the packet to pipe/queue (args->cookie)
 *      IP_FW_CONTINUE  Continue processing on another cpu.
 */
static int
ipfw_chk(struct ip_fw_args *args)
{
        /*
         * Local variables hold state during the processing of a packet.
         *
         * IMPORTANT NOTE: to speed up the processing of rules, there
         * are some assumption on the values of the variables, which
         * are documented here. Should you change them, please check
         * the implementation of the various instructions to make sure
         * that they still work.
         *
         * args->eh     The MAC header. It is non-null for a layer2
         *      packet, it is NULL for a layer-3 packet.
         *
         * m | args->m  Pointer to the mbuf, as received from the caller.
         *      It may change if ipfw_chk() does an m_pullup, or if it
         *      consumes the packet because it calls send_reject().
         *      XXX This has to change, so that ipfw_chk() never modifies
         *      or consumes the buffer.
         * ip   is simply an alias of the value of m, and it is kept
         *      in sync with it (the packet is  supposed to start with
         *      the ip header).
         */
        struct mbuf *m = args->m;
        struct ip *ip = mtod(m, struct ip *);

        /*
         * oif | args->oif      If NULL, ipfw_chk has been called on the
         *      inbound path (ether_input, ip_input).
         *      If non-NULL, ipfw_chk has been called on the outbound path
         *      (ether_output, ip_output).
         */
        struct ifnet *oif = args->oif;

        struct ip_fw *f = NULL;         /* matching rule */
        int retval = IP_FW_PASS;
        struct m_tag *mtag;
        struct divert_info *divinfo;
        struct ipfw_state *s;

        /*
         * hlen The length of the IPv4 header.
         *      hlen >0 means we have an IPv4 packet.
         */
        u_int hlen = 0;         /* hlen >0 means we have an IP pkt */

        struct ip_fw_local lc;

        /*
         * dyn_dir = MATCH_UNKNOWN when rules unchecked,
         *      MATCH_NONE when checked and not matched (dyn_f = NULL),
         *      MATCH_FORWARD or MATCH_REVERSE otherwise (dyn_f != NULL)
         */
        int dyn_dir = MATCH_UNKNOWN;
        struct ip_fw *dyn_f = NULL;
        int cpuid = mycpuid;
        struct ipfw_context *ctx;

        ASSERT_NETISR_NCPUS(cpuid);
        ctx = ipfw_ctx[cpuid];

        if (m->m_pkthdr.fw_flags & IPFW_MBUF_GENERATED)
                return IP_FW_PASS;      /* accept */

        if (args->eh == NULL ||         /* layer 3 packet */
            (m->m_pkthdr.len >= sizeof(struct ip) &&
             ntohs(args->eh->ether_type) == ETHERTYPE_IP))
                hlen = ip->ip_hl << 2;

        memset(&lc, 0, sizeof(lc));

        m = ipfw_setup_local(m, hlen, args, &lc, &ip);
        if (m == NULL)
                goto pullup_failed;

        if (args->rule) {
                /*
                 * Packet has already been tagged. Look for the next rule
                 * to restart processing.
                 *
                 * If fw_one_pass != 0 then just accept it.
                 * XXX should not happen here, but optimized out in
                 * the caller.
                 */
                if (fw_one_pass && (args->flags & IP_FWARG_F_CONT) == 0)
                        return IP_FW_PASS;
                args->flags &= ~IP_FWARG_F_CONT;

                /* This rule is being/has been flushed */
                if (ipfw_flushing)
                        return IP_FW_DENY;

                KASSERT(args->rule->cpuid == cpuid,
                        ("rule used on cpu%d", cpuid));

                /* This rule was deleted */
                if (args->rule->rule_flags & IPFW_RULE_F_INVALID)
                        return IP_FW_DENY;

                if (args->xlat != NULL) {
                        struct ipfw_xlat *x = args->xlat;

                        /* This xlat is being deleted. */
                        if (x->xlat_invalid)
                                return IP_FW_DENY;

                        f = args->rule;

                        dyn_f = f;
                        dyn_dir = (args->flags & IP_FWARG_F_XLATFWD) ?
                            MATCH_FORWARD : MATCH_REVERSE;

                        if (args->flags & IP_FWARG_F_XLATINS) {
                                KASSERT(x->xlat_flags & IPFW_STATE_F_XLATSLAVE,
                                    ("not slave %u state", x->xlat_type));
                                s = ipfw_state_link(ctx, &x->xlat_st);
                                if (s != NULL) {
                                        ctx->ipfw_xlate_conflicts++;
                                        if (IPFW_STATE_ISDEAD(s)) {
                                                ipfw_state_remove(ctx, s);
                                                s = ipfw_state_link(ctx,
                                                    &x->xlat_st);
                                        }
                                        if (s != NULL) {
                                                if (bootverbose) {
                                                        kprintf("ipfw: "
                                                        "slave %u state "
                                                        "conflicts %u state\n",
                                                        x->xlat_type,
                                                        s->st_type);
                                                }
                                                ipfw_xlat_invalidate(x);
                                                return IP_FW_DENY;
                                        }
                                        ctx->ipfw_xlate_cresolved++;
                                }
                        } else {
                                ipfw_state_update(&args->f_id, dyn_dir,
                                    lc.tcp, &x->xlat_st);
                        }
                } else {
                        /* TODO: setup dyn_f, dyn_dir */

                        f = args->rule->next_rule;
                        if (f == NULL)
                                f = lookup_next_rule(args->rule);
                }
        } else {
                /*
                 * Find the starting rule. It can be either the first
                 * one, or the one after divert_rule if asked so.
                 */
                int skipto;

                KKASSERT((args->flags &
                    (IP_FWARG_F_XLATINS | IP_FWARG_F_CONT)) == 0);
                KKASSERT(args->xlat == NULL);

                mtag = m_tag_find(m, PACKET_TAG_IPFW_DIVERT, NULL);
                if (mtag != NULL) {
                        divinfo = m_tag_data(mtag);
                        skipto = divinfo->skipto;
                } else {
                        skipto = 0;
                }

                f = ctx->ipfw_layer3_chain;
                if (args->eh == NULL && skipto != 0) {
                        /* No skipto during rule flushing */
                        if (ipfw_flushing)
                                return IP_FW_DENY;

                        if (skipto >= IPFW_DEFAULT_RULE)
                                return IP_FW_DENY; /* invalid */

                        while (f && f->rulenum <= skipto)
                                f = f->next;
                        if (f == NULL)  /* drop packet */
                                return IP_FW_DENY;
                } else if (ipfw_flushing) {
                        /* Rules are being flushed; skip to default rule */
                        f = ctx->ipfw_default_rule;
                }
        }
        if ((mtag = m_tag_find(m, PACKET_TAG_IPFW_DIVERT, NULL)) != NULL)
                m_tag_delete(m, mtag);

        /*
         * Now scan the rules, and parse microinstructions for each rule.
         */
        for (; f; f = f->next) {
                int l, cmdlen;
                ipfw_insn *cmd;
                int skip_or; /* skip rest of OR block */

again:
                if (ctx->ipfw_set_disable & (1 << f->set)) {
                        args->xlat = NULL;
                        continue;
                }

                if (args->xlat != NULL) {
                        args->xlat = NULL;
                        l = f->cmd_len - f->act_ofs;
                        cmd = ACTION_PTR(f);
                } else {
                        l = f->cmd_len;
                        cmd = f->cmd;
                }

                skip_or = 0;
                for (; l > 0; l -= cmdlen, cmd += cmdlen) {
                        int match;

                        /*
                         * check_body is a jump target used when we find a
                         * CHECK_STATE, and need to jump to the body of
                         * the target rule.
                         */
check_body:
                        cmdlen = F_LEN(cmd);
                        /*
                         * An OR block (insn_1 || .. || insn_n) has the
                         * F_OR bit set in all but the last instruction.
                         * The first match will set "skip_or", and cause
                         * the following instructions to be skipped until
                         * past the one with the F_OR bit clear.
                         */
                        if (skip_or) {          /* skip this instruction */
                                if ((cmd->len & F_OR) == 0)
                                        skip_or = 0;    /* next one is good */
                                continue;
                        }
                        match = 0; /* set to 1 if we succeed */

                        switch (cmd->opcode) {
                        /*
                         * The first set of opcodes compares the packet's
                         * fields with some pattern, setting 'match' if a
                         * match is found. At the end of the loop there is
                         * logic to deal with F_NOT and F_OR flags associated
                         * with the opcode.
                         */
                        case O_NOP:
                                match = 1;
                                break;

                        case O_FORWARD_MAC:
                                kprintf("ipfw: opcode %d unimplemented\n",
                                        cmd->opcode);
                                break;

                        case O_GID:
                        case O_UID:
                                /*
                                 * We only check offset == 0 && proto != 0,
                                 * as this ensures that we have an IPv4
                                 * packet with the ports info.
                                 */
                                if (lc.offset!=0)
                                        break;

                                match = ipfw_match_uid(&args->f_id, oif,
                                        cmd->opcode,
                                        (uid_t)((ipfw_insn_u32 *)cmd)->d[0]);
                                break;

                        case O_RECV:
                                match = iface_match(m->m_pkthdr.rcvif,
                                    (ipfw_insn_if *)cmd);
                                break;

                        case O_XMIT:
                                match = iface_match(oif, (ipfw_insn_if *)cmd);
                                break;

                        case O_VIA:
                                match = iface_match(oif ? oif :
                                    m->m_pkthdr.rcvif, (ipfw_insn_if *)cmd);
                                break;

                        case O_MACADDR2:
                                if (args->eh != NULL) { /* have MAC header */
                                        uint32_t *want = (uint32_t *)
                                                ((ipfw_insn_mac *)cmd)->addr;
                                        uint32_t *mask = (uint32_t *)
                                                ((ipfw_insn_mac *)cmd)->mask;
                                        uint32_t *hdr = (uint32_t *)args->eh;

                                        match =
                                        (want[0] == (hdr[0] & mask[0]) &&
                                         want[1] == (hdr[1] & mask[1]) &&
                                         want[2] == (hdr[2] & mask[2]));
                                }
                                break;

                        case O_MAC_TYPE:
                                if (args->eh != NULL) {
                                        uint16_t t =
                                            ntohs(args->eh->ether_type);
                                        uint16_t *p =
                                            ((ipfw_insn_u16 *)cmd)->ports;
                                        int i;

                                        /* Special vlan handling */
                                        if (m->m_flags & M_VLANTAG)
                                                t = ETHERTYPE_VLAN;

                                        for (i = cmdlen - 1; !match && i > 0;
                                             i--, p += 2) {
                                                match =
                                                (t >= p[0] && t <= p[1]);
                                        }
                                }
                                break;

                        case O_FRAG:
                                match = (hlen > 0 && lc.offset != 0);
                                break;

                        case O_IPFRAG:
                                if (hlen > 0) {
                                        uint16_t off;

                                        if (args->eh != NULL)
                                                off = ntohs(ip->ip_off);
                                        else
                                                off = ip->ip_off;
                                        if (off & (IP_MF | IP_OFFMASK))
                                                match = 1;
                                }
                                break;

                        case O_IN:      /* "out" is "not in" */
                                match = (oif == NULL);
                                break;

                        case O_LAYER2:
                                match = (args->eh != NULL);
                                break;

                        case O_PROTO:
                                /*
                                 * We do not allow an arg of 0 so the
                                 * check of "proto" only suffices.
                                 */
                                match = (lc.proto == cmd->arg1);
                                break;

                        case O_IP_SRC:
                                match = (hlen > 0 &&
                                    ((ipfw_insn_ip *)cmd)->addr.s_addr ==
                                    lc.src_ip.s_addr);
                                break;

                        case O_IP_SRC_MASK:
                                match = (hlen > 0 &&
                                    ((ipfw_insn_ip *)cmd)->addr.s_addr ==
                                     (lc.src_ip.s_addr &
                                     ((ipfw_insn_ip *)cmd)->mask.s_addr));
                                break;

                        case O_IP_SRC_ME:
                                if (hlen > 0) {
                                        struct ifnet *tif;

                                        tif = INADDR_TO_IFP(&lc.src_ip);
                                        match = (tif != NULL);
                                }
                                break;

                        case O_IP_SRC_TABLE:
                                match = ipfw_table_lookup(ctx, cmd->arg1,
                                    &lc.src_ip);
                                break;

                        case O_IP_SRC_IFIP:
                                match = ipfw_match_ifip((ipfw_insn_ifip *)cmd,
                                    &lc.src_ip);
                                break;

                        case O_IP_DST_SET:
                        case O_IP_SRC_SET:
                                if (hlen > 0) {
                                        uint32_t *d = (uint32_t *)(cmd + 1);
                                        uint32_t addr =
                                            cmd->opcode == O_IP_DST_SET ?
                                                args->f_id.dst_ip :
                                                args->f_id.src_ip;

                                        if (addr < d[0])
                                                break;
                                        addr -= d[0]; /* subtract base */
                                        match =
                                        (addr < cmd->arg1) &&
                                         (d[1 + (addr >> 5)] &
                                          (1 << (addr & 0x1f)));
                                }
                                break;

                        case O_IP_DST:
                                match = (hlen > 0 &&
                                    ((ipfw_insn_ip *)cmd)->addr.s_addr ==
                                    lc.dst_ip.s_addr);
                                break;

                        case O_IP_DST_MASK:
                                match = (hlen > 0) &&
                                    (((ipfw_insn_ip *)cmd)->addr.s_addr ==
                                     (lc.dst_ip.s_addr &
                                     ((ipfw_insn_ip *)cmd)->mask.s_addr));
                                break;

                        case O_IP_DST_ME:
                                if (hlen > 0) {
                                        struct ifnet *tif;

                                        tif = INADDR_TO_IFP(&lc.dst_ip);
                                        match = (tif != NULL);
                                }
                                break;

                        case O_IP_DST_TABLE:
                                match = ipfw_table_lookup(ctx, cmd->arg1,
                                    &lc.dst_ip);
                                break;

                        case O_IP_DST_IFIP:
                                match = ipfw_match_ifip((ipfw_insn_ifip *)cmd,
                                    &lc.dst_ip);
                                break;

                        case O_IP_SRCPORT:
                        case O_IP_DSTPORT:
                                /*
                                 * offset == 0 && proto != 0 is enough
                                 * to guarantee that we have an IPv4
                                 * packet with port info.
                                 */
                                if ((lc.proto==IPPROTO_UDP ||
                                     lc.proto==IPPROTO_TCP)
                                    && lc.offset == 0) {
                                        uint16_t x =
                                            (cmd->opcode == O_IP_SRCPORT) ?
                                                lc.src_port : lc.dst_port;
                                        uint16_t *p =
                                            ((ipfw_insn_u16 *)cmd)->ports;
                                        int i;

                                        for (i = cmdlen - 1; !match && i > 0;
                                             i--, p += 2) {
                                                match =
                                                (x >= p[0] && x <= p[1]);
                                        }
                                }
                                break;

                        case O_ICMPCODE:
                                match = (lc.offset == 0 &&
                                    lc.proto==IPPROTO_ICMP &&
                                    icmpcode_match(ip, (ipfw_insn_u32 *)cmd));
                                break;

                        case O_ICMPTYPE:
                                match = (lc.offset == 0 &&
                                    lc.proto==IPPROTO_ICMP &&
                                    icmptype_match(ip, (ipfw_insn_u32 *)cmd));
                                break;

                        case O_IPOPT:
                                match = (hlen > 0 && ipopts_match(ip, cmd));
                                break;

                        case O_IPVER:
                                match = (hlen > 0 && cmd->arg1 == ip->ip_v);
                                break;

                        case O_IPTTL:
                                match = (hlen > 0 && cmd->arg1 == ip->ip_ttl);
                                break;

                        case O_IPID:
                                match = (hlen > 0 &&
                                    cmd->arg1 == ntohs(ip->ip_id));
                                break;

                        case O_IPLEN:
                                match = (hlen > 0 && cmd->arg1 == lc.ip_len);
                                break;

                        case O_IPPRECEDENCE:
                                match = (hlen > 0 &&
                                    (cmd->arg1 == (ip->ip_tos & 0xe0)));
                                break;

                        case O_IPTOS:
                                match = (hlen > 0 &&
                                    flags_match(cmd, ip->ip_tos));
                                break;

                        case O_TCPFLAGS:
                                match = (lc.proto == IPPROTO_TCP &&
                                    lc.offset == 0 &&
                                    flags_match(cmd,
                                        L3HDR(struct tcphdr,ip)->th_flags));
                                break;

                        case O_TCPOPTS:
                                match = (lc.proto == IPPROTO_TCP &&
                                    lc.offset == 0 && tcpopts_match(ip, cmd));
                                break;

                        case O_TCPSEQ:
                                match = (lc.proto == IPPROTO_TCP &&
                                    lc.offset == 0 &&
                                    ((ipfw_insn_u32 *)cmd)->d[0] ==
                                        L3HDR(struct tcphdr,ip)->th_seq);
                                break;

                        case O_TCPACK:
                                match = (lc.proto == IPPROTO_TCP &&
                                    lc.offset == 0 &&
                                    ((ipfw_insn_u32 *)cmd)->d[0] ==
                                        L3HDR(struct tcphdr,ip)->th_ack);
                                break;

                        case O_TCPWIN:
                                match = (lc.proto == IPPROTO_TCP &&
                                    lc.offset == 0 &&
                                    cmd->arg1 ==
                                        L3HDR(struct tcphdr,ip)->th_win);
                                break;

                        case O_ESTAB:
                                /* reject packets which have SYN only */
                                /* XXX should i also check for TH_ACK ? */
                                match = (lc.proto == IPPROTO_TCP &&
                                    lc.offset == 0 &&
                                    (L3HDR(struct tcphdr,ip)->th_flags &
                                     (TH_RST | TH_ACK | TH_SYN)) != TH_SYN);
                                break;

                        case O_LOG:
                                if (fw_verbose) {
                                        ipfw_log(ctx, f, hlen, args->eh, m,
                                            oif);
                                }
                                match = 1;
                                break;

                        case O_PROB:
                                match = (krandom() <
                                        ((ipfw_insn_u32 *)cmd)->d[0]);
                                break;

                        /*
                         * The second set of opcodes represents 'actions',
                         * i.e. the terminal part of a rule once the packet
                         * matches all previous patterns.
                         * Typically there is only one action for each rule,
                         * and the opcode is stored at the end of the rule
                         * (but there are exceptions -- see below).
                         *
                         * In general, here we set retval and terminate the
                         * outer loop (would be a 'break 3' in some language,
                         * but we need to do a 'goto done').
                         *
                         * Exceptions:
                         * O_COUNT and O_SKIPTO actions:
                         *   instead of terminating, we jump to the next rule
                         *   ('goto next_rule', equivalent to a 'break 2'),
                         *   or to the SKIPTO target ('goto again' after
                         *   having set f, cmd and l), respectively.
                         *
                         * O_LIMIT and O_KEEP_STATE, O_REDIRECT: these opcodes
                         *   are not real 'actions', and are stored right
                         *   before the 'action' part of the rule.
                         *   These opcodes try to install an entry in the
                         *   state tables; if successful, we continue with
                         *   the next opcode (match=1; break;), otherwise
                         *   the packet must be dropped ('goto done' after
                         *   setting retval).  If static rules are changed
                         *   during the state installation, the packet will
                         *   be dropped and rule's stats will not beupdated
                         *   ('return IP_FW_DENY').
                         *
                         * O_PROBE_STATE and O_CHECK_STATE: these opcodes
                         *   cause a lookup of the state table, and a jump
                         *   to the 'action' part of the parent rule
                         *   ('goto check_body') if an entry is found, or
                         *   (CHECK_STATE only) a jump to the next rule if
                         *   the entry is not found ('goto next_rule').
                         *   The result of the lookup is cached to make
                         *   further instances of these opcodes are
                         *   effectively NOPs.  If static rules are changed
                         *   during the state looking up, the packet will
                         *   be dropped and rule's stats will not be updated
                         *   ('return IP_FW_DENY').
                         */
                        case O_REDIRECT:
                                if (f->cross_rules == NULL) {
                                        /*
                                         * This rule was not completely setup;
                                         * move on to the next rule.
                                         */
                                        goto next_rule;
                                }
                                /*
                                 * Apply redirect only on input path and
                                 * only to non-fragment TCP segments or
                                 * UDP datagrams.
                                 *
                                 * Does _not_ work with layer2 filtering.
                                 */
                                if (oif != NULL || args->eh != NULL ||
                                    (ip->ip_off & (IP_MF | IP_OFFMASK)) ||
                                    (lc.proto != IPPROTO_TCP &&
                                     lc.proto != IPPROTO_UDP))
                                        break;
                                /* FALL THROUGH */
                        case O_LIMIT:
                        case O_KEEP_STATE:
                                if (hlen == 0)
                                        break;
                                s = ipfw_state_install(ctx, f,
                                    (ipfw_insn_limit *)cmd, args, lc.tcp);
                                if (s == NULL) {
                                        retval = IP_FW_DENY;
                                        goto done; /* error/limit violation */
                                }
                                s->st_pcnt++;
                                s->st_bcnt += lc.ip_len;

                                if (s->st_type == O_REDIRECT) {
                                        struct in_addr oaddr;
                                        uint16_t oport;
                                        struct ipfw_xlat *slave_x, *x;
                                        struct ipfw_state *dup;

                                        x = (struct ipfw_xlat *)s;
                                        ipfw_xlate(x, m, &oaddr, &oport);
                                        m = ipfw_rehashm(m, hlen, args, &lc,
                                            &ip);
                                        if (m == NULL) {
                                                ipfw_state_del(ctx, s);
                                                goto pullup_failed;
                                        }

                                        cpuid = netisr_hashcpu(
                                            m->m_pkthdr.hash);

                                        slave_x = (struct ipfw_xlat *)
                                            ipfw_state_alloc(ctx, &args->f_id,
                                            O_REDIRECT, f->cross_rules[cpuid],
                                            lc.tcp);
                                        if (slave_x == NULL) {
                                                ipfw_state_del(ctx, s);
                                                retval = IP_FW_DENY;
                                                goto done;
                                        }
                                        slave_x->xlat_addr = oaddr.s_addr;
                                        slave_x->xlat_port = oport;
                                        slave_x->xlat_dir = MATCH_REVERSE;
                                        slave_x->xlat_flags |=
                                            IPFW_STATE_F_XLATSRC |
                                            IPFW_STATE_F_XLATSLAVE;

                                        slave_x->xlat_pair = x;
                                        slave_x->xlat_pcpu = mycpuid;
                                        x->xlat_pair = slave_x;
                                        x->xlat_pcpu = cpuid;

                                        ctx->ipfw_xlated++;
                                        if (cpuid != mycpuid) {
                                                ctx->ipfw_xlate_split++;
                                                ipfw_xlate_redispatch(
                                                    m, cpuid, x,
                                                    IPFW_XLATE_INSERT |
                                                    IPFW_XLATE_FORWARD);
                                                args->m = NULL;
                                                return (IP_FW_REDISPATCH);
                                        }

                                        dup = ipfw_state_link(ctx,
                                            &slave_x->xlat_st);
                                        if (dup != NULL) {
                                                ctx->ipfw_xlate_conflicts++;
                                                if (IPFW_STATE_ISDEAD(dup)) {
                                                        ipfw_state_remove(ctx,
                                                            dup);
                                                        dup = ipfw_state_link(
                                                        ctx, &slave_x->xlat_st);
                                                }
                                                if (dup != NULL) {
                                                        if (bootverbose) {
                                                            kprintf("ipfw: "
                                                            "slave %u state "
                                                            "conflicts "
                                                            "%u state\n",
                                                            x->xlat_type,
                                                            s->st_type);
                                                        }
                                                        ipfw_state_del(ctx, s);
                                                        return (IP_FW_DENY);
                                                }
                                                ctx->ipfw_xlate_cresolved++;
                                        }
                                }
                                match = 1;
                                break;

                        case O_PROBE_STATE:
                        case O_CHECK_STATE:
                                /*
                                 * States are checked at the first keep-state 
                                 * check-state occurrence, with the result
                                 * being stored in dyn_dir.  The compiler
                                 * introduces a PROBE_STATE instruction for
                                 * us when we have a KEEP_STATE/LIMIT/RDR
                                 * (because PROBE_STATE needs to be run first).
                                 */
                                s = NULL;
                                if (dyn_dir == MATCH_UNKNOWN) {
                                        s = ipfw_state_lookup(ctx,
                                            &args->f_id, &dyn_dir, lc.tcp);
                                }
                                if (s == NULL ||
                                    (s->st_type == O_REDIRECT &&
                                     (args->eh != NULL ||
                                      (ip->ip_off & (IP_MF | IP_OFFMASK)) ||
                                      (lc.proto != IPPROTO_TCP &&
                                       lc.proto != IPPROTO_UDP)))) {
                                        /*
                                         * State not found. If CHECK_STATE,
                                         * skip to next rule, if PROBE_STATE
                                         * just ignore and continue with next
                                         * opcode.
                                         */
                                        if (cmd->opcode == O_CHECK_STATE)
                                                goto next_rule;
                                        match = 1;
                                        break;
                                }

                                s->st_pcnt++;
                                s->st_bcnt += lc.ip_len;

                                if (s->st_type == O_REDIRECT) {
                                        struct ipfw_xlat *x =
                                            (struct ipfw_xlat *)s;

                                        if (oif != NULL &&
                                            x->xlat_ifp == NULL) {
                                                KASSERT(x->xlat_flags &
                                                    IPFW_STATE_F_XLATSLAVE,
                                                    ("master rdr state "
                                                     "missing ifp"));
                                                x->xlat_ifp = oif;
                                        } else if (
                                            (oif != NULL && x->xlat_ifp!=oif) ||
                                            (oif == NULL &&
                                             x->xlat_ifp!=m->m_pkthdr.rcvif)) {
                                                retval = IP_FW_DENY;
                                                goto done;
                                        }
                                        if (x->xlat_dir != dyn_dir)
                                                goto skip_xlate;

                                        ipfw_xlate(x, m, NULL, NULL);
                                        m = ipfw_rehashm(m, hlen, args, &lc,
                                            &ip);
                                        if (m == NULL)
                                                goto pullup_failed;

                                        cpuid = netisr_hashcpu(
                                            m->m_pkthdr.hash);
                                        if (cpuid != mycpuid) {
                                                uint32_t xlate = 0;

                                                if (oif != NULL) {
                                                        xlate |=
                                                            IPFW_XLATE_OUTPUT;
                                                }
                                                if (dyn_dir == MATCH_FORWARD) {
                                                        xlate |=
                                                            IPFW_XLATE_FORWARD;
                                                }
                                                ipfw_xlate_redispatch(m, cpuid,
                                                    x, xlate);
                                                args->m = NULL;
                                                return (IP_FW_REDISPATCH);
                                        }

                                        KKASSERT(x->xlat_pcpu == mycpuid);
                                        ipfw_state_update(&args->f_id, dyn_dir,
                                            lc.tcp, &x->xlat_pair->xlat_st);
                                }
skip_xlate:
                                /*
                                 * Found a rule from a state; jump to the
                                 * 'action' part of the rule.
                                 */
                                f = s->st_rule;
                                KKASSERT(f->cpuid == mycpuid);

                                cmd = ACTION_PTR(f);
                                l = f->cmd_len - f->act_ofs;
                                dyn_f = f;
                                goto check_body;

                        case O_ACCEPT:
                                retval = IP_FW_PASS;    /* accept */
                                goto done;

                        case O_DEFRAG:
                                if (f->cross_rules == NULL) {
                                        /*
                                         * This rule was not completely setup;
                                         * move on to the next rule.
                                         */
                                        goto next_rule;
                                }

                                /*
                                 * Don't defrag for l2 packets, output packets
                                 * or non-fragments.
                                 */
                                if (oif != NULL || args->eh != NULL ||
                                    (ip->ip_off & (IP_MF | IP_OFFMASK)) == 0)
                                        goto next_rule;

                                ctx->ipfw_frags++;
                                m = ip_reass(m);
                                args->m = m;
                                if (m == NULL) {
                                        retval = IP_FW_PASS;
                                        goto done;
                                }
                                ctx->ipfw_defraged++;
                                KASSERT((m->m_flags & M_HASH) == 0,
                                    ("hash not cleared"));

                                /* Update statistics */
                                f->pcnt++;
                                f->bcnt += lc.ip_len;
                                f->timestamp = time_second;

                                ip = mtod(m, struct ip *);
                                hlen = ip->ip_hl << 2;
                                ip->ip_len += hlen;

                                ip->ip_len = htons(ip->ip_len);
                                ip->ip_off = htons(ip->ip_off);

                                ip_hashfn(&m, 0);
                                args->m = m;
                                if (m == NULL)
                                        goto pullup_failed;

                                KASSERT(m->m_flags & M_HASH, ("no hash"));
                                cpuid = netisr_hashcpu(m->m_pkthdr.hash);
                                if (cpuid != mycpuid) {
                                        /*
                                         * NOTE:
                                         * ip_len/ip_off are in network byte
                                         * order.
                                         */
                                        ctx->ipfw_defrag_remote++;
                                        ipfw_defrag_redispatch(m, cpuid, f);
                                        args->m = NULL;
                                        return (IP_FW_REDISPATCH);
                                }

                                /* 'm' might be changed by ip_hashfn(). */
                                ip = mtod(m, struct ip *);
                                ip->ip_len = ntohs(ip->ip_len);
                                ip->ip_off = ntohs(ip->ip_off);

                                m = ipfw_setup_local(m, hlen, args, &lc, &ip);
                                if (m == NULL)
                                        goto pullup_failed;

                                /* Move on. */
                                goto next_rule;

                        case O_PIPE:
                        case O_QUEUE:
                                args->rule = f; /* report matching rule */
                                args->cookie = cmd->arg1;
                                retval = IP_FW_DUMMYNET;
                                goto done;

                        case O_DIVERT:
                        case O_TEE:
                                if (args->eh) /* not on layer 2 */
                                        break;

                                mtag = m_tag_get(PACKET_TAG_IPFW_DIVERT,
                                    sizeof(*divinfo), M_INTWAIT | M_NULLOK);
                                if (mtag == NULL) {
                                        retval = IP_FW_DENY;
                                        goto done;
                                }
                                divinfo = m_tag_data(mtag);

                                divinfo->skipto = f->rulenum;
                                divinfo->port = cmd->arg1;
                                divinfo->tee = (cmd->opcode == O_TEE);
                                m_tag_prepend(m, mtag);

                                args->cookie = cmd->arg1;
                                retval = (cmd->opcode == O_DIVERT) ?
                                         IP_FW_DIVERT : IP_FW_TEE;
                                goto done;

                        case O_COUNT:
                        case O_SKIPTO:
                                f->pcnt++;      /* update stats */
                                f->bcnt += lc.ip_len;
                                f->timestamp = time_second;
                                if (cmd->opcode == O_COUNT)
                                        goto next_rule;
                                /* handle skipto */
                                if (f->next_rule == NULL)
                                        lookup_next_rule(f);
                                f = f->next_rule;
                                goto again;

                        case O_REJECT:
                                /*
                                 * Drop the packet and send a reject notice
                                 * if the packet is not ICMP (or is an ICMP
                                 * query), and it is not multicast/broadcast.
                                 */
                                if (hlen > 0 &&
                                    (lc.proto != IPPROTO_ICMP ||
                                     is_icmp_query(ip)) &&
                                    !(m->m_flags & (M_BCAST|M_MCAST)) &&
                                    !IN_MULTICAST(ntohl(lc.dst_ip.s_addr))) {
                                        send_reject(args, cmd->arg1,
                                            lc.offset, lc.ip_len);
                                        retval = IP_FW_DENY;
                                        goto done;
                                }
                                /* FALLTHROUGH */
                        case O_DENY:
                                retval = IP_FW_DENY;
                                goto done;

                        case O_FORWARD_IP:
                                if (args->eh)   /* not valid on layer2 pkts */
                                        break;
                                if (!dyn_f || dyn_dir == MATCH_FORWARD) {
                                        struct sockaddr_in *sin;

                                        mtag = m_tag_get(PACKET_TAG_IPFORWARD,
                                            sizeof(*sin), M_INTWAIT | M_NULLOK);
                                        if (mtag == NULL) {
                                                retval = IP_FW_DENY;
                                                goto done;
                                        }
                                        sin = m_tag_data(mtag);

                                        /* Structure copy */
                                        *sin = ((ipfw_insn_sa *)cmd)->sa;

                                        m_tag_prepend(m, mtag);
                                        m->m_pkthdr.fw_flags |=
                                                IPFORWARD_MBUF_TAGGED;
                                        m->m_pkthdr.fw_flags &=
                                                ~BRIDGE_MBUF_TAGGED;
                                }
                                retval = IP_FW_PASS;
                                goto done;

                        default:
                                panic("-- unknown opcode %d", cmd->opcode);
                        } /* end of switch() on opcodes */

                        if (cmd->len & F_NOT)
                                match = !match;

                        if (match) {
                                if (cmd->len & F_OR)
                                        skip_or = 1;
                        } else {
                                if (!(cmd->len & F_OR)) /* not an OR block, */
                                        break;          /* try next rule    */
                        }

                }       /* end of inner for, scan opcodes */

next_rule:;             /* try next rule                */

        }               /* end of outer for, scan rules */
        kprintf("+++ ipfw: ouch!, skip past end of rules, denying packet\n");
        return IP_FW_DENY;

done:
        /* Update statistics */
        f->pcnt++;
        f->bcnt += lc.ip_len;
        f->timestamp = time_second;
        return retval;

pullup_failed:
        if (fw_verbose)
                kprintf("pullup failed\n");
        return IP_FW_DENY;
}

static struct mbuf *
ipfw_dummynet_io(struct mbuf *m, int pipe_nr, int dir, struct ip_fw_args *fwa)
{
        struct m_tag *mtag;
        struct dn_pkt *pkt;
        ipfw_insn *cmd;
        const struct ipfw_flow_id *id;
        struct dn_flow_id *fid;

        M_ASSERTPKTHDR(m);

        mtag = m_tag_get(PACKET_TAG_DUMMYNET, sizeof(*pkt),
            M_INTWAIT | M_NULLOK);
        if (mtag == NULL) {
                m_freem(m);
                return (NULL);
        }
        m_tag_prepend(m, mtag);

        pkt = m_tag_data(mtag);
        bzero(pkt, sizeof(*pkt));

        cmd = fwa->rule->cmd + fwa->rule->act_ofs;
        if (cmd->opcode == O_LOG)
                cmd += F_LEN(cmd);
        KASSERT(cmd->opcode == O_PIPE || cmd->opcode == O_QUEUE,
                ("Rule is not PIPE or QUEUE, opcode %d", cmd->opcode));

        pkt->dn_m = m;
        pkt->dn_flags = (dir & DN_FLAGS_DIR_MASK);
        pkt->ifp = fwa->oif;
        pkt->pipe_nr = pipe_nr;

        pkt->cpuid = mycpuid;
        pkt->msgport = netisr_curport();

        id = &fwa->f_id;
        fid = &pkt->id;
        fid->fid_dst_ip = id->dst_ip;
        fid->fid_src_ip = id->src_ip;
        fid->fid_dst_port = id->dst_port;
        fid->fid_src_port = id->src_port;
        fid->fid_proto = id->proto;
        fid->fid_flags = id->flags;

        ipfw_ref_rule(fwa->rule);
        pkt->dn_priv = fwa->rule;
        pkt->dn_unref_priv = ipfw_unref_rule;

        if (cmd->opcode == O_PIPE)
                pkt->dn_flags |= DN_FLAGS_IS_PIPE;

        m->m_pkthdr.fw_flags |= DUMMYNET_MBUF_TAGGED;
        return (m);
}

/*
 * When a rule is added/deleted, clear the next_rule pointers in all rules.
 * These will be reconstructed on the fly as packets are matched.
 */
static void
ipfw_flush_rule_ptrs(struct ipfw_context *ctx)
{
        struct ip_fw *rule;

        for (rule = ctx->ipfw_layer3_chain; rule; rule = rule->next)
                rule->next_rule = NULL;
}

static void
ipfw_inc_static_count(struct ip_fw *rule)
{
        /* Static rule's counts are updated only on CPU0 */
        KKASSERT(mycpuid == 0);

        static_count++;
        static_ioc_len += IOC_RULESIZE(rule);
}

static void
ipfw_dec_static_count(struct ip_fw *rule)
{
        int l = IOC_RULESIZE(rule);

        /* Static rule's counts are updated only on CPU0 */
        KKASSERT(mycpuid == 0);

        KASSERT(static_count > 0, ("invalid static count %u", static_count));
        static_count--;

        KASSERT(static_ioc_len >= l,
                ("invalid static len %u", static_ioc_len));
        static_ioc_len -= l;
}

static void
ipfw_link_sibling(struct netmsg_ipfw *fwmsg, struct ip_fw *rule)
{
        if (fwmsg->sibling != NULL) {
                KKASSERT(mycpuid > 0 && fwmsg->sibling->cpuid == mycpuid - 1);
                fwmsg->sibling->sibling = rule;
        }
        fwmsg->sibling = rule;
}

static struct ip_fw *
ipfw_create_rule(const struct ipfw_ioc_rule *ioc_rule, uint32_t rule_flags)
{
        struct ip_fw *rule;

        rule = kmalloc(RULESIZE(ioc_rule), M_IPFW, M_WAITOK | M_ZERO);

        rule->act_ofs = ioc_rule->act_ofs;
        rule->cmd_len = ioc_rule->cmd_len;
        rule->rulenum = ioc_rule->rulenum;
        rule->set = ioc_rule->set;
        rule->usr_flags = ioc_rule->usr_flags;

        bcopy(ioc_rule->cmd, rule->cmd, rule->cmd_len * 4 /* XXX */);

        rule->refcnt = 1;
        rule->cpuid = mycpuid;
        rule->rule_flags = rule_flags;

        return rule;
}

static void
ipfw_add_rule_dispatch(netmsg_t nmsg)
{
        struct netmsg_ipfw *fwmsg = (struct netmsg_ipfw *)nmsg;
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct ip_fw *rule;

        ASSERT_NETISR_NCPUS(mycpuid);

        rule = ipfw_create_rule(fwmsg->ioc_rule, fwmsg->rule_flags);

        /*
         * Insert rule into the pre-determined position
         */
        if (fwmsg->prev_rule != NULL) {
                struct ip_fw *prev, *next;

                prev = fwmsg->prev_rule;
                KKASSERT(prev->cpuid == mycpuid);

                next = fwmsg->next_rule;
                KKASSERT(next->cpuid == mycpuid);

                rule->next = next;
                prev->next = rule;

                /*
                 * Move to the position on the next CPU
                 * before the msg is forwarded.
                 */
                fwmsg->prev_rule = prev->sibling;
                fwmsg->next_rule = next->sibling;
        } else {
                KKASSERT(fwmsg->next_rule == NULL);
                rule->next = ctx->ipfw_layer3_chain;
                ctx->ipfw_layer3_chain = rule;
        }

        /* Link rule CPU sibling */
        ipfw_link_sibling(fwmsg, rule);

        ipfw_flush_rule_ptrs(ctx);

        if (mycpuid == 0) {
                /* Statistics only need to be updated once */
                ipfw_inc_static_count(rule);

                /* Return the rule on CPU0 */
                nmsg->lmsg.u.ms_resultp = rule;
        }

        if (rule->rule_flags & IPFW_RULE_F_GENTRACK)
                rule->track_ruleid = (uintptr_t)nmsg->lmsg.u.ms_resultp;

        if (fwmsg->cross_rules != NULL) {
                /* Save rules for later use. */
                fwmsg->cross_rules[mycpuid] = rule;
        }

        netisr_forwardmsg(&nmsg->base, mycpuid + 1);
}

static void
ipfw_crossref_rule_dispatch(netmsg_t nmsg)
{
        struct netmsg_ipfw *fwmsg = (struct netmsg_ipfw *)nmsg;
        struct ip_fw *rule = fwmsg->sibling;
        int sz = sizeof(struct ip_fw *) * netisr_ncpus;

        ASSERT_NETISR_NCPUS(mycpuid);
        KASSERT(rule->rule_flags & IPFW_RULE_F_CROSSREF,
            ("not crossref rule"));

        rule->cross_rules = kmalloc(sz, M_IPFW, M_WAITOK);
        memcpy(rule->cross_rules, fwmsg->cross_rules, sz);

        fwmsg->sibling = rule->sibling;
        netisr_forwardmsg(&fwmsg->base, mycpuid + 1);
}

/*
 * Add a new rule to the list.  Copy the rule into a malloc'ed area,
 * then possibly create a rule number and add the rule to the list.
 * Update the rule_number in the input struct so the caller knows
 * it as well.
 */
static void
ipfw_add_rule(struct ipfw_ioc_rule *ioc_rule, uint32_t rule_flags)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct netmsg_ipfw fwmsg;
        struct ip_fw *f, *prev, *rule;

        ASSERT_NETISR0;

        /*
         * If rulenum is 0, find highest numbered rule before the
         * default rule, and add rule number incremental step.
         */
        if (ioc_rule->rulenum == 0) {
                int step = autoinc_step;

                KKASSERT(step >= IPFW_AUTOINC_STEP_MIN &&
                         step <= IPFW_AUTOINC_STEP_MAX);

                /*
                 * Locate the highest numbered rule before default
                 */
                for (f = ctx->ipfw_layer3_chain; f; f = f->next) {
                        if (f->rulenum == IPFW_DEFAULT_RULE)
                                break;
                        ioc_rule->rulenum = f->rulenum;
                }
                if (ioc_rule->rulenum < IPFW_DEFAULT_RULE - step)
                        ioc_rule->rulenum += step;
        }
        KASSERT(ioc_rule->rulenum != IPFW_DEFAULT_RULE &&
                ioc_rule->rulenum != 0,
                ("invalid rule num %d", ioc_rule->rulenum));

        /*
         * Now find the right place for the new rule in the sorted list.
         */
        for (prev = NULL, f = ctx->ipfw_layer3_chain; f;
             prev = f, f = f->next) {
                if (f->rulenum > ioc_rule->rulenum) {
                        /* Found the location */
                        break;
                }
        }
        KASSERT(f != NULL, ("no default rule?!"));

        /*
         * Duplicate the rule onto each CPU.
         * The rule duplicated on CPU0 will be returned.
         */
        bzero(&fwmsg, sizeof(fwmsg));
        netmsg_init(&fwmsg.base, NULL, &curthread->td_msgport, MSGF_PRIORITY,
            ipfw_add_rule_dispatch);
        fwmsg.ioc_rule = ioc_rule;
        fwmsg.prev_rule = prev;
        fwmsg.next_rule = prev == NULL ? NULL : f;
        fwmsg.rule_flags = rule_flags;
        if (rule_flags & IPFW_RULE_F_CROSSREF) {
                fwmsg.cross_rules = kmalloc(
                    sizeof(struct ip_fw *) * netisr_ncpus, M_TEMP,
                    M_WAITOK | M_ZERO);
        }

        netisr_domsg_global(&fwmsg.base);
        KKASSERT(fwmsg.prev_rule == NULL && fwmsg.next_rule == NULL);

        rule = fwmsg.base.lmsg.u.ms_resultp;
        KKASSERT(rule != NULL && rule->cpuid == mycpuid);

        if (fwmsg.cross_rules != NULL) {
                netmsg_init(&fwmsg.base, NULL, &curthread->td_msgport,
                    MSGF_PRIORITY, ipfw_crossref_rule_dispatch);
                fwmsg.sibling = rule;
                netisr_domsg_global(&fwmsg.base);
                KKASSERT(fwmsg.sibling == NULL);

                kfree(fwmsg.cross_rules, M_TEMP);

#ifdef KLD_MODULE
                atomic_add_int(&ipfw_gd.ipfw_refcnt, 1);
#endif
        }

        DPRINTF("++ installed rule %d, static count now %d\n",
                rule->rulenum, static_count);
}

/*
 * Free storage associated with a static rule (including derived
 * states/tracks).
 * The caller is in charge of clearing rule pointers to avoid
 * dangling pointers.
 * @return a pointer to the next entry.
 * Arguments are not checked, so they better be correct.
 */
static struct ip_fw *
ipfw_delete_rule(struct ipfw_context *ctx,
                 struct ip_fw *prev, struct ip_fw *rule)
{
        struct ip_fw *n;

        n = rule->next;
        if (prev == NULL)
                ctx->ipfw_layer3_chain = n;
        else
                prev->next = n;

        /* Mark the rule as invalid */
        rule->rule_flags |= IPFW_RULE_F_INVALID;
        rule->next_rule = NULL;
        rule->sibling = NULL;
#ifdef foo
        /* Don't reset cpuid here; keep various assertion working */
        rule->cpuid = -1;
#endif

        /* Statistics only need to be updated once */
        if (mycpuid == 0)
                ipfw_dec_static_count(rule);

        if ((rule->rule_flags & IPFW_RULE_F_CROSSREF) == 0) {
                /* Try to free this rule */
                ipfw_free_rule(rule);
        } else {
                /* TODO: check staging area. */
                if (mycpuid == 0) {
                        rule->next = ipfw_gd.ipfw_crossref_free;
                        ipfw_gd.ipfw_crossref_free = rule;
                }
        }

        /* Return the next rule */
        return n;
}

static void
ipfw_flush_dispatch(netmsg_t nmsg)
{
        int kill_default = nmsg->lmsg.u.ms_result;
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct ip_fw *rule;

        ASSERT_NETISR_NCPUS(mycpuid);

        /*
         * Flush states.
         */
        ipfw_state_flush(ctx, NULL);
        KASSERT(ctx->ipfw_state_cnt == 0,
            ("%d pcpu states remain", ctx->ipfw_state_cnt));
        ctx->ipfw_state_loosecnt = 0;
        ctx->ipfw_state_lastexp = 0;

        /*
         * Flush tracks.
         */
        ipfw_track_flush(ctx, NULL);
        ctx->ipfw_track_lastexp = 0;
        if (ctx->ipfw_trkcnt_spare != NULL) {
                kfree(ctx->ipfw_trkcnt_spare, M_IPFW);
                ctx->ipfw_trkcnt_spare = NULL;
        }

        ipfw_flush_rule_ptrs(ctx); /* more efficient to do outside the loop */

        while ((rule = ctx->ipfw_layer3_chain) != NULL &&
               (kill_default || rule->rulenum != IPFW_DEFAULT_RULE))
                ipfw_delete_rule(ctx, NULL, rule);

        netisr_forwardmsg(&nmsg->base, mycpuid + 1);
}

/*
 * Deletes all rules from a chain (including the default rule
 * if the second argument is set).
 */
static void
ipfw_flush(int kill_default)
{
        struct netmsg_base nmsg;
#ifdef INVARIANTS
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        int state_cnt;
#endif

        ASSERT_NETISR0;

        /*
         * If 'kill_default' then caller has done the necessary
         * msgport syncing; unnecessary to do it again.
         */
        if (!kill_default) {
                /*
                 * Let ipfw_chk() know the rules are going to
                 * be flushed, so it could jump directly to
                 * the default rule.
                 */
                ipfw_flushing = 1;
                /* XXX use priority sync */
                netmsg_service_sync();
        }

        /*
         * Press the 'flush' button
         */
        bzero(&nmsg, sizeof(nmsg));
        netmsg_init(&nmsg, NULL, &curthread->td_msgport, MSGF_PRIORITY,
            ipfw_flush_dispatch);
        nmsg.lmsg.u.ms_result = kill_default;
        netisr_domsg_global(&nmsg);
        ipfw_gd.ipfw_state_loosecnt = 0;
        ipfw_gd.ipfw_state_globexp = 0;
        ipfw_gd.ipfw_track_globexp = 0;

#ifdef INVARIANTS
        state_cnt = ipfw_state_cntcoll();
        KASSERT(state_cnt == 0, ("%d states remain", state_cnt));

        KASSERT(ipfw_gd.ipfw_trkcnt_cnt == 0,
            ("%d trkcnts remain", ipfw_gd.ipfw_trkcnt_cnt));

        if (kill_default) {
                KASSERT(static_count == 0,
                        ("%u static rules remain", static_count));
                KASSERT(static_ioc_len == 0,
                        ("%u bytes of static rules remain", static_ioc_len));
        } else {
                KASSERT(static_count == 1,
                        ("%u static rules remain", static_count));
                KASSERT(static_ioc_len == IOC_RULESIZE(ctx->ipfw_default_rule),
                        ("%u bytes of static rules remain, should be %lu",
                         static_ioc_len,
                         (u_long)IOC_RULESIZE(ctx->ipfw_default_rule)));
        }
#endif

        /* Flush is done */
        ipfw_flushing = 0;
}

static void
ipfw_alt_delete_rule_dispatch(netmsg_t nmsg)
{
        struct netmsg_del *dmsg = (struct netmsg_del *)nmsg;
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct ip_fw *rule, *prev;

        ASSERT_NETISR_NCPUS(mycpuid);

        rule = dmsg->start_rule;
        KKASSERT(rule->cpuid == mycpuid);
        dmsg->start_rule = rule->sibling;

        prev = dmsg->prev_rule;
        if (prev != NULL) {
                KKASSERT(prev->cpuid == mycpuid);

                /*
                 * Move to the position on the next CPU
                 * before the msg is forwarded.
                 */
                dmsg->prev_rule = prev->sibling;
        }

        /*
         * flush pointers outside the loop, then delete all matching
         * rules.  'prev' remains the same throughout the cycle.
         */
        ipfw_flush_rule_ptrs(ctx);
        while (rule && rule->rulenum == dmsg->rulenum) {
                if (rule->rule_flags & IPFW_RULE_F_GENSTATE) {
                        /* Flush states generated by this rule. */
                        ipfw_state_flush(ctx, rule);
                }
                if (rule->rule_flags & IPFW_RULE_F_GENTRACK) {
                        /* Flush tracks generated by this rule. */
                        ipfw_track_flush(ctx, rule);
                }
                rule = ipfw_delete_rule(ctx, prev, rule);
        }

        netisr_forwardmsg(&nmsg->base, mycpuid + 1);
}

static int
ipfw_alt_delete_rule(uint16_t rulenum)
{
        struct ip_fw *prev, *rule;
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct netmsg_del dmsg;

        ASSERT_NETISR0;

        /*
         * Locate first rule to delete
         */
        for (prev = NULL, rule = ctx->ipfw_layer3_chain;
             rule && rule->rulenum < rulenum;
             prev = rule, rule = rule->next)
                ; /* EMPTY */
        if (rule->rulenum != rulenum)
                return EINVAL;

        /*
         * Get rid of the rule duplications on all CPUs
         */
        bzero(&dmsg, sizeof(dmsg));
        netmsg_init(&dmsg.base, NULL, &curthread->td_msgport, MSGF_PRIORITY,
            ipfw_alt_delete_rule_dispatch);
        dmsg.prev_rule = prev;
        dmsg.start_rule = rule;
        dmsg.rulenum = rulenum;

        netisr_domsg_global(&dmsg.base);
        KKASSERT(dmsg.prev_rule == NULL && dmsg.start_rule == NULL);
        return 0;
}

static void
ipfw_alt_delete_ruleset_dispatch(netmsg_t nmsg)
{
        struct netmsg_del *dmsg = (struct netmsg_del *)nmsg;
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct ip_fw *prev, *rule;
#ifdef INVARIANTS
        int del = 0;
#endif

        ASSERT_NETISR_NCPUS(mycpuid);

        ipfw_flush_rule_ptrs(ctx);

        prev = NULL;
        rule = ctx->ipfw_layer3_chain;
        while (rule != NULL) {
                if (rule->set == dmsg->from_set) {
                        if (rule->rule_flags & IPFW_RULE_F_GENSTATE) {
                                /* Flush states generated by this rule. */
                                ipfw_state_flush(ctx, rule);
                        }
                        if (rule->rule_flags & IPFW_RULE_F_GENTRACK) {
                                /* Flush tracks generated by this rule. */
                                ipfw_track_flush(ctx, rule);
                        }
                        rule = ipfw_delete_rule(ctx, prev, rule);
#ifdef INVARIANTS
                        del = 1;
#endif
                } else {
                        prev = rule;
                        rule = rule->next;
                }
        }
        KASSERT(del, ("no match set?!"));

        netisr_forwardmsg(&nmsg->base, mycpuid + 1);
}

static int
ipfw_alt_delete_ruleset(uint8_t set)
{
        struct netmsg_del dmsg;
        int del;
        struct ip_fw *rule;
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];

        ASSERT_NETISR0;

        /*
         * Check whether the 'set' exists.  If it exists,
         * then check whether any rules within the set will
         * try to create states.
         */
        del = 0;
        for (rule = ctx->ipfw_layer3_chain; rule; rule = rule->next) {
                if (rule->set == set)
                        del = 1;
        }
        if (!del)
                return 0; /* XXX EINVAL? */

        /*
         * Delete this set
         */
        bzero(&dmsg, sizeof(dmsg));
        netmsg_init(&dmsg.base, NULL, &curthread->td_msgport, MSGF_PRIORITY,
            ipfw_alt_delete_ruleset_dispatch);
        dmsg.from_set = set;
        netisr_domsg_global(&dmsg.base);

        return 0;
}

static void
ipfw_alt_move_rule_dispatch(netmsg_t nmsg)
{
        struct netmsg_del *dmsg = (struct netmsg_del *)nmsg;
        struct ip_fw *rule;

        ASSERT_NETISR_NCPUS(mycpuid);

        rule = dmsg->start_rule;
        KKASSERT(rule->cpuid == mycpuid);

        /*
         * Move to the position on the next CPU
         * before the msg is forwarded.
         */
        dmsg->start_rule = rule->sibling;

        while (rule && rule->rulenum <= dmsg->rulenum) {
                if (rule->rulenum == dmsg->rulenum)
                        rule->set = dmsg->to_set;
                rule = rule->next;
        }
        netisr_forwardmsg(&nmsg->base, mycpuid + 1);
}

static int
ipfw_alt_move_rule(uint16_t rulenum, uint8_t set)
{
        struct netmsg_del dmsg;
        struct netmsg_base *nmsg;
        struct ip_fw *rule;
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];

        ASSERT_NETISR0;

        /*
         * Locate first rule to move
         */
        for (rule = ctx->ipfw_layer3_chain; rule && rule->rulenum <= rulenum;
             rule = rule->next) {
                if (rule->rulenum == rulenum && rule->set != set)
                        break;
        }
        if (rule == NULL || rule->rulenum > rulenum)
                return 0; /* XXX error? */

        bzero(&dmsg, sizeof(dmsg));
        nmsg = &dmsg.base;
        netmsg_init(nmsg, NULL, &curthread->td_msgport, MSGF_PRIORITY,
            ipfw_alt_move_rule_dispatch);
        dmsg.start_rule = rule;
        dmsg.rulenum = rulenum;
        dmsg.to_set = set;

        netisr_domsg_global(nmsg);
        KKASSERT(dmsg.start_rule == NULL);
        return 0;
}

static void
ipfw_alt_move_ruleset_dispatch(netmsg_t nmsg)
{
        struct netmsg_del *dmsg = (struct netmsg_del *)nmsg;
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct ip_fw *rule;

        ASSERT_NETISR_NCPUS(mycpuid);

        for (rule = ctx->ipfw_layer3_chain; rule; rule = rule->next) {
                if (rule->set == dmsg->from_set)
                        rule->set = dmsg->to_set;
        }
        netisr_forwardmsg(&nmsg->base, mycpuid + 1);
}

static int
ipfw_alt_move_ruleset(uint8_t from_set, uint8_t to_set)
{
        struct netmsg_del dmsg;
        struct netmsg_base *nmsg;

        ASSERT_NETISR0;

        bzero(&dmsg, sizeof(dmsg));
        nmsg = &dmsg.base;
        netmsg_init(nmsg, NULL, &curthread->td_msgport, MSGF_PRIORITY,
            ipfw_alt_move_ruleset_dispatch);
        dmsg.from_set = from_set;
        dmsg.to_set = to_set;

        netisr_domsg_global(nmsg);
        return 0;
}

static void
ipfw_alt_swap_ruleset_dispatch(netmsg_t nmsg)
{
        struct netmsg_del *dmsg = (struct netmsg_del *)nmsg;
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct ip_fw *rule;

        ASSERT_NETISR_NCPUS(mycpuid);

        for (rule = ctx->ipfw_layer3_chain; rule; rule = rule->next) {
                if (rule->set == dmsg->from_set)
                        rule->set = dmsg->to_set;
                else if (rule->set == dmsg->to_set)
                        rule->set = dmsg->from_set;
        }
        netisr_forwardmsg(&nmsg->base, mycpuid + 1);
}

static int
ipfw_alt_swap_ruleset(uint8_t set1, uint8_t set2)
{
        struct netmsg_del dmsg;
        struct netmsg_base *nmsg;

        ASSERT_NETISR0;

        bzero(&dmsg, sizeof(dmsg));
        nmsg = &dmsg.base;
        netmsg_init(nmsg, NULL, &curthread->td_msgport, MSGF_PRIORITY,
            ipfw_alt_swap_ruleset_dispatch);
        dmsg.from_set = set1;
        dmsg.to_set = set2;

        netisr_domsg_global(nmsg);
        return 0;
}

/*
 * Remove all rules with given number, and also do set manipulation.
 *
 * The argument is an uint32_t. The low 16 bit are the rule or set number,
 * the next 8 bits are the new set, the top 8 bits are the command:
 *
 *      0       delete rules with given number
 *      1       delete rules with given set number
 *      2       move rules with given number to new set
 *      3       move rules with given set number to new set
 *      4       swap sets with given numbers
 */
static int
ipfw_ctl_alter(uint32_t arg)
{
        uint16_t rulenum;
        uint8_t cmd, new_set;
        int error = 0;

        ASSERT_NETISR0;

        rulenum = arg & 0xffff;
        cmd = (arg >> 24) & 0xff;
        new_set = (arg >> 16) & 0xff;

        if (cmd > 4)
                return EINVAL;
        if (new_set >= IPFW_DEFAULT_SET)
                return EINVAL;
        if (cmd == 0 || cmd == 2) {
                if (rulenum == IPFW_DEFAULT_RULE)
                        return EINVAL;
        } else {
                if (rulenum >= IPFW_DEFAULT_SET)
                        return EINVAL;
        }

        switch (cmd) {
        case 0: /* delete rules with given number */
                error = ipfw_alt_delete_rule(rulenum);
                break;

        case 1: /* delete all rules with given set number */
                error = ipfw_alt_delete_ruleset(rulenum);
                break;

        case 2: /* move rules with given number to new set */
                error = ipfw_alt_move_rule(rulenum, new_set);
                break;

        case 3: /* move rules with given set number to new set */
                error = ipfw_alt_move_ruleset(rulenum, new_set);
                break;

        case 4: /* swap two sets */
                error = ipfw_alt_swap_ruleset(rulenum, new_set);
                break;
        }
        return error;
}

/*
 * Clear counters for a specific rule.
 */
static void
clear_counters(struct ip_fw *rule, int log_only)
{
        ipfw_insn_log *l = (ipfw_insn_log *)ACTION_PTR(rule);

        if (log_only == 0) {
                rule->bcnt = rule->pcnt = 0;
                rule->timestamp = 0;
        }
        if (l->o.opcode == O_LOG)
                l->log_left = l->max_log;
}

static void
ipfw_zero_entry_dispatch(netmsg_t nmsg)
{
        struct netmsg_zent *zmsg = (struct netmsg_zent *)nmsg;
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct ip_fw *rule;

        ASSERT_NETISR_NCPUS(mycpuid);

        if (zmsg->rulenum == 0) {
                KKASSERT(zmsg->start_rule == NULL);

                ctx->ipfw_norule_counter = 0;
                for (rule = ctx->ipfw_layer3_chain; rule; rule = rule->next)
                        clear_counters(rule, zmsg->log_only);
        } else {
                struct ip_fw *start = zmsg->start_rule;

                KKASSERT(start->cpuid == mycpuid);
                KKASSERT(start->rulenum == zmsg->rulenum);

                /*
                 * We can have multiple rules with the same number, so we
                 * need to clear them all.
                 */
                for (rule = start; rule && rule->rulenum == zmsg->rulenum;
                     rule = rule->next)
                        clear_counters(rule, zmsg->log_only);

                /*
                 * Move to the position on the next CPU
                 * before the msg is forwarded.
                 */
                zmsg->start_rule = start->sibling;
        }
        netisr_forwardmsg(&nmsg->base, mycpuid + 1);
}

/*
 * Reset some or all counters on firewall rules.
 * @arg frwl is null to clear all entries, or contains a specific
 * rule number.
 * @arg log_only is 1 if we only want to reset logs, zero otherwise.
 */
static int
ipfw_ctl_zero_entry(int rulenum, int log_only)
{
        struct netmsg_zent zmsg;
        struct netmsg_base *nmsg;
        const char *msg;
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];

        ASSERT_NETISR0;

        bzero(&zmsg, sizeof(zmsg));
        nmsg = &zmsg.base;
        netmsg_init(nmsg, NULL, &curthread->td_msgport, MSGF_PRIORITY,
            ipfw_zero_entry_dispatch);
        zmsg.log_only = log_only;

        if (rulenum == 0) {
                msg = log_only ? "ipfw: All logging counts reset.\n"
                               : "ipfw: Accounting cleared.\n";
        } else {
                struct ip_fw *rule;

                /*
                 * Locate the first rule with 'rulenum'
                 */
                for (rule = ctx->ipfw_layer3_chain; rule; rule = rule->next) {
                        if (rule->rulenum == rulenum)
                                break;
                }
                if (rule == NULL) /* we did not find any matching rules */
                        return (EINVAL);
                zmsg.start_rule = rule;
                zmsg.rulenum = rulenum;

                msg = log_only ? "ipfw: Entry %d logging count reset.\n"
                               : "ipfw: Entry %d cleared.\n";
        }
        netisr_domsg_global(nmsg);
        KKASSERT(zmsg.start_rule == NULL);

        if (fw_verbose)
                log(LOG_SECURITY | LOG_NOTICE, msg, rulenum);
        return (0);
}

/*
 * Check validity of the structure before insert.
 * Fortunately rules are simple, so this mostly need to check rule sizes.
 */
static int
ipfw_check_ioc_rule(struct ipfw_ioc_rule *rule, int size, uint32_t *rule_flags)
{
        int l, cmdlen = 0;
        int have_action = 0;
        ipfw_insn *cmd;

        *rule_flags = 0;

        /* Check for valid size */
        if (size < sizeof(*rule)) {
                kprintf("ipfw: rule too short\n");
                return EINVAL;
        }
        l = IOC_RULESIZE(rule);
        if (l != size) {
                kprintf("ipfw: size mismatch (have %d want %d)\n", size, l);
                return EINVAL;
        }

        /* Check rule number */
        if (rule->rulenum == IPFW_DEFAULT_RULE) {
                kprintf("ipfw: invalid rule number\n");
                return EINVAL;
        }

        /*
         * Now go for the individual checks. Very simple ones, basically only
         * instruction sizes.
         */
        for (l = rule->cmd_len, cmd = rule->cmd; l > 0;
             l -= cmdlen, cmd += cmdlen) {
                cmdlen = F_LEN(cmd);
                if (cmdlen > l) {
                        kprintf("ipfw: opcode %d size truncated\n",
                                cmd->opcode);
                        return EINVAL;
                }

                DPRINTF("ipfw: opcode %d\n", cmd->opcode);

                if (cmd->opcode == O_KEEP_STATE || cmd->opcode == O_LIMIT ||
                    IPFW_ISXLAT(cmd->opcode)) {
                        /* This rule will generate states. */
                        *rule_flags |= IPFW_RULE_F_GENSTATE;
                        if (cmd->opcode == O_LIMIT)
                                *rule_flags |= IPFW_RULE_F_GENTRACK;
                }
                if (cmd->opcode == O_DEFRAG || IPFW_ISXLAT(cmd->opcode))
                        *rule_flags |= IPFW_RULE_F_CROSSREF;
                if (cmd->opcode == O_IP_SRC_IFIP ||
                    cmd->opcode == O_IP_DST_IFIP) {
                        *rule_flags |= IPFW_RULE_F_DYNIFADDR;
                        cmd->arg1 &= IPFW_IFIP_SETTINGS;
                }

                switch (cmd->opcode) {
                case O_NOP:
                case O_PROBE_STATE:
                case O_KEEP_STATE:
                case O_PROTO:
                case O_IP_SRC_ME:
                case O_IP_DST_ME:
                case O_LAYER2:
                case O_IN:
                case O_FRAG:
                case O_IPFRAG:
                case O_IPOPT:
                case O_IPLEN:
                case O_IPID:
                case O_IPTOS:
                case O_IPPRECEDENCE:
                case O_IPTTL:
                case O_IPVER:
                case O_TCPWIN:
                case O_TCPFLAGS:
                case O_TCPOPTS:
                case O_ESTAB:
                        if (cmdlen != F_INSN_SIZE(ipfw_insn))
                                goto bad_size;
                        break;

                case O_IP_SRC_TABLE:
                case O_IP_DST_TABLE:
                        if (cmdlen != F_INSN_SIZE(ipfw_insn))
                                goto bad_size;
                        if (cmd->arg1 >= ipfw_table_max) {
                                kprintf("ipfw: invalid table id %u, max %d\n",
                                    cmd->arg1, ipfw_table_max);
                                return EINVAL;
                        }
                        break;

                case O_IP_SRC_IFIP:
                case O_IP_DST_IFIP:
                        if (cmdlen != F_INSN_SIZE(ipfw_insn_ifip))
                                goto bad_size;
                        break;

                case O_ICMPCODE:
                case O_ICMPTYPE:
                        if (cmdlen < F_INSN_SIZE(ipfw_insn_u32))
                                goto bad_size;
                        break;

                case O_UID:
                case O_GID:
                case O_IP_SRC:
                case O_IP_DST:
                case O_TCPSEQ:
                case O_TCPACK:
                case O_PROB:
                        if (cmdlen != F_INSN_SIZE(ipfw_insn_u32))
                                goto bad_size;
                        break;

                case O_LIMIT:
                        if (cmdlen != F_INSN_SIZE(ipfw_insn_limit))
                                goto bad_size;
                        break;
                case O_REDIRECT:
                        if (cmdlen != F_INSN_SIZE(ipfw_insn_rdr))
                                goto bad_size;
                        break;

                case O_LOG:
                        if (cmdlen != F_INSN_SIZE(ipfw_insn_log))
                                goto bad_size;

                        ((ipfw_insn_log *)cmd)->log_left =
                            ((ipfw_insn_log *)cmd)->max_log;

                        break;

                case O_IP_SRC_MASK:
                case O_IP_DST_MASK:
                        if (cmdlen != F_INSN_SIZE(ipfw_insn_ip))
                                goto bad_size;
                        if (((ipfw_insn_ip *)cmd)->mask.s_addr == 0) {
                                kprintf("ipfw: opcode %d, useless rule\n",
                                        cmd->opcode);
                                return EINVAL;
                        }
                        break;

                case O_IP_SRC_SET:
                case O_IP_DST_SET:
                        if (cmd->arg1 == 0 || cmd->arg1 > 256) {
                                kprintf("ipfw: invalid set size %d\n",
                                        cmd->arg1);
                                return EINVAL;
                        }
                        if (cmdlen != F_INSN_SIZE(ipfw_insn_u32) +
                            (cmd->arg1+31)/32 )
                                goto bad_size;
                        break;

                case O_MACADDR2:
                        if (cmdlen != F_INSN_SIZE(ipfw_insn_mac))
                                goto bad_size;
                        break;

                case O_MAC_TYPE:
                case O_IP_SRCPORT:
                case O_IP_DSTPORT: /* XXX artificial limit, 30 port pairs */
                        if (cmdlen < 2 || cmdlen > 31)
                                goto bad_size;
                        break;

                case O_RECV:
                case O_XMIT:
                case O_VIA:
                        if (cmdlen != F_INSN_SIZE(ipfw_insn_if))
                                goto bad_size;
                        break;

                case O_PIPE:
                case O_QUEUE:
                        if (cmdlen != F_INSN_SIZE(ipfw_insn_pipe))
                                goto bad_size;
                        goto check_action;

                case O_FORWARD_IP:
                        if (cmdlen != F_INSN_SIZE(ipfw_insn_sa)) {
                                goto bad_size;
                        } else {
                                in_addr_t fwd_addr;

                                fwd_addr = ((ipfw_insn_sa *)cmd)->
                                           sa.sin_addr.s_addr;
                                if (IN_MULTICAST(ntohl(fwd_addr))) {
                                        kprintf("ipfw: try forwarding to "
                                                "multicast address\n");
                                        return EINVAL;
                                }
                        }
                        goto check_action;

                case O_FORWARD_MAC: /* XXX not implemented yet */
                case O_CHECK_STATE:
                case O_COUNT:
                case O_ACCEPT:
                case O_DENY:
                case O_REJECT:
                case O_SKIPTO:
                case O_DIVERT:
                case O_TEE:
                case O_DEFRAG:
                        if (cmdlen != F_INSN_SIZE(ipfw_insn))
                                goto bad_size;
check_action:
                        if (have_action) {
                                kprintf("ipfw: opcode %d, multiple actions"
                                        " not allowed\n",
                                        cmd->opcode);
                                return EINVAL;
                        }
                        have_action = 1;
                        if (l != cmdlen) {
                                kprintf("ipfw: opcode %d, action must be"
                                        " last opcode\n",
                                        cmd->opcode);
                                return EINVAL;
                        }
                        break;
                default:
                        kprintf("ipfw: opcode %d, unknown opcode\n",
                                cmd->opcode);
                        return EINVAL;
                }
        }
        if (have_action == 0) {
                kprintf("ipfw: missing action\n");
                return EINVAL;
        }
        return 0;

bad_size:
        kprintf("ipfw: opcode %d size %d wrong\n",
                cmd->opcode, cmdlen);
        return EINVAL;
}

static int
ipfw_ctl_add_rule(struct sockopt *sopt)
{
        struct ipfw_ioc_rule *ioc_rule;
        size_t size;
        uint32_t rule_flags;
        int error;

        ASSERT_NETISR0;
        
        size = sopt->sopt_valsize;
        if (size > (sizeof(uint32_t) * IPFW_RULE_SIZE_MAX) ||
            size < sizeof(*ioc_rule)) {
                return EINVAL;
        }
        if (size != (sizeof(uint32_t) * IPFW_RULE_SIZE_MAX)) {
                sopt->sopt_val = krealloc(sopt->sopt_val, sizeof(uint32_t) *
                                          IPFW_RULE_SIZE_MAX, M_TEMP, M_WAITOK);
        }
        ioc_rule = sopt->sopt_val;

        error = ipfw_check_ioc_rule(ioc_rule, size, &rule_flags);
        if (error)
                return error;

        ipfw_add_rule(ioc_rule, rule_flags);

        if (sopt->sopt_dir == SOPT_GET)
                sopt->sopt_valsize = IOC_RULESIZE(ioc_rule);
        return 0;
}

static void *
ipfw_copy_rule(const struct ipfw_context *ctx, const struct ip_fw *rule,
    struct ipfw_ioc_rule *ioc_rule)
{
        const struct ip_fw *sibling;
#ifdef INVARIANTS
        int i;
#endif

        ASSERT_NETISR0;
        KASSERT(rule->cpuid == 0, ("rule does not belong to cpu0"));

        ioc_rule->act_ofs = rule->act_ofs;
        ioc_rule->cmd_len = rule->cmd_len;
        ioc_rule->rulenum = rule->rulenum;
        ioc_rule->set = rule->set;
        ioc_rule->usr_flags = rule->usr_flags;

        ioc_rule->set_disable = ctx->ipfw_set_disable;
        ioc_rule->static_count = static_count;
        ioc_rule->static_len = static_ioc_len;

        /*
         * Visit (read-only) all of the rule's duplications to get
         * the necessary statistics
         */
#ifdef INVARIANTS
        i = 0;
#endif
        ioc_rule->pcnt = 0;
        ioc_rule->bcnt = 0;
        ioc_rule->timestamp = 0;
        for (sibling = rule; sibling != NULL; sibling = sibling->sibling) {
                ioc_rule->pcnt += sibling->pcnt;
                ioc_rule->bcnt += sibling->bcnt;
                if (sibling->timestamp > ioc_rule->timestamp)
                        ioc_rule->timestamp = sibling->timestamp;
#ifdef INVARIANTS
                ++i;
#endif
        }
        KASSERT(i == netisr_ncpus,
            ("static rule is not duplicated on netisr_ncpus %d", netisr_ncpus));

        bcopy(rule->cmd, ioc_rule->cmd, ioc_rule->cmd_len * 4 /* XXX */);

        return ((uint8_t *)ioc_rule + IOC_RULESIZE(ioc_rule));
}

static boolean_t
ipfw_track_copy(const struct ipfw_trkcnt *trk, struct ipfw_ioc_state *ioc_state)
{
        struct ipfw_ioc_flowid *ioc_id;

        if (trk->tc_expire == 0) {
                /* Not a scanned one. */
                return (FALSE);
        }

        ioc_state->expire = TIME_LEQ(trk->tc_expire, time_uptime) ?
            0 : trk->tc_expire - time_uptime;
        ioc_state->pcnt = 0;
        ioc_state->bcnt = 0;

        ioc_state->dyn_type = O_LIMIT_PARENT;
        ioc_state->count = trk->tc_count;

        ioc_state->rulenum = trk->tc_rulenum;

        ioc_id = &ioc_state->id;
        ioc_id->type = ETHERTYPE_IP;
        ioc_id->u.ip.proto = trk->tc_proto;
        ioc_id->u.ip.src_ip = trk->tc_saddr;
        ioc_id->u.ip.dst_ip = trk->tc_daddr;
        ioc_id->u.ip.src_port = trk->tc_sport;
        ioc_id->u.ip.dst_port = trk->tc_dport;

        return (TRUE);
}

static boolean_t
ipfw_state_copy(const struct ipfw_state *s, struct ipfw_ioc_state *ioc_state)
{
        struct ipfw_ioc_flowid *ioc_id;

        if (IPFW_STATE_SCANSKIP(s))
                return (FALSE);

        ioc_state->expire = TIME_LEQ(s->st_expire, time_uptime) ?
            0 : s->st_expire - time_uptime;
        ioc_state->pcnt = s->st_pcnt;
        ioc_state->bcnt = s->st_bcnt;

        ioc_state->dyn_type = s->st_type;
        ioc_state->count = 0;

        ioc_state->rulenum = s->st_rule->rulenum;

        ioc_id = &ioc_state->id;
        ioc_id->type = ETHERTYPE_IP;
        ioc_id->u.ip.proto = s->st_proto;
        ipfw_key_4tuple(&s->st_key,
            &ioc_id->u.ip.src_ip, &ioc_id->u.ip.src_port,
            &ioc_id->u.ip.dst_ip, &ioc_id->u.ip.dst_port);

        if (IPFW_ISXLAT(s->st_type)) {
                const struct ipfw_xlat *x = (const struct ipfw_xlat *)s;

                if (x->xlat_port == 0)
                        ioc_state->xlat_port = ioc_id->u.ip.dst_port;
                else
                        ioc_state->xlat_port = ntohs(x->xlat_port);
                ioc_state->xlat_addr = ntohl(x->xlat_addr);

                ioc_state->pcnt += x->xlat_pair->xlat_pcnt;
                ioc_state->bcnt += x->xlat_pair->xlat_bcnt;
        }

        return (TRUE);
}

static void
ipfw_state_copy_dispatch(netmsg_t nmsg)
{
        struct netmsg_cpstate *nm = (struct netmsg_cpstate *)nmsg;
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        const struct ipfw_state *s;
        const struct ipfw_track *t;

        ASSERT_NETISR_NCPUS(mycpuid);
        KASSERT(nm->state_cnt < nm->state_cntmax,
            ("invalid state count %d, max %d",
             nm->state_cnt, nm->state_cntmax));

        TAILQ_FOREACH(s, &ctx->ipfw_state_list, st_link) {
                if (ipfw_state_copy(s, nm->ioc_state)) {
                        nm->ioc_state++;
                        nm->state_cnt++;
                        if (nm->state_cnt == nm->state_cntmax)
                                goto done;
                }
        }

        /*
         * Prepare tracks in the global track tree for userland.
         */
        TAILQ_FOREACH(t, &ctx->ipfw_track_list, t_link) {
                struct ipfw_trkcnt *trk;

                if (t->t_count == NULL) /* anchor */
                        continue;
                trk = t->t_trkcnt;

                /*
                 * Only one netisr can run this function at
                 * any time, and only this function accesses
                 * trkcnt's tc_expire, so this is safe w/o
                 * ipfw_gd.ipfw_trkcnt_token.
                 */
                if (trk->tc_expire > t->t_expire)
                        continue;
                trk->tc_expire = t->t_expire;
        }

        /*
         * Copy tracks in the global track tree to userland in
         * the last netisr.
         */
        if (mycpuid == netisr_ncpus - 1) {
                struct ipfw_trkcnt *trk;

                KASSERT(nm->state_cnt < nm->state_cntmax,
                    ("invalid state count %d, max %d",
                     nm->state_cnt, nm->state_cntmax));

                IPFW_TRKCNT_TOKGET;
                RB_FOREACH(trk, ipfw_trkcnt_tree, &ipfw_gd.ipfw_trkcnt_tree) {
                        if (ipfw_track_copy(trk, nm->ioc_state)) {
                                nm->ioc_state++;
                                nm->state_cnt++;
                                if (nm->state_cnt == nm->state_cntmax) {
                                        IPFW_TRKCNT_TOKREL;
                                        goto done;
                                }
                        }
                }
                IPFW_TRKCNT_TOKREL;
        }
done:
        if (nm->state_cnt == nm->state_cntmax) {
                /* No more space; done. */
                netisr_replymsg(&nm->base, 0);
        } else {
                netisr_forwardmsg(&nm->base, mycpuid + 1);
        }
}

static int
ipfw_ctl_get_rules(struct sockopt *sopt)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct ip_fw *rule;
        void *bp;
        size_t size;
        int state_cnt;

        ASSERT_NETISR0;

        /*
         * pass up a copy of the current rules. Static rules
         * come first (the last of which has number IPFW_DEFAULT_RULE),
         * followed by a possibly empty list of states.
         */

        size = static_ioc_len;  /* size of static rules */

        /*
         * Size of the states.
         * XXX take tracks as state for userland compat.
         */
        state_cnt = ipfw_state_cntcoll() + ipfw_gd.ipfw_trkcnt_cnt;
        state_cnt = (state_cnt * 5) / 4; /* leave 25% headroom */
        size += state_cnt * sizeof(struct ipfw_ioc_state);

        if (sopt->sopt_valsize < size) {
                /* short length, no need to return incomplete rules */
                /* XXX: if superuser, no need to zero buffer */
                bzero(sopt->sopt_val, sopt->sopt_valsize); 
                return 0;
        }
        bp = sopt->sopt_val;

        for (rule = ctx->ipfw_layer3_chain; rule; rule = rule->next)
                bp = ipfw_copy_rule(ctx, rule, bp);

        if (state_cnt) {
                struct netmsg_cpstate nm;
#ifdef INVARIANTS
                size_t old_size = size;
#endif

                netmsg_init(&nm.base, NULL, &curthread->td_msgport,
                    MSGF_PRIORITY, ipfw_state_copy_dispatch);
                nm.ioc_state = bp;
                nm.state_cntmax = state_cnt;
                nm.state_cnt = 0;
                netisr_domsg_global(&nm.base);

                /*
                 * The # of states may be shrinked after the snapshot
                 * of the state count was taken.  To give user a correct
                 * state count, nm->state_cnt is used to recalculate
                 * the actual size.
                 */
                size = static_ioc_len +
                    (nm.state_cnt * sizeof(struct ipfw_ioc_state));
                KKASSERT(size <= old_size);
        }

        sopt->sopt_valsize = size;
        return 0;
}

static void
ipfw_set_disable_dispatch(netmsg_t nmsg)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];

        ASSERT_NETISR_NCPUS(mycpuid);

        ctx->ipfw_set_disable = nmsg->lmsg.u.ms_result32;
        netisr_forwardmsg(&nmsg->base, mycpuid + 1);
}

static void
ipfw_ctl_set_disable(uint32_t disable, uint32_t enable)
{
        struct netmsg_base nmsg;
        uint32_t set_disable;

        ASSERT_NETISR0;

        /* IPFW_DEFAULT_SET is always enabled */
        enable |= (1 << IPFW_DEFAULT_SET);
        set_disable = (ipfw_ctx[mycpuid]->ipfw_set_disable | disable) & ~enable;

        bzero(&nmsg, sizeof(nmsg));
        netmsg_init(&nmsg, NULL, &curthread->td_msgport, MSGF_PRIORITY,
            ipfw_set_disable_dispatch);
        nmsg.lmsg.u.ms_result32 = set_disable;

        netisr_domsg_global(&nmsg);
}

static void
ipfw_table_create_dispatch(netmsg_t nm)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        int tblid = nm->lmsg.u.ms_result;

        ASSERT_NETISR_NCPUS(mycpuid);

        if (!rn_inithead((void **)&ctx->ipfw_tables[tblid],
            rn_cpumaskhead(mycpuid), 32))
                panic("ipfw: create table%d failed", tblid);

        netisr_forwardmsg(&nm->base, mycpuid + 1);
}

static int
ipfw_table_create(struct sockopt *sopt)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct ipfw_ioc_table *tbl;
        struct netmsg_base nm;

        ASSERT_NETISR0;

        if (sopt->sopt_valsize != sizeof(*tbl))
                return (EINVAL);

        tbl = sopt->sopt_val;
        if (tbl->tableid < 0 || tbl->tableid >= ipfw_table_max)
                return (EINVAL);

        if (ctx->ipfw_tables[tbl->tableid] != NULL)
                return (EEXIST);

        netmsg_init(&nm, NULL, &curthread->td_msgport, MSGF_PRIORITY,
            ipfw_table_create_dispatch);
        nm.lmsg.u.ms_result = tbl->tableid;
        netisr_domsg_global(&nm);

        return (0);
}

static void
ipfw_table_killrn(struct radix_node_head *rnh, struct radix_node *rn)
{
        struct radix_node *ret;

        ret = rnh->rnh_deladdr(rn->rn_key, rn->rn_mask, rnh);
        if (ret != rn)
                panic("deleted other table entry");
        kfree(ret, M_IPFW);
}

static int
ipfw_table_killent(struct radix_node *rn, void *xrnh)
{

        ipfw_table_killrn(xrnh, rn);
        return (0);
}

static void
ipfw_table_flush_oncpu(struct ipfw_context *ctx, int tableid,
    int destroy)
{
        struct radix_node_head *rnh;

        ASSERT_NETISR_NCPUS(mycpuid);

        rnh = ctx->ipfw_tables[tableid];
        rnh->rnh_walktree(rnh, ipfw_table_killent, rnh);
        if (destroy) {
                Free(rnh);
                ctx->ipfw_tables[tableid] = NULL;
        }
}

static void
ipfw_table_flush_dispatch(netmsg_t nmsg)
{
        struct netmsg_tblflush *nm = (struct netmsg_tblflush *)nmsg;
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];

        ASSERT_NETISR_NCPUS(mycpuid);

        ipfw_table_flush_oncpu(ctx, nm->tableid, nm->destroy);
        netisr_forwardmsg(&nm->base, mycpuid + 1);
}

static void
ipfw_table_flushall_oncpu(struct ipfw_context *ctx, int destroy)
{
        int i;

        ASSERT_NETISR_NCPUS(mycpuid);

        for (i = 0; i < ipfw_table_max; ++i) {
                if (ctx->ipfw_tables[i] != NULL)
                        ipfw_table_flush_oncpu(ctx, i, destroy);
        }
}

static void
ipfw_table_flushall_dispatch(netmsg_t nmsg)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];

        ASSERT_NETISR_NCPUS(mycpuid);

        ipfw_table_flushall_oncpu(ctx, 0);
        netisr_forwardmsg(&nmsg->base, mycpuid + 1);
}

static int
ipfw_table_flush(struct sockopt *sopt)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct ipfw_ioc_table *tbl;
        struct netmsg_tblflush nm;

        ASSERT_NETISR0;

        if (sopt->sopt_valsize != sizeof(*tbl))
                return (EINVAL);

        tbl = sopt->sopt_val;
        if (sopt->sopt_name == IP_FW_TBL_FLUSH && tbl->tableid < 0) {
                netmsg_init(&nm.base, NULL, &curthread->td_msgport,
                    MSGF_PRIORITY, ipfw_table_flushall_dispatch);
                netisr_domsg_global(&nm.base);
                return (0);
        }

        if (tbl->tableid < 0 || tbl->tableid >= ipfw_table_max)
                return (EINVAL);

        if (ctx->ipfw_tables[tbl->tableid] == NULL)
                return (ENOENT);

        netmsg_init(&nm.base, NULL, &curthread->td_msgport, MSGF_PRIORITY,
            ipfw_table_flush_dispatch);
        nm.tableid = tbl->tableid;
        nm.destroy = 0;
        if (sopt->sopt_name == IP_FW_TBL_DESTROY)
                nm.destroy = 1;
        netisr_domsg_global(&nm.base);

        return (0);
}

static int
ipfw_table_cntent(struct radix_node *rn __unused, void *xcnt)
{
        int *cnt = xcnt;

        (*cnt)++;
        return (0);
}

static int
ipfw_table_cpent(struct radix_node *rn, void *xcp)
{
        struct ipfw_table_cp *cp = xcp;
        struct ipfw_tblent *te = (struct ipfw_tblent *)rn;
        struct ipfw_ioc_tblent *ioc_te;
#ifdef INVARIANTS
        int cnt;
#endif

        KASSERT(cp->te_idx < cp->te_cnt, ("invalid table cp idx %d, cnt %d",
            cp->te_idx, cp->te_cnt));
        ioc_te = &cp->te[cp->te_idx];

        if (te->te_nodes->rn_mask != NULL) {
                memcpy(&ioc_te->netmask, te->te_nodes->rn_mask,
                    *te->te_nodes->rn_mask);
        } else {
                ioc_te->netmask.sin_len = 0;
        }
        memcpy(&ioc_te->key, &te->te_key, sizeof(ioc_te->key));

        ioc_te->use = te->te_use;
        ioc_te->last_used = te->te_lastuse;
#ifdef INVARIANTS
        cnt = 1;
#endif

        while ((te = te->te_sibling) != NULL) {
#ifdef INVARIANTS
                ++cnt;
#endif
                ioc_te->use += te->te_use;
                if (te->te_lastuse > ioc_te->last_used)
                        ioc_te->last_used = te->te_lastuse;
        }
        KASSERT(cnt == netisr_ncpus,
            ("invalid # of tblent %d, should be %d", cnt, netisr_ncpus));

        cp->te_idx++;

        return (0);
}

static int
ipfw_table_get(struct sockopt *sopt)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct radix_node_head *rnh;
        struct ipfw_ioc_table *tbl;
        struct ipfw_ioc_tblcont *cont;
        struct ipfw_table_cp cp;
        int cnt = 0, sz;

        ASSERT_NETISR0;

        if (sopt->sopt_valsize < sizeof(*tbl))
                return (EINVAL);

        tbl = sopt->sopt_val;
        if (tbl->tableid < 0) {
                struct ipfw_ioc_tbllist *list;
                int i;

                /*
                 * List available table ids.
                 */
                for (i = 0; i < ipfw_table_max; ++i) {
                        if (ctx->ipfw_tables[i] != NULL)
                                ++cnt;
                }

                sz = __offsetof(struct ipfw_ioc_tbllist, tables[cnt]);
                if (sopt->sopt_valsize < sz) {
                        bzero(sopt->sopt_val, sopt->sopt_valsize);
                        return (E2BIG);
                }
                list = sopt->sopt_val;
                list->tablecnt = cnt;

                cnt = 0;
                for (i = 0; i < ipfw_table_max; ++i) {
                        if (ctx->ipfw_tables[i] != NULL) {
                                KASSERT(cnt < list->tablecnt,
                                    ("invalid idx %d, cnt %d",
                                     cnt, list->tablecnt));
                                list->tables[cnt++] = i;
                        }
                }
                sopt->sopt_valsize = sz;
                return (0);
        } else if (tbl->tableid >= ipfw_table_max) {
                return (EINVAL);
        }

        rnh = ctx->ipfw_tables[tbl->tableid];
        if (rnh == NULL)
                return (ENOENT);
        rnh->rnh_walktree(rnh, ipfw_table_cntent, &cnt);

        sz = __offsetof(struct ipfw_ioc_tblcont, ent[cnt]);
        if (sopt->sopt_valsize < sz) {
                bzero(sopt->sopt_val, sopt->sopt_valsize);
                return (E2BIG);
        }
        cont = sopt->sopt_val;
        cont->entcnt = cnt;

        cp.te = cont->ent;
        cp.te_idx = 0;
        cp.te_cnt = cnt;
        rnh->rnh_walktree(rnh, ipfw_table_cpent, &cp);

        sopt->sopt_valsize = sz;
        return (0);
}

static void
ipfw_table_add_dispatch(netmsg_t nmsg)
{
        struct netmsg_tblent *nm = (struct netmsg_tblent *)nmsg;
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct radix_node_head *rnh;
        struct ipfw_tblent *te;

        ASSERT_NETISR_NCPUS(mycpuid);

        rnh = ctx->ipfw_tables[nm->tableid];

        te = kmalloc(sizeof(*te), M_IPFW, M_WAITOK | M_ZERO);
        te->te_nodes->rn_key = (char *)&te->te_key;
        memcpy(&te->te_key, nm->key, sizeof(te->te_key));

        if (rnh->rnh_addaddr((char *)&te->te_key, (char *)nm->netmask, rnh,
            te->te_nodes) == NULL) {
                if (mycpuid == 0) {
                        kfree(te, M_IPFW);
                        netisr_replymsg(&nm->base, EEXIST);
                        return;
                }
                panic("rnh_addaddr failed");
        }

        /* Link siblings. */
        if (nm->sibling != NULL)
                nm->sibling->te_sibling = te;
        nm->sibling = te;

        netisr_forwardmsg(&nm->base, mycpuid + 1);
}

static void
ipfw_table_del_dispatch(netmsg_t nmsg)
{
        struct netmsg_tblent *nm = (struct netmsg_tblent *)nmsg;
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct radix_node_head *rnh;
        struct radix_node *rn;

        ASSERT_NETISR_NCPUS(mycpuid);

        rnh = ctx->ipfw_tables[nm->tableid];
        rn = rnh->rnh_deladdr((char *)nm->key, (char *)nm->netmask, rnh);
        if (rn == NULL) {
                if (mycpuid == 0) {
                        netisr_replymsg(&nm->base, ESRCH);
                        return;
                }
                panic("rnh_deladdr failed");
        }
        kfree(rn, M_IPFW);

        netisr_forwardmsg(&nm->base, mycpuid + 1);
}

static int
ipfw_table_alt(struct sockopt *sopt)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct ipfw_ioc_tblcont *tbl;
        struct ipfw_ioc_tblent *te;
        struct sockaddr_in key0;
        struct sockaddr *netmask = NULL, *key;
        struct netmsg_tblent nm;

        ASSERT_NETISR0;

        if (sopt->sopt_valsize != sizeof(*tbl))
                return (EINVAL);
        tbl = sopt->sopt_val;

        if (tbl->tableid < 0  || tbl->tableid >= ipfw_table_max)
                return (EINVAL);
        if (tbl->entcnt != 1)
                return (EINVAL);

        if (ctx->ipfw_tables[tbl->tableid] == NULL)
                return (ENOENT);
        te = &tbl->ent[0];

        if (te->key.sin_family != AF_INET ||
            te->key.sin_port != 0 ||
            te->key.sin_len != sizeof(struct sockaddr_in))
                return (EINVAL);
        key = (struct sockaddr *)&te->key;

        if (te->netmask.sin_len != 0) {
                if (te->netmask.sin_port != 0 ||
                    te->netmask.sin_len > sizeof(struct sockaddr_in))
                        return (EINVAL);
                netmask = (struct sockaddr *)&te->netmask;
                sa_maskedcopy(key, (struct sockaddr *)&key0, netmask);
                key = (struct sockaddr *)&key0;
        }

        if (sopt->sopt_name == IP_FW_TBL_ADD) {
                netmsg_init(&nm.base, NULL, &curthread->td_msgport,
                    MSGF_PRIORITY, ipfw_table_add_dispatch);
        } else {
                netmsg_init(&nm.base, NULL, &curthread->td_msgport,
                    MSGF_PRIORITY, ipfw_table_del_dispatch);
        }
        nm.key = key;
        nm.netmask = netmask;
        nm.tableid = tbl->tableid;
        nm.sibling = NULL;
        return (netisr_domsg_global(&nm.base));
}

static int
ipfw_table_zeroent(struct radix_node *rn, void *arg __unused)
{
        struct ipfw_tblent *te = (struct ipfw_tblent *)rn;

        te->te_use = 0;
        te->te_lastuse = 0;
        return (0);
}

static void
ipfw_table_zero_dispatch(netmsg_t nmsg)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct radix_node_head *rnh;

        ASSERT_NETISR_NCPUS(mycpuid);

        rnh = ctx->ipfw_tables[nmsg->lmsg.u.ms_result];
        rnh->rnh_walktree(rnh, ipfw_table_zeroent, NULL);

        netisr_forwardmsg(&nmsg->base, mycpuid + 1);
}

static void
ipfw_table_zeroall_dispatch(netmsg_t nmsg)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        int i;

        ASSERT_NETISR_NCPUS(mycpuid);

        for (i = 0; i < ipfw_table_max; ++i) {
                struct radix_node_head *rnh = ctx->ipfw_tables[i];

                if (rnh != NULL)
                        rnh->rnh_walktree(rnh, ipfw_table_zeroent, NULL);
        }
        netisr_forwardmsg(&nmsg->base, mycpuid + 1);
}

static int
ipfw_table_zero(struct sockopt *sopt)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct netmsg_base nm;
        struct ipfw_ioc_table *tbl;

        ASSERT_NETISR0;

        if (sopt->sopt_valsize != sizeof(*tbl))
                return (EINVAL);
        tbl = sopt->sopt_val;

        if (tbl->tableid < 0) {
                netmsg_init(&nm, NULL, &curthread->td_msgport, MSGF_PRIORITY,
                    ipfw_table_zeroall_dispatch);
                netisr_domsg_global(&nm);
                return (0);
        } else if (tbl->tableid >= ipfw_table_max) {
                return (EINVAL);
        } else if (ctx->ipfw_tables[tbl->tableid] == NULL) {
                return (ENOENT);
        }

        netmsg_init(&nm, NULL, &curthread->td_msgport, MSGF_PRIORITY,
            ipfw_table_zero_dispatch);
        nm.lmsg.u.ms_result = tbl->tableid;
        netisr_domsg_global(&nm);

        return (0);
}

static int
ipfw_table_killexp(struct radix_node *rn, void *xnm)
{
        struct netmsg_tblexp *nm = xnm;
        struct ipfw_tblent *te = (struct ipfw_tblent *)rn;

        if (te->te_expired) {
                ipfw_table_killrn(nm->rnh, rn);
                nm->expcnt++;
        }
        return (0);
}

static void
ipfw_table_expire_dispatch(netmsg_t nmsg)
{
        struct netmsg_tblexp *nm = (struct netmsg_tblexp *)nmsg;
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct radix_node_head *rnh;

        ASSERT_NETISR_NCPUS(mycpuid);

        rnh = ctx->ipfw_tables[nm->tableid];
        nm->rnh = rnh;
        rnh->rnh_walktree(rnh, ipfw_table_killexp, nm);

        KASSERT(nm->expcnt == nm->cnt * (mycpuid + 1),
            ("not all expired addresses (%d) were deleted (%d)",
             nm->cnt * (mycpuid + 1), nm->expcnt));

        netisr_forwardmsg(&nm->base, mycpuid + 1);
}

static void
ipfw_table_expireall_dispatch(netmsg_t nmsg)
{
        struct netmsg_tblexp *nm = (struct netmsg_tblexp *)nmsg;
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        int i;

        ASSERT_NETISR_NCPUS(mycpuid);

        for (i = 0; i < ipfw_table_max; ++i) {
                struct radix_node_head *rnh = ctx->ipfw_tables[i];

                if (rnh == NULL)
                        continue;
                nm->rnh = rnh;
                rnh->rnh_walktree(rnh, ipfw_table_killexp, nm);
        }

        KASSERT(nm->expcnt == nm->cnt * (mycpuid + 1),
            ("not all expired addresses (%d) were deleted (%d)",
             nm->cnt * (mycpuid + 1), nm->expcnt));

        netisr_forwardmsg(&nm->base, mycpuid + 1);
}

static int
ipfw_table_markexp(struct radix_node *rn, void *xnm)
{
        struct netmsg_tblexp *nm = xnm;
        struct ipfw_tblent *te;
        time_t lastuse;

        te = (struct ipfw_tblent *)rn;
        lastuse = te->te_lastuse;

        while ((te = te->te_sibling) != NULL) {
                if (te->te_lastuse > lastuse)
                        lastuse = te->te_lastuse;
        }
        if (!TIME_LEQ(lastuse + nm->expire, time_second)) {
                /* Not expired */
                return (0);
        }

        te = (struct ipfw_tblent *)rn;
        te->te_expired = 1;
        while ((te = te->te_sibling) != NULL)
                te->te_expired = 1;
        nm->cnt++;

        return (0);
}

static int
ipfw_table_expire(struct sockopt *sopt)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct netmsg_tblexp nm;
        struct ipfw_ioc_tblexp *tbl;
        struct radix_node_head *rnh;

        ASSERT_NETISR0;

        if (sopt->sopt_valsize != sizeof(*tbl))
                return (EINVAL);
        tbl = sopt->sopt_val;
        tbl->expcnt = 0;

        nm.expcnt = 0;
        nm.cnt = 0;
        nm.expire = tbl->expire;

        if (tbl->tableid < 0) {
                int i;

                for (i = 0; i < ipfw_table_max; ++i) {
                        rnh = ctx->ipfw_tables[i];
                        if (rnh == NULL)
                                continue;
                        rnh->rnh_walktree(rnh, ipfw_table_markexp, &nm);
                }
                if (nm.cnt == 0) {
                        /* No addresses can be expired. */
                        return (0);
                }
                tbl->expcnt = nm.cnt;

                netmsg_init(&nm.base, NULL, &curthread->td_msgport,
                    MSGF_PRIORITY, ipfw_table_expireall_dispatch);
                nm.tableid = -1;
                netisr_domsg_global(&nm.base);
                KASSERT(nm.expcnt == nm.cnt * netisr_ncpus,
                    ("not all expired addresses (%d) were deleted (%d)",
                     nm.cnt * netisr_ncpus, nm.expcnt));

                return (0);
        } else if (tbl->tableid >= ipfw_table_max) {
                return (EINVAL);
        }

        rnh = ctx->ipfw_tables[tbl->tableid];
        if (rnh == NULL)
                return (ENOENT);
        rnh->rnh_walktree(rnh, ipfw_table_markexp, &nm);
        if (nm.cnt == 0) {
                /* No addresses can be expired. */
                return (0);
        }
        tbl->expcnt = nm.cnt;

        netmsg_init(&nm.base, NULL, &curthread->td_msgport, MSGF_PRIORITY,
            ipfw_table_expire_dispatch);
        nm.tableid = tbl->tableid;
        netisr_domsg_global(&nm.base);
        KASSERT(nm.expcnt == nm.cnt * netisr_ncpus,
            ("not all expired addresses (%d) were deleted (%d)",
             nm.cnt * netisr_ncpus, nm.expcnt));
        return (0);
}

static void
ipfw_crossref_free_dispatch(netmsg_t nmsg)
{
        struct ip_fw *rule = nmsg->lmsg.u.ms_resultp;

        KKASSERT((rule->rule_flags &
            (IPFW_RULE_F_CROSSREF | IPFW_RULE_F_INVALID)) ==
            (IPFW_RULE_F_CROSSREF | IPFW_RULE_F_INVALID));
        ipfw_free_rule(rule);

        netisr_replymsg(&nmsg->base, 0);
}

static void
ipfw_crossref_reap(void)
{
        struct ip_fw *rule, *prev = NULL;

        ASSERT_NETISR0;

        rule = ipfw_gd.ipfw_crossref_free;
        while (rule != NULL) {
                uint64_t inflight = 0;
                int i;

                for (i = 0; i < netisr_ncpus; ++i)
                        inflight += rule->cross_rules[i]->cross_refs;
                if (inflight == 0) {
                        struct ip_fw *f = rule;

                        /*
                         * Unlink.
                         */
                        rule = rule->next;
                        if (prev != NULL)
                                prev->next = rule;
                        else
                                ipfw_gd.ipfw_crossref_free = rule;

                        /*
                         * Free.
                         */
                        for (i = 1; i < netisr_ncpus; ++i) {
                                struct netmsg_base nm;

                                netmsg_init(&nm, NULL, &curthread->td_msgport,
                                    MSGF_PRIORITY, ipfw_crossref_free_dispatch);
                                nm.lmsg.u.ms_resultp = f->cross_rules[i];
                                netisr_domsg(&nm, i);
                        }
                        KKASSERT((f->rule_flags &
                            (IPFW_RULE_F_CROSSREF | IPFW_RULE_F_INVALID)) ==
                            (IPFW_RULE_F_CROSSREF | IPFW_RULE_F_INVALID));
                        ipfw_unref_rule(f);
                } else {
                        prev = rule;
                        rule = rule->next;
                }
        }

        if (ipfw_gd.ipfw_crossref_free != NULL) {
                callout_reset(&ipfw_gd.ipfw_crossref_ch, hz,
                    ipfw_crossref_timeo, NULL);
        }
}

/*
 * {set|get}sockopt parser.
 */
static int
ipfw_ctl(struct sockopt *sopt)
{
        int error, rulenum;
        uint32_t *masks;
        size_t size;

        ASSERT_NETISR0;

        error = 0;

        switch (sopt->sopt_name) {
        case IP_FW_GET:
                error = ipfw_ctl_get_rules(sopt);
                break;

        case IP_FW_FLUSH:
                ipfw_flush(0 /* keep default rule */);
                break;

        case IP_FW_ADD:
                error = ipfw_ctl_add_rule(sopt);
                break;

        case IP_FW_DEL:
                /*
                 * IP_FW_DEL is used for deleting single rules or sets,
                 * and (ab)used to atomically manipulate sets.
                 * Argument size is used to distinguish between the two:
                 *    sizeof(uint32_t)
                 *      delete single rule or set of rules,
                 *      or reassign rules (or sets) to a different set.
                 *    2 * sizeof(uint32_t)
                 *      atomic disable/enable sets.
                 *      first uint32_t contains sets to be disabled,
                 *      second uint32_t contains sets to be enabled.
                 */
                masks = sopt->sopt_val;
                size = sopt->sopt_valsize;
                if (size == sizeof(*masks)) {
                        /*
                         * Delete or reassign static rule
                         */
                        error = ipfw_ctl_alter(masks[0]);
                } else if (size == (2 * sizeof(*masks))) {
                        /*
                         * Set enable/disable
                         */
                        ipfw_ctl_set_disable(masks[0], masks[1]);
                } else {
                        error = EINVAL;
                }
                break;

        case IP_FW_ZERO:
        case IP_FW_RESETLOG: /* argument is an int, the rule number */
                rulenum = 0;

                if (sopt->sopt_val != 0) {
                    error = soopt_to_kbuf(sopt, &rulenum,
                            sizeof(int), sizeof(int));
                    if (error)
                        break;
                }
                error = ipfw_ctl_zero_entry(rulenum,
                        sopt->sopt_name == IP_FW_RESETLOG);
                break;

        case IP_FW_TBL_CREATE:
                error = ipfw_table_create(sopt);
                break;

        case IP_FW_TBL_ADD:
        case IP_FW_TBL_DEL:
                error = ipfw_table_alt(sopt);
                break;

        case IP_FW_TBL_FLUSH:
        case IP_FW_TBL_DESTROY:
                error = ipfw_table_flush(sopt);
                break;

        case IP_FW_TBL_GET:
                error = ipfw_table_get(sopt);
                break;

        case IP_FW_TBL_ZERO:
                error = ipfw_table_zero(sopt);
                break;

        case IP_FW_TBL_EXPIRE:
                error = ipfw_table_expire(sopt);
                break;

        default:
                kprintf("ipfw_ctl invalid option %d\n", sopt->sopt_name);
                error = EINVAL;
        }

        ipfw_crossref_reap();
        return error;
}

static void
ipfw_keepalive_done(struct ipfw_context *ctx)
{

        KASSERT(ctx->ipfw_flags & IPFW_FLAG_KEEPALIVE,
            ("keepalive is not in progress"));
        ctx->ipfw_flags &= ~IPFW_FLAG_KEEPALIVE;
        callout_reset(&ctx->ipfw_keepalive_ch, dyn_keepalive_period * hz,
            ipfw_keepalive, NULL);
}

static void
ipfw_keepalive_more(struct ipfw_context *ctx)
{
        struct netmsg_base *nm = &ctx->ipfw_keepalive_more;

        KASSERT(ctx->ipfw_flags & IPFW_FLAG_KEEPALIVE,
            ("keepalive is not in progress"));
        KASSERT(nm->lmsg.ms_flags & MSGF_DONE,
            ("keepalive more did not finish"));
        netisr_sendmsg_oncpu(nm);
}

static void
ipfw_keepalive_loop(struct ipfw_context *ctx, struct ipfw_state *anchor)
{
        struct ipfw_state *s;
        int scanned = 0, expired = 0, kept = 0;

        KASSERT(ctx->ipfw_flags & IPFW_FLAG_KEEPALIVE,
            ("keepalive is not in progress"));

        while ((s = TAILQ_NEXT(anchor, st_link)) != NULL) {
                uint32_t ack_rev, ack_fwd;
                struct ipfw_flow_id id;
                uint8_t send_dir;

                if (scanned++ >= ipfw_state_scan_max) {
                        ipfw_keepalive_more(ctx);
                        return;
                }

                TAILQ_REMOVE(&ctx->ipfw_state_list, anchor, st_link);
                TAILQ_INSERT_AFTER(&ctx->ipfw_state_list, s, anchor, st_link);

                /*
                 * NOTE:
                 * Don't use IPFW_STATE_SCANSKIP; need to perform keepalive
                 * on slave xlat.
                 */
                if (s->st_type == O_ANCHOR)
                        continue;

                if (IPFW_STATE_ISDEAD(s)) {
                        ipfw_state_remove(ctx, s);
                        if (++expired >= ipfw_state_expire_max) {
                                ipfw_keepalive_more(ctx);
                                return;
                        }
                        continue;
                }

                /*
                 * Keep alive processing
                 */

                if (s->st_proto != IPPROTO_TCP)
                        continue;
                if ((s->st_state & IPFW_STATE_TCPSTATES) != BOTH_SYN)
                        continue;
                if (TIME_LEQ(time_uptime + dyn_keepalive_interval,
                    s->st_expire))
                        continue;       /* too early */

                ipfw_key_4tuple(&s->st_key, &id.src_ip, &id.src_port,
                    &id.dst_ip, &id.dst_port);
                ack_rev = s->st_ack_rev;
                ack_fwd = s->st_ack_fwd;

#define SEND_FWD        0x1
#define SEND_REV        0x2

                if (IPFW_ISXLAT(s->st_type)) {
                        const struct ipfw_xlat *x = (const struct ipfw_xlat *)s;

                        if (x->xlat_dir == MATCH_FORWARD)
                                send_dir = SEND_FWD;
                        else
                                send_dir = SEND_REV;
                } else {
                        send_dir = SEND_FWD | SEND_REV;
                }

                if (send_dir & SEND_REV)
                        send_pkt(&id, ack_rev - 1, ack_fwd, TH_SYN);
                if (send_dir & SEND_FWD)
                        send_pkt(&id, ack_fwd - 1, ack_rev, 0);

#undef SEND_FWD
#undef SEND_REV

                if (++kept >= ipfw_keepalive_max) {
                        ipfw_keepalive_more(ctx);
                        return;
                }
        }
        TAILQ_REMOVE(&ctx->ipfw_state_list, anchor, st_link);
        ipfw_keepalive_done(ctx);
}

static void
ipfw_keepalive_more_dispatch(netmsg_t nm)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct ipfw_state *anchor;

        ASSERT_NETISR_NCPUS(mycpuid);
        KASSERT(ctx->ipfw_flags & IPFW_FLAG_KEEPALIVE,
            ("keepalive is not in progress"));

        /* Reply ASAP */
        netisr_replymsg(&nm->base, 0);

        anchor = &ctx->ipfw_keepalive_anch;
        if (!dyn_keepalive || ctx->ipfw_state_cnt == 0) {
                TAILQ_REMOVE(&ctx->ipfw_state_list, anchor, st_link);
                ipfw_keepalive_done(ctx);
                return;
        }
        ipfw_keepalive_loop(ctx, anchor);
}

/*
 * This procedure is only used to handle keepalives. It is invoked
 * every dyn_keepalive_period
 */
static void
ipfw_keepalive_dispatch(netmsg_t nm)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct ipfw_state *anchor;

        ASSERT_NETISR_NCPUS(mycpuid);
        KASSERT((ctx->ipfw_flags & IPFW_FLAG_KEEPALIVE) == 0,
            ("keepalive is in progress"));
        ctx->ipfw_flags |= IPFW_FLAG_KEEPALIVE;

        /* Reply ASAP */
        crit_enter();
        netisr_replymsg(&nm->base, 0);
        crit_exit();

        if (!dyn_keepalive || ctx->ipfw_state_cnt == 0) {
                ipfw_keepalive_done(ctx);
                return;
        }

        anchor = &ctx->ipfw_keepalive_anch;
        TAILQ_INSERT_HEAD(&ctx->ipfw_state_list, anchor, st_link);
        ipfw_keepalive_loop(ctx, anchor);
}

/*
 * This procedure is only used to handle keepalives. It is invoked
 * every dyn_keepalive_period
 */
static void
ipfw_keepalive(void *dummy __unused)
{
        struct netmsg_base *msg;

        KKASSERT(mycpuid < netisr_ncpus);
        msg = &ipfw_ctx[mycpuid]->ipfw_keepalive_nm;

        crit_enter();
        if (msg->lmsg.ms_flags & MSGF_DONE)
                netisr_sendmsg_oncpu(msg);
        crit_exit();
}

static void
ipfw_ip_input_dispatch(netmsg_t nmsg)
{
        struct netmsg_genpkt *nm = (struct netmsg_genpkt *)nmsg;
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct mbuf *m = nm->m;
        struct ip_fw *rule = nm->arg1;

        ASSERT_NETISR_NCPUS(mycpuid);
        KASSERT(rule->cpuid == mycpuid,
            ("rule does not belong to cpu%d", mycpuid));
        KASSERT(m->m_pkthdr.fw_flags & IPFW_MBUF_CONTINUE,
            ("mbuf does not have ipfw continue rule"));

        KASSERT(ctx->ipfw_cont_rule == NULL,
            ("pending ipfw continue rule"));
        ctx->ipfw_cont_rule = rule;
        ip_input(m);

        /* May not be cleared, if ipfw was unload/disabled. */
        ctx->ipfw_cont_rule = NULL;

        /*
         * This rule is no longer used; decrement its cross_refs,
         * so this rule can be deleted.
         */
        rule->cross_refs--;
}

static void
ipfw_defrag_redispatch(struct mbuf *m, int cpuid, struct ip_fw *rule)
{
        struct netmsg_genpkt *nm;

        KASSERT(cpuid != mycpuid, ("continue on the same cpu%d", cpuid));

        /*
         * NOTE:
         * Bump cross_refs to prevent this rule and its siblings
         * from being deleted, while this mbuf is inflight.  The
         * cross_refs of the sibling rule on the target cpu will
         * be decremented, once this mbuf is going to be filtered
         * on the target cpu.
         */
        rule->cross_refs++;
        m->m_pkthdr.fw_flags |= IPFW_MBUF_CONTINUE;

        nm = &m->m_hdr.mh_genmsg;
        netmsg_init(&nm->base, NULL, &netisr_apanic_rport, 0,
            ipfw_ip_input_dispatch);
        nm->m = m;
        nm->arg1 = rule->cross_rules[cpuid];
        netisr_sendmsg(&nm->base, cpuid);
}

static void
ipfw_init_args(struct ip_fw_args *args, struct mbuf *m, struct ifnet *oif)
{

        args->flags = 0;
        args->rule = NULL;
        args->xlat = NULL;

        if (m->m_pkthdr.fw_flags & DUMMYNET_MBUF_TAGGED) {
                struct m_tag *mtag;

                /* Extract info from dummynet tag */
                mtag = m_tag_find(m, PACKET_TAG_DUMMYNET, NULL);
                KKASSERT(mtag != NULL);
                args->rule = ((struct dn_pkt *)m_tag_data(mtag))->dn_priv;
                KKASSERT(args->rule != NULL);

                m_tag_delete(m, mtag);
                m->m_pkthdr.fw_flags &= ~DUMMYNET_MBUF_TAGGED;
        } else if (m->m_pkthdr.fw_flags & IPFW_MBUF_CONTINUE) {
                struct ipfw_context *ctx = ipfw_ctx[mycpuid];

                KKASSERT(ctx->ipfw_cont_rule != NULL);
                args->rule = ctx->ipfw_cont_rule;
                ctx->ipfw_cont_rule = NULL;

                if (ctx->ipfw_cont_xlat != NULL) {
                        args->xlat = ctx->ipfw_cont_xlat;
                        ctx->ipfw_cont_xlat = NULL;
                        if (m->m_pkthdr.fw_flags & IPFW_MBUF_XLATINS) {
                                args->flags |= IP_FWARG_F_XLATINS;
                                m->m_pkthdr.fw_flags &= ~IPFW_MBUF_XLATINS;
                        }
                        if (m->m_pkthdr.fw_flags & IPFW_MBUF_XLATFWD) {
                                args->flags |= IP_FWARG_F_XLATFWD;
                                m->m_pkthdr.fw_flags &= ~IPFW_MBUF_XLATFWD;
                        }
                }
                KKASSERT((m->m_pkthdr.fw_flags &
                    (IPFW_MBUF_XLATINS | IPFW_MBUF_XLATFWD)) == 0);

                args->flags |= IP_FWARG_F_CONT;
                m->m_pkthdr.fw_flags &= ~IPFW_MBUF_CONTINUE;
        }

        args->eh = NULL;
        args->oif = oif;
        args->m = m;
}

static int
ipfw_check_in(void *arg, struct mbuf **m0, struct ifnet *ifp, int dir)
{
        struct ip_fw_args args;
        struct mbuf *m = *m0;
        int tee = 0, error = 0, ret;

        ipfw_init_args(&args, m, NULL);

        ret = ipfw_chk(&args);
        m = args.m;
        if (m == NULL) {
                if (ret != IP_FW_REDISPATCH)
                        error = EACCES;
                goto back;
        }

        switch (ret) {
        case IP_FW_PASS:
                break;

        case IP_FW_DENY:
                m_freem(m);
                m = NULL;
                error = EACCES;
                break;

        case IP_FW_DUMMYNET:
                /* Send packet to the appropriate pipe */
                m = ipfw_dummynet_io(m, args.cookie, DN_TO_IP_IN, &args);
                break;

        case IP_FW_TEE:
                tee = 1;
                /* FALL THROUGH */

        case IP_FW_DIVERT:
                /*
                 * Must clear bridge tag when changing
                 */
                m->m_pkthdr.fw_flags &= ~BRIDGE_MBUF_TAGGED;
                if (ip_divert_p != NULL) {
                        m = ip_divert_p(m, tee, 1);
                } else {
                        m_freem(m);
                        m = NULL;
                        /* not sure this is the right error msg */
                        error = EACCES;
                }
                break;

        default:
                panic("unknown ipfw return value: %d", ret);
        }
back:
        *m0 = m;
        return error;
}

static int
ipfw_check_out(void *arg, struct mbuf **m0, struct ifnet *ifp, int dir)
{
        struct ip_fw_args args;
        struct mbuf *m = *m0;
        int tee = 0, error = 0, ret;

        ipfw_init_args(&args, m, ifp);

        ret = ipfw_chk(&args);
        m = args.m;
        if (m == NULL) {
                if (ret != IP_FW_REDISPATCH)
                        error = EACCES;
                goto back;
        }

        switch (ret) {
        case IP_FW_PASS:
                break;

        case IP_FW_DENY:
                m_freem(m);
                m = NULL;
                error = EACCES;
                break;

        case IP_FW_DUMMYNET:
                m = ipfw_dummynet_io(m, args.cookie, DN_TO_IP_OUT, &args);
                break;

        case IP_FW_TEE:
                tee = 1;
                /* FALL THROUGH */

        case IP_FW_DIVERT:
                if (ip_divert_p != NULL) {
                        m = ip_divert_p(m, tee, 0);
                } else {
                        m_freem(m);
                        m = NULL;
                        /* not sure this is the right error msg */
                        error = EACCES;
                }
                break;

        default:
                panic("unknown ipfw return value: %d", ret);
        }
back:
        *m0 = m;
        return error;
}

static void
ipfw_hook(void)
{
        struct pfil_head *pfh;

        ASSERT_NETISR0;

        pfh = pfil_head_get(PFIL_TYPE_AF, AF_INET);
        if (pfh == NULL)
                return;

        pfil_add_hook(ipfw_check_in, NULL, PFIL_IN, pfh);
        pfil_add_hook(ipfw_check_out, NULL, PFIL_OUT, pfh);
}

static void
ipfw_dehook(void)
{
        struct pfil_head *pfh;

        ASSERT_NETISR0;

        pfh = pfil_head_get(PFIL_TYPE_AF, AF_INET);
        if (pfh == NULL)
                return;

        pfil_remove_hook(ipfw_check_in, NULL, PFIL_IN, pfh);
        pfil_remove_hook(ipfw_check_out, NULL, PFIL_OUT, pfh);
}

static int
ipfw_sysctl_dyncnt(SYSCTL_HANDLER_ARGS)
{
        int dyn_cnt;

        dyn_cnt = ipfw_state_cntcoll();
        dyn_cnt += ipfw_gd.ipfw_trkcnt_cnt;

        return (sysctl_handle_int(oidp, &dyn_cnt, 0, req));
}

static int
ipfw_sysctl_statecnt(SYSCTL_HANDLER_ARGS)
{
        int state_cnt;

        state_cnt = ipfw_state_cntcoll();
        return (sysctl_handle_int(oidp, &state_cnt, 0, req));
}

static int
ipfw_sysctl_statemax(SYSCTL_HANDLER_ARGS)
{
        int state_max, error;

        state_max = ipfw_state_max;
        error = sysctl_handle_int(oidp, &state_max, 0, req);
        if (error || req->newptr == NULL)
                return (error);

        if (state_max < 1)
                return (EINVAL);

        ipfw_state_max_set(state_max);
        return (0);
}

static int
ipfw_sysctl_dynmax(SYSCTL_HANDLER_ARGS)
{
        int dyn_max, error;

        dyn_max = ipfw_state_max + ipfw_track_max;

        error = sysctl_handle_int(oidp, &dyn_max, 0, req);
        if (error || req->newptr == NULL)
                return (error);

        if (dyn_max < 2)
                return (EINVAL);

        ipfw_state_max_set(dyn_max / 2);
        ipfw_track_max = dyn_max / 2;
        return (0);
}

static void
ipfw_sysctl_enable_dispatch(netmsg_t nmsg)
{
        int enable = nmsg->lmsg.u.ms_result;

        ASSERT_NETISR0;

        if (fw_enable == enable)
                goto reply;

        fw_enable = enable;
        if (fw_enable)
                ipfw_hook();
        else
                ipfw_dehook();
reply:
        netisr_replymsg(&nmsg->base, 0);
}

static int
ipfw_sysctl_enable(SYSCTL_HANDLER_ARGS)
{
        struct netmsg_base nmsg;
        int enable, error;

        enable = fw_enable;
        error = sysctl_handle_int(oidp, &enable, 0, req);
        if (error || req->newptr == NULL)
                return error;

        netmsg_init(&nmsg, NULL, &curthread->td_msgport, MSGF_PRIORITY,
            ipfw_sysctl_enable_dispatch);
        nmsg.lmsg.u.ms_result = enable;

        return netisr_domsg(&nmsg, 0);
}

static int
ipfw_sysctl_autoinc_step(SYSCTL_HANDLER_ARGS)
{
        return sysctl_int_range(oidp, arg1, arg2, req,
               IPFW_AUTOINC_STEP_MIN, IPFW_AUTOINC_STEP_MAX);
}

static int
ipfw_sysctl_scancnt(SYSCTL_HANDLER_ARGS)
{

        return sysctl_int_range(oidp, arg1, arg2, req, 1, INT_MAX);
}

static int
ipfw_sysctl_stat(SYSCTL_HANDLER_ARGS)
{
        u_long stat = 0;
        int cpu, error;

        for (cpu = 0; cpu < netisr_ncpus; ++cpu)
                stat += *((u_long *)((uint8_t *)ipfw_ctx[cpu] + arg2));

        error = sysctl_handle_long(oidp, &stat, 0, req);
        if (error || req->newptr == NULL)
                return (error);

        /* Zero out this stat. */
        for (cpu = 0; cpu < netisr_ncpus; ++cpu)
                *((u_long *)((uint8_t *)ipfw_ctx[cpu] + arg2)) = 0;
        return (0);
}

static void
ipfw_ctx_init_dispatch(netmsg_t nmsg)
{
        struct netmsg_ipfw *fwmsg = (struct netmsg_ipfw *)nmsg;
        struct ipfw_context *ctx;
        struct ip_fw *def_rule;

        ASSERT_NETISR_NCPUS(mycpuid);

        ctx = kmalloc(__offsetof(struct ipfw_context,
            ipfw_tables[ipfw_table_max]), M_IPFW, M_WAITOK | M_ZERO);

        RB_INIT(&ctx->ipfw_state_tree);
        TAILQ_INIT(&ctx->ipfw_state_list);

        RB_INIT(&ctx->ipfw_track_tree);
        TAILQ_INIT(&ctx->ipfw_track_list);

        callout_init_mp(&ctx->ipfw_stateto_ch);
        netmsg_init(&ctx->ipfw_stateexp_nm, NULL, &netisr_adone_rport,
            MSGF_DROPABLE | MSGF_PRIORITY, ipfw_state_expire_dispatch);
        ctx->ipfw_stateexp_anch.st_type = O_ANCHOR;
        netmsg_init(&ctx->ipfw_stateexp_more, NULL, &netisr_adone_rport,
            MSGF_DROPABLE, ipfw_state_expire_more_dispatch);

        callout_init_mp(&ctx->ipfw_trackto_ch);
        netmsg_init(&ctx->ipfw_trackexp_nm, NULL, &netisr_adone_rport,
            MSGF_DROPABLE | MSGF_PRIORITY, ipfw_track_expire_dispatch);
        netmsg_init(&ctx->ipfw_trackexp_more, NULL, &netisr_adone_rport,
            MSGF_DROPABLE, ipfw_track_expire_more_dispatch);

        callout_init_mp(&ctx->ipfw_keepalive_ch);
        netmsg_init(&ctx->ipfw_keepalive_nm, NULL, &netisr_adone_rport,
            MSGF_DROPABLE | MSGF_PRIORITY, ipfw_keepalive_dispatch);
        ctx->ipfw_keepalive_anch.st_type = O_ANCHOR;
        netmsg_init(&ctx->ipfw_keepalive_more, NULL, &netisr_adone_rport,
            MSGF_DROPABLE, ipfw_keepalive_more_dispatch);

        callout_init_mp(&ctx->ipfw_xlatreap_ch);
        netmsg_init(&ctx->ipfw_xlatreap_nm, NULL, &netisr_adone_rport,
            MSGF_DROPABLE | MSGF_PRIORITY, ipfw_xlat_reap_dispatch);
        TAILQ_INIT(&ctx->ipfw_xlatreap);

        ipfw_ctx[mycpuid] = ctx;

        def_rule = kmalloc(sizeof(*def_rule), M_IPFW, M_WAITOK | M_ZERO);

        def_rule->act_ofs = 0;
        def_rule->rulenum = IPFW_DEFAULT_RULE;
        def_rule->cmd_len = 1;
        def_rule->set = IPFW_DEFAULT_SET;

        def_rule->cmd[0].len = 1;
#ifdef IPFIREWALL_DEFAULT_TO_ACCEPT
        def_rule->cmd[0].opcode = O_ACCEPT;
#else
        if (filters_default_to_accept)
                def_rule->cmd[0].opcode = O_ACCEPT;
        else
                def_rule->cmd[0].opcode = O_DENY;
#endif

        def_rule->refcnt = 1;
        def_rule->cpuid = mycpuid;

        /* Install the default rule */
        ctx->ipfw_default_rule = def_rule;
        ctx->ipfw_layer3_chain = def_rule;

        /* Link rule CPU sibling */
        ipfw_link_sibling(fwmsg, def_rule);

        /* Statistics only need to be updated once */
        if (mycpuid == 0)
                ipfw_inc_static_count(def_rule);

        netisr_forwardmsg(&nmsg->base, mycpuid + 1);
}

static void
ipfw_crossref_reap_dispatch(netmsg_t nmsg)
{

        crit_enter();
        /* Reply ASAP */
        netisr_replymsg(&nmsg->base, 0);
        crit_exit();
        ipfw_crossref_reap();
}

static void
ipfw_crossref_timeo(void *dummy __unused)
{
        struct netmsg_base *msg = &ipfw_gd.ipfw_crossref_nm;

        KKASSERT(mycpuid == 0);

        crit_enter();
        if (msg->lmsg.ms_flags & MSGF_DONE)
                netisr_sendmsg_oncpu(msg);
        crit_exit();
}

static void
ipfw_ifaddr_dispatch(netmsg_t nmsg)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];
        struct ifnet *ifp = nmsg->lmsg.u.ms_resultp;
        struct ip_fw *f;

        ASSERT_NETISR_NCPUS(mycpuid);

        for (f = ctx->ipfw_layer3_chain; f != NULL; f = f->next) {
                int l, cmdlen;
                ipfw_insn *cmd;

                if ((f->rule_flags & IPFW_RULE_F_DYNIFADDR) == 0)
                        continue;

                for (l = f->cmd_len, cmd = f->cmd; l > 0;
                     l -= cmdlen, cmd += cmdlen) {
                        cmdlen = F_LEN(cmd);
                        if (cmd->opcode == O_IP_SRC_IFIP ||
                            cmd->opcode == O_IP_DST_IFIP) {
                                if (strncmp(ifp->if_xname,
                                    ((ipfw_insn_ifip *)cmd)->ifname,
                                    IFNAMSIZ) == 0)
                                        cmd->arg1 &= ~IPFW_IFIP_VALID;
                        }
                }
        }
        netisr_forwardmsg(&nmsg->base, mycpuid + 1);
}

static void
ipfw_ifaddr(void *arg __unused, struct ifnet *ifp,
    enum ifaddr_event event __unused, struct ifaddr *ifa __unused)
{
        struct netmsg_base nm;

        netmsg_init(&nm, NULL, &curthread->td_msgport, MSGF_PRIORITY,
            ipfw_ifaddr_dispatch);
        nm.lmsg.u.ms_resultp = ifp;
        netisr_domsg_global(&nm);
}

static void
ipfw_init_dispatch(netmsg_t nmsg)
{
        struct netmsg_ipfw fwmsg;
        int error = 0, cpu;

        ASSERT_NETISR0;

        if (IPFW_LOADED) {
                kprintf("IP firewall already loaded\n");
                error = EEXIST;
                goto reply;
        }

        if (ipfw_table_max > UINT16_MAX || ipfw_table_max <= 0)
                ipfw_table_max = UINT16_MAX;

        /* Initialize global track tree. */
        RB_INIT(&ipfw_gd.ipfw_trkcnt_tree);
        IPFW_TRKCNT_TOKINIT;

        /* GC for freed crossref rules. */
        callout_init_mp(&ipfw_gd.ipfw_crossref_ch);
        netmsg_init(&ipfw_gd.ipfw_crossref_nm, NULL, &netisr_adone_rport,
            MSGF_PRIORITY | MSGF_DROPABLE, ipfw_crossref_reap_dispatch);

        ipfw_state_max_set(ipfw_state_max);
        ipfw_state_headroom = 8 * netisr_ncpus;

        bzero(&fwmsg, sizeof(fwmsg));
        netmsg_init(&fwmsg.base, NULL, &curthread->td_msgport, MSGF_PRIORITY,
            ipfw_ctx_init_dispatch);
        netisr_domsg_global(&fwmsg.base);

        ip_fw_chk_ptr = ipfw_chk;
        ip_fw_ctl_ptr = ipfw_ctl;
        ip_fw_dn_io_ptr = ipfw_dummynet_io;

        kprintf("ipfw2 initialized, default to %s, logging ",
                ipfw_ctx[mycpuid]->ipfw_default_rule->cmd[0].opcode ==
                O_ACCEPT ? "accept" : "deny");

#ifdef IPFIREWALL_VERBOSE
        fw_verbose = 1;
#endif
#ifdef IPFIREWALL_VERBOSE_LIMIT
        verbose_limit = IPFIREWALL_VERBOSE_LIMIT;
#endif
        if (fw_verbose == 0) {
                kprintf("disabled\n");
        } else if (verbose_limit == 0) {
                kprintf("unlimited\n");
        } else {
                kprintf("limited to %d packets/entry by default\n",
                        verbose_limit);
        }

        ip_fw_loaded = 1;
        for (cpu = 0; cpu < netisr_ncpus; ++cpu) {
                callout_reset_bycpu(&ipfw_ctx[cpu]->ipfw_stateto_ch, hz,
                    ipfw_state_expire_ipifunc, NULL, cpu);
                callout_reset_bycpu(&ipfw_ctx[cpu]->ipfw_trackto_ch, hz,
                    ipfw_track_expire_ipifunc, NULL, cpu);
                callout_reset_bycpu(&ipfw_ctx[cpu]->ipfw_keepalive_ch, hz,
                    ipfw_keepalive, NULL, cpu);
        }

        if (fw_enable)
                ipfw_hook();

        ipfw_ifaddr_event = EVENTHANDLER_REGISTER(ifaddr_event, ipfw_ifaddr,
            NULL, EVENTHANDLER_PRI_ANY);
        if (ipfw_ifaddr_event == NULL)
                kprintf("ipfw: ifaddr_event register failed\n");

reply:
        netisr_replymsg(&nmsg->base, error);
}

static int
ipfw_init(void)
{
        struct netmsg_base smsg;

        netmsg_init(&smsg, NULL, &curthread->td_msgport, MSGF_PRIORITY,
            ipfw_init_dispatch);
        return netisr_domsg(&smsg, 0);
}

#ifdef KLD_MODULE

static void
ipfw_ctx_fini_dispatch(netmsg_t nmsg)
{
        struct ipfw_context *ctx = ipfw_ctx[mycpuid];

        ASSERT_NETISR_NCPUS(mycpuid);

        callout_cancel(&ctx->ipfw_stateto_ch);
        callout_cancel(&ctx->ipfw_trackto_ch);
        callout_cancel(&ctx->ipfw_keepalive_ch);
        callout_cancel(&ctx->ipfw_xlatreap_ch);

        crit_enter();
        netisr_dropmsg(&ctx->ipfw_stateexp_more);
        netisr_dropmsg(&ctx->ipfw_stateexp_nm);
        netisr_dropmsg(&ctx->ipfw_trackexp_more);
        netisr_dropmsg(&ctx->ipfw_trackexp_nm);
        netisr_dropmsg(&ctx->ipfw_keepalive_more);
        netisr_dropmsg(&ctx->ipfw_keepalive_nm);
        netisr_dropmsg(&ctx->ipfw_xlatreap_nm);
        crit_exit();

        ipfw_table_flushall_oncpu(ctx, 1);

        netisr_forwardmsg(&nmsg->base, mycpuid + 1);
}

static void
ipfw_fini_dispatch(netmsg_t nmsg)
{
        struct netmsg_base nm;
        int error = 0, cpu;

        ASSERT_NETISR0;

        ipfw_crossref_reap();

        if (ipfw_gd.ipfw_refcnt != 0) {
                error = EBUSY;
                goto reply;
        }

        ip_fw_loaded = 0;
        ipfw_dehook();

        /* Synchronize any inflight state/track expire IPIs. */
        lwkt_synchronize_ipiqs("ipfwfini");

        netmsg_init(&nm, NULL, &curthread->td_msgport, MSGF_PRIORITY,
            ipfw_ctx_fini_dispatch);
        netisr_domsg_global(&nm);

        callout_cancel(&ipfw_gd.ipfw_crossref_ch);
        crit_enter();
        netisr_dropmsg(&ipfw_gd.ipfw_crossref_nm);
        crit_exit();

        if (ipfw_ifaddr_event != NULL)
                EVENTHANDLER_DEREGISTER(ifaddr_event, ipfw_ifaddr_event);

        ip_fw_chk_ptr = NULL;
        ip_fw_ctl_ptr = NULL;
        ip_fw_dn_io_ptr = NULL;
        ipfw_flush(1 /* kill default rule */);

        /* Free pre-cpu context */
        for (cpu = 0; cpu < netisr_ncpus; ++cpu)
                kfree(ipfw_ctx[cpu], M_IPFW);

        kprintf("IP firewall unloaded\n");
reply:
        netisr_replymsg(&nmsg->base, error);
}

static void
ipfw_fflush_dispatch(netmsg_t nmsg)
{

        ipfw_flush(0 /* keep default rule */);
        ipfw_crossref_reap();
        netisr_replymsg(&nmsg->base, 0);
}

static int
ipfw_fini(void)
{
        struct netmsg_base smsg;
        int i = 0;

        for (;;) {
                netmsg_init(&smsg, NULL, &curthread->td_msgport, MSGF_PRIORITY,
                    ipfw_fflush_dispatch);
                netisr_domsg(&smsg, 0);

                if (ipfw_gd.ipfw_refcnt == 0)
                        break;
                kprintf("ipfw: flush pending %d\n", ++i);
                tsleep(&smsg, 0, "ipfwff", (3 * hz) / 2);
        }

        netmsg_init(&smsg, NULL, &curthread->td_msgport, MSGF_PRIORITY,
            ipfw_fini_dispatch);
        return netisr_domsg(&smsg, 0);
}

#endif  /* KLD_MODULE */

static int
ipfw_modevent(module_t mod, int type, void *unused)
{
        int err = 0;

        switch (type) {
        case MOD_LOAD:
                err = ipfw_init();
                break;

        case MOD_UNLOAD:
#ifndef KLD_MODULE
                kprintf("ipfw statically compiled, cannot unload\n");
                err = EBUSY;
#else
                err = ipfw_fini();
#endif
                break;
        default:
                break;
        }
        return err;
}

static moduledata_t ipfwmod = {
        "ipfw",
        ipfw_modevent,
        0
};
DECLARE_MODULE(ipfw, ipfwmod, SI_SUB_PROTO_END, SI_ORDER_ANY);
MODULE_VERSION(ipfw, 1);