sys/net/if.c

   1 /*
   2  * Copyright (c) 1980, 1986, 1993
   3  *      The Regents of the University of California.  All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  * 3. Neither the name of the University nor the names of its contributors
  14  *    may be used to endorse or promote products derived from this software
  15  *    without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  *      @(#)if.c        8.3 (Berkeley) 1/4/94
  30  * $FreeBSD: src/sys/net/if.c,v 1.185 2004/03/13 02:35:03 brooks Exp $
  31  */
  32
  33 #include "opt_inet6.h"
  34 #include "opt_inet.h"
  35 #include "opt_ifpoll.h"
  36
  37 #include <sys/param.h>
  38 #include <sys/malloc.h>
  39 #include <sys/mbuf.h>
  40 #include <sys/systm.h>
  41 #include <sys/proc.h>
  42 #include <sys/caps.h>
  43 #include <sys/protosw.h>
  44 #include <sys/socket.h>
  45 #include <sys/socketvar.h>
  46 #include <sys/socketops.h>
  47 #include <sys/kernel.h>
  48 #include <sys/ktr.h>
  49 #include <sys/mutex.h>
  50 #include <sys/lock.h>
  51 #include <sys/sockio.h>
  52 #include <sys/syslog.h>
  53 #include <sys/sysctl.h>
  54 #include <sys/domain.h>
  55 #include <sys/thread.h>
  56 #include <sys/serialize.h>
  57 #include <sys/bus.h>
  58 #include <sys/jail.h>
  59
  60 #include <sys/thread2.h>
  61 #include <sys/msgport2.h>
  62 #include <sys/mutex2.h>
  63
  64 #include <net/if.h>
  65 #include <net/if_arp.h>
  66 #include <net/if_dl.h>
  67 #include <net/if_types.h>
  68 #include <net/if_var.h>
  69 #include <net/if_ringmap.h>
  70 #include <net/ifq_var.h>
  71 #include <net/radix.h>
  72 #include <net/route.h>
  73 #include <net/if_clone.h>
  74 #include <net/netisr2.h>
  75 #include <net/netmsg2.h>
  76
  77 #include <machine/atomic.h>
  78 #include <machine/stdarg.h>
  79 #include <machine/smp.h>
  80
  81 #if defined(INET) || defined(INET6)
  82 #include <netinet/in.h>
  83 #include <netinet/in_var.h>
  84 #include <netinet/if_ether.h>
  85 #ifdef INET6
  86 #include <netinet6/in6_var.h>
  87 #include <netinet6/in6_ifattach.h>
  88 #endif /* INET6 */
  89 #endif /* INET || INET6 */
  90
  91 struct netmsg_ifaddr {
  92         struct netmsg_base base;
  93         struct ifaddr   *ifa;
  94         struct ifnet    *ifp;
  95         int             tail;
  96 };
  97
  98 struct ifsubq_stage_head {
  99         TAILQ_HEAD(, ifsubq_stage)      stg_head;
 100 } __cachealign;
 101
 102 struct if_ringmap {
 103         int             rm_cnt;
 104         int             rm_grid;
 105         int             rm_cpumap[];
 106 };
 107
 108 #define RINGMAP_FLAG_NONE               0x0
 109 #define RINGMAP_FLAG_POWEROF2           0x1
 110
 111 /*
 112  * System initialization
 113  */
 114 static void     if_attachdomain(void *);
 115 static void     if_attachdomain1(struct ifnet *);
 116 static int      ifconf(u_long, caddr_t, struct ucred *);
 117 static void     ifinit(void *);
 118 static void     ifnetinit(void *);
 119 static void     if_slowtimo(void *);
 120 static int      if_rtdel(struct radix_node *, void *);
 121 static void     if_slowtimo_dispatch(netmsg_t);
 122
 123 /* Helper functions */
 124 static void     ifsq_watchdog_reset(struct ifsubq_watchdog *);
 125 static int      if_delmulti_serialized(struct ifnet *, struct sockaddr *);
 126 static struct ifnet_array *ifnet_array_alloc(int);
 127 static void     ifnet_array_free(struct ifnet_array *);
 128 static struct ifnet_array *ifnet_array_add(struct ifnet *,
 129                     const struct ifnet_array *);
 130 static struct ifnet_array *ifnet_array_del(struct ifnet *,
 131                     const struct ifnet_array *);
 132 static struct ifg_group *if_creategroup(const char *);
 133 static int      if_destroygroup(struct ifg_group *);
 134 static int      if_delgroup_locked(struct ifnet *, const char *);
 135 static int      if_getgroups(struct ifgroupreq *, struct ifnet *);
 136 static int      if_getgroupmembers(struct ifgroupreq *);
 137
 138 #ifdef INET6
 139 /*
 140  * XXX: declare here to avoid to include many inet6 related files..
 141  * should be more generalized?
 142  */
 143 extern void     nd6_setmtu(struct ifnet *);
 144 #endif
 145
 146 SYSCTL_NODE(_net, PF_LINK, link, CTLFLAG_RW, 0, "Link layers");
 147 SYSCTL_NODE(_net_link, 0, generic, CTLFLAG_RW, 0, "Generic link-management");
 148 SYSCTL_NODE(_net_link, OID_AUTO, ringmap, CTLFLAG_RW, 0, "link ringmap");
 149
 150 static int ifsq_stage_cntmax = 16;
 151 TUNABLE_INT("net.link.stage_cntmax", &ifsq_stage_cntmax);
 152 SYSCTL_INT(_net_link, OID_AUTO, stage_cntmax, CTLFLAG_RW,
 153     &ifsq_stage_cntmax, 0, "ifq staging packet count max");
 154
 155 static int if_stats_compat = 0;
 156 SYSCTL_INT(_net_link, OID_AUTO, stats_compat, CTLFLAG_RW,
 157     &if_stats_compat, 0, "Compat the old ifnet stats");
 158
 159 static int if_ringmap_dumprdr = 0;
 160 SYSCTL_INT(_net_link_ringmap, OID_AUTO, dump_rdr, CTLFLAG_RW,
 161     &if_ringmap_dumprdr, 0, "dump redirect table");
 162
 163 /* Interface description */
 164 static unsigned int ifdescr_maxlen = 1024;
 165 SYSCTL_UINT(_net, OID_AUTO, ifdescr_maxlen, CTLFLAG_RW,
 166         &ifdescr_maxlen, 0,
 167         "administrative maximum length for interface description");
 168
 169 SYSINIT(interfaces, SI_SUB_PROTO_IF, SI_ORDER_FIRST, ifinit, NULL);
 170 SYSINIT(ifnet, SI_SUB_PRE_DRIVERS, SI_ORDER_ANY, ifnetinit, NULL);
 171
 172 static if_com_alloc_t *if_com_alloc[256];
 173 static if_com_free_t *if_com_free[256];
 174
 175 MALLOC_DEFINE(M_IFADDR, "ifaddr", "interface address");
 176 MALLOC_DEFINE(M_IFMADDR, "ether_multi", "link-level multicast address");
 177 MALLOC_DEFINE(M_IFNET, "ifnet", "interface structure");
 178 MALLOC_DEFINE(M_IFDESCR, "ifdescr", "ifnet descriptions");
 179
 180 int                     ifqmaxlen = IFQ_MAXLEN;
 181 struct ifnethead        ifnet = TAILQ_HEAD_INITIALIZER(ifnet);
 182 struct ifgrouphead      ifg_head = TAILQ_HEAD_INITIALIZER(ifg_head);
 183 static struct lock      ifgroup_lock;
 184
 185 static struct ifnet_array       ifnet_array0;
 186 static struct ifnet_array       *ifnet_array = &ifnet_array0;
 187
 188 static struct callout           if_slowtimo_timer;
 189 static struct netmsg_base       if_slowtimo_netmsg;
 190
 191 int                     if_index = 0;
 192 struct ifnet            **ifindex2ifnet = NULL;
 193 static struct mtx       ifnet_mtx = MTX_INITIALIZER("ifnet");
 194
 195 static struct ifsubq_stage_head ifsubq_stage_heads[MAXCPU];
 196
 197 #ifdef notyet
 198 #define IFQ_KTR_STRING          "ifq=%p"
 199 #define IFQ_KTR_ARGS            struct ifaltq *ifq
 200 #ifndef KTR_IFQ
 201 #define KTR_IFQ                 KTR_ALL
 202 #endif
 203 KTR_INFO_MASTER(ifq);
 204 KTR_INFO(KTR_IFQ, ifq, enqueue, 0, IFQ_KTR_STRING, IFQ_KTR_ARGS);
 205 KTR_INFO(KTR_IFQ, ifq, dequeue, 1, IFQ_KTR_STRING, IFQ_KTR_ARGS);
 206 #define logifq(name, arg)       KTR_LOG(ifq_ ## name, arg)
 207
 208 #define IF_START_KTR_STRING     "ifp=%p"
 209 #define IF_START_KTR_ARGS       struct ifnet *ifp
 210 #ifndef KTR_IF_START
 211 #define KTR_IF_START            KTR_ALL
 212 #endif
 213 KTR_INFO_MASTER(if_start);
 214 KTR_INFO(KTR_IF_START, if_start, run, 0,
 215          IF_START_KTR_STRING, IF_START_KTR_ARGS);
 216 KTR_INFO(KTR_IF_START, if_start, sched, 1,
 217          IF_START_KTR_STRING, IF_START_KTR_ARGS);
 218 KTR_INFO(KTR_IF_START, if_start, avoid, 2,
 219          IF_START_KTR_STRING, IF_START_KTR_ARGS);
 220 KTR_INFO(KTR_IF_START, if_start, contend_sched, 3,
 221          IF_START_KTR_STRING, IF_START_KTR_ARGS);
 222 KTR_INFO(KTR_IF_START, if_start, chase_sched, 4,
 223          IF_START_KTR_STRING, IF_START_KTR_ARGS);
 224 #define logifstart(name, arg)   KTR_LOG(if_start_ ## name, arg)
 225 #endif /* notyet */
 226
 227 /*
 228  * Network interface utility routines.
 229  *
 230  * Routines with ifa_ifwith* names take sockaddr *'s as
 231  * parameters.
 232  */
 233 /* ARGSUSED */
 234 static void
 235 ifinit(void *dummy)
 236 {
 237         lockinit(&ifgroup_lock, "ifgroup", 0, 0);
 238
 239         callout_init_mp(&if_slowtimo_timer);
 240         netmsg_init(&if_slowtimo_netmsg, NULL, &netisr_adone_rport,
 241             MSGF_PRIORITY, if_slowtimo_dispatch);
 242
 243         /* Start if_slowtimo */
 244         lwkt_sendmsg(netisr_cpuport(0), &if_slowtimo_netmsg.lmsg);
 245 }
 246
 247 static void
 248 ifsq_ifstart_ipifunc(void *arg)
 249 {
 250         struct ifaltq_subque *ifsq = arg;
 251         struct lwkt_msg *lmsg = ifsq_get_ifstart_lmsg(ifsq, mycpuid);
 252
 253         crit_enter();
 254         if (lmsg->ms_flags & MSGF_DONE)
 255                 lwkt_sendmsg_oncpu(netisr_cpuport(mycpuid), lmsg);
 256         crit_exit();
 257 }
 258
 259 static __inline void
 260 ifsq_stage_remove(struct ifsubq_stage_head *head, struct ifsubq_stage *stage)
 261 {
 262         KKASSERT(stage->stg_flags & IFSQ_STAGE_FLAG_QUED);
 263         TAILQ_REMOVE(&head->stg_head, stage, stg_link);
 264         stage->stg_flags &= ~(IFSQ_STAGE_FLAG_QUED | IFSQ_STAGE_FLAG_SCHED);
 265         stage->stg_cnt = 0;
 266         stage->stg_len = 0;
 267 }
 268
 269 static __inline void
 270 ifsq_stage_insert(struct ifsubq_stage_head *head, struct ifsubq_stage *stage)
 271 {
 272         KKASSERT((stage->stg_flags &
 273             (IFSQ_STAGE_FLAG_QUED | IFSQ_STAGE_FLAG_SCHED)) == 0);
 274         stage->stg_flags |= IFSQ_STAGE_FLAG_QUED;
 275         TAILQ_INSERT_TAIL(&head->stg_head, stage, stg_link);
 276 }
 277
 278 /*
 279  * Schedule ifnet.if_start on the subqueue owner CPU
 280  */
 281 static void
 282 ifsq_ifstart_schedule(struct ifaltq_subque *ifsq, int force)
 283 {
 284         int cpu;
 285
 286         if (!force && curthread->td_type == TD_TYPE_NETISR &&
 287             ifsq_stage_cntmax > 0) {
 288                 struct ifsubq_stage *stage = ifsq_get_stage(ifsq, mycpuid);
 289
 290                 stage->stg_cnt = 0;
 291                 stage->stg_len = 0;
 292                 if ((stage->stg_flags & IFSQ_STAGE_FLAG_QUED) == 0)
 293                         ifsq_stage_insert(&ifsubq_stage_heads[mycpuid], stage);
 294                 stage->stg_flags |= IFSQ_STAGE_FLAG_SCHED;
 295                 return;
 296         }
 297
 298         cpu = ifsq_get_cpuid(ifsq);
 299         if (cpu != mycpuid)
 300                 lwkt_send_ipiq(globaldata_find(cpu), ifsq_ifstart_ipifunc, ifsq);
 301         else
 302                 ifsq_ifstart_ipifunc(ifsq);
 303 }
 304
 305 /*
 306  * NOTE:
 307  * This function will release ifnet.if_start subqueue interlock,
 308  * if ifnet.if_start for the subqueue does not need to be scheduled
 309  */
 310 static __inline int
 311 ifsq_ifstart_need_schedule(struct ifaltq_subque *ifsq, int running)
 312 {
 313         if (!running || ifsq_is_empty(ifsq)
 314 #ifdef ALTQ
 315             || ifsq->ifsq_altq->altq_tbr != NULL
 316 #endif
 317         ) {
 318                 ALTQ_SQ_LOCK(ifsq);
 319                 /*
 320                  * ifnet.if_start subqueue interlock is released, if:
 321                  * 1) Hardware can not take any packets, due to
 322                  *    o  interface is marked down
 323                  *    o  hardware queue is full (ifsq_is_oactive)
 324                  *    Under the second situation, hardware interrupt
 325                  *    or polling(4) will call/schedule ifnet.if_start
 326                  *    on the subqueue when hardware queue is ready
 327                  * 2) There is no packet in the subqueue.
 328                  *    Further ifq_dispatch or ifq_handoff will call/
 329                  *    schedule ifnet.if_start on the subqueue.
 330                  * 3) TBR is used and it does not allow further
 331                  *    dequeueing.
 332                  *    TBR callout will call ifnet.if_start on the
 333                  *    subqueue.
 334                  */
 335                 if (!running || !ifsq_data_ready(ifsq)) {
 336                         ifsq_clr_started(ifsq);
 337                         ALTQ_SQ_UNLOCK(ifsq);
 338                         return 0;
 339                 }
 340                 ALTQ_SQ_UNLOCK(ifsq);
 341         }
 342         return 1;
 343 }
 344
 345 static void
 346 ifsq_ifstart_dispatch(netmsg_t msg)
 347 {
 348         struct lwkt_msg *lmsg = &msg->base.lmsg;
 349         struct ifaltq_subque *ifsq = lmsg->u.ms_resultp;
 350         struct ifnet *ifp = ifsq_get_ifp(ifsq);
 351         struct globaldata *gd = mycpu;
 352         int running = 0, need_sched;
 353
 354         crit_enter_gd(gd);
 355
 356         lwkt_replymsg(lmsg, 0); /* reply ASAP */
 357
 358         if (gd->gd_cpuid != ifsq_get_cpuid(ifsq)) {
 359                 /*
 360                  * We need to chase the subqueue owner CPU change.
 361                  */
 362                 ifsq_ifstart_schedule(ifsq, 1);
 363                 crit_exit_gd(gd);
 364                 return;
 365         }
 366
 367         ifsq_serialize_hw(ifsq);
 368         if ((ifp->if_flags & IFF_RUNNING) && !ifsq_is_oactive(ifsq)) {
 369                 ifp->if_start(ifp, ifsq);
 370                 if ((ifp->if_flags & IFF_RUNNING) && !ifsq_is_oactive(ifsq))
 371                         running = 1;
 372         }
 373         need_sched = ifsq_ifstart_need_schedule(ifsq, running);
 374         ifsq_deserialize_hw(ifsq);
 375
 376         if (need_sched) {
 377                 /*
 378                  * More data need to be transmitted, ifnet.if_start is
 379                  * scheduled on the subqueue owner CPU, and we keep going.
 380                  * NOTE: ifnet.if_start subqueue interlock is not released.
 381                  */
 382                 ifsq_ifstart_schedule(ifsq, 0);
 383         }
 384
 385         crit_exit_gd(gd);
 386 }
 387
 388 /* Device driver ifnet.if_start helper function */
 389 void
 390 ifsq_devstart(struct ifaltq_subque *ifsq)
 391 {
 392         struct ifnet *ifp = ifsq_get_ifp(ifsq);
 393         int running = 0;
 394
 395         ASSERT_ALTQ_SQ_SERIALIZED_HW(ifsq);
 396
 397         ALTQ_SQ_LOCK(ifsq);
 398         if (ifsq_is_started(ifsq) || !ifsq_data_ready(ifsq)) {
 399                 ALTQ_SQ_UNLOCK(ifsq);
 400                 return;
 401         }
 402         ifsq_set_started(ifsq);
 403         ALTQ_SQ_UNLOCK(ifsq);
 404
 405         ifp->if_start(ifp, ifsq);
 406
 407         if ((ifp->if_flags & IFF_RUNNING) && !ifsq_is_oactive(ifsq))
 408                 running = 1;
 409
 410         if (ifsq_ifstart_need_schedule(ifsq, running)) {
 411                 /*
 412                  * More data need to be transmitted, ifnet.if_start is
 413                  * scheduled on ifnet's CPU, and we keep going.
 414                  * NOTE: ifnet.if_start interlock is not released.
 415                  */
 416                 ifsq_ifstart_schedule(ifsq, 0);
 417         }
 418 }
 419
 420 void
 421 if_devstart(struct ifnet *ifp)
 422 {
 423         ifsq_devstart(ifq_get_subq_default(&ifp->if_snd));
 424 }
 425
 426 /* Device driver ifnet.if_start schedule helper function */
 427 void
 428 ifsq_devstart_sched(struct ifaltq_subque *ifsq)
 429 {
 430         ifsq_ifstart_schedule(ifsq, 1);
 431 }
 432
 433 void
 434 if_devstart_sched(struct ifnet *ifp)
 435 {
 436         ifsq_devstart_sched(ifq_get_subq_default(&ifp->if_snd));
 437 }
 438
 439 static void
 440 if_default_serialize(struct ifnet *ifp, enum ifnet_serialize slz __unused)
 441 {
 442         lwkt_serialize_enter(ifp->if_serializer);
 443 }
 444
 445 static void
 446 if_default_deserialize(struct ifnet *ifp, enum ifnet_serialize slz __unused)
 447 {
 448         lwkt_serialize_exit(ifp->if_serializer);
 449 }
 450
 451 static int
 452 if_default_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz __unused)
 453 {
 454         return lwkt_serialize_try(ifp->if_serializer);
 455 }
 456
 457 #ifdef INVARIANTS
 458 static void
 459 if_default_serialize_assert(struct ifnet *ifp,
 460                             enum ifnet_serialize slz __unused,
 461                             boolean_t serialized)
 462 {
 463         if (serialized)
 464                 ASSERT_SERIALIZED(ifp->if_serializer);
 465         else
 466                 ASSERT_NOT_SERIALIZED(ifp->if_serializer);
 467 }
 468 #endif
 469
 470 /*
 471  * Attach an interface to the list of "active" interfaces.
 472  *
 473  * The serializer is optional.
 474  */
 475 void
 476 if_attach(struct ifnet *ifp, lwkt_serialize_t serializer)
 477 {
 478         unsigned socksize;
 479         int namelen, masklen;
 480         struct sockaddr_dl *sdl, *sdl_addr;
 481         struct ifaddr *ifa;
 482         struct ifaltq *ifq;
 483         struct ifnet **old_ifindex2ifnet = NULL;
 484         struct ifnet_array *old_ifnet_array;
 485         int i, q, qlen;
 486         char qlenname[64];
 487
 488         static int if_indexlim = 8;
 489
 490         if (ifp->if_serialize != NULL) {
 491                 KASSERT(ifp->if_deserialize != NULL &&
 492                         ifp->if_tryserialize != NULL &&
 493                         ifp->if_serialize_assert != NULL,
 494                         ("serialize functions are partially setup"));
 495
 496                 /*
 497                  * If the device supplies serialize functions,
 498                  * then clear if_serializer to catch any invalid
 499                  * usage of this field.
 500                  */
 501                 KASSERT(serializer == NULL,
 502                         ("both serialize functions and default serializer "
 503                          "are supplied"));
 504                 ifp->if_serializer = NULL;
 505         } else {
 506                 KASSERT(ifp->if_deserialize == NULL &&
 507                         ifp->if_tryserialize == NULL &&
 508                         ifp->if_serialize_assert == NULL,
 509                         ("serialize functions are partially setup"));
 510                 ifp->if_serialize = if_default_serialize;
 511                 ifp->if_deserialize = if_default_deserialize;
 512                 ifp->if_tryserialize = if_default_tryserialize;
 513 #ifdef INVARIANTS
 514                 ifp->if_serialize_assert = if_default_serialize_assert;
 515 #endif
 516
 517                 /*
 518                  * The serializer can be passed in from the device,
 519                  * allowing the same serializer to be used for both
 520                  * the interrupt interlock and the device queue.
 521                  * If not specified, the netif structure will use an
 522                  * embedded serializer.
 523                  */
 524                 if (serializer == NULL) {
 525                         serializer = &ifp->if_default_serializer;
 526                         lwkt_serialize_init(serializer);
 527                 }
 528                 ifp->if_serializer = serializer;
 529         }
 530
 531         /*
 532          * Make if_addrhead available on all CPUs, since they
 533          * could be accessed by any threads.
 534          */
 535         ifp->if_addrheads = kmalloc(ncpus * sizeof(struct ifaddrhead),
 536                                     M_IFADDR, M_WAITOK | M_ZERO);
 537         for (i = 0; i < ncpus; ++i)
 538                 TAILQ_INIT(&ifp->if_addrheads[i]);
 539
 540         TAILQ_INIT(&ifp->if_multiaddrs);
 541         TAILQ_INIT(&ifp->if_groups);
 542         getmicrotime(&ifp->if_lastchange);
 543         if_addgroup(ifp, IFG_ALL);
 544
 545         /*
 546          * create a Link Level name for this device
 547          */
 548         namelen = strlen(ifp->if_xname);
 549         masklen = offsetof(struct sockaddr_dl, sdl_data[0]) + namelen;
 550         socksize = masklen + ifp->if_addrlen;
 551         if (socksize < sizeof(*sdl))
 552                 socksize = sizeof(*sdl);
 553         socksize = RT_ROUNDUP(socksize);
 554         ifa = ifa_create(sizeof(struct ifaddr) + 2 * socksize);
 555         sdl = sdl_addr = (struct sockaddr_dl *)(ifa + 1);
 556         sdl->sdl_len = socksize;
 557         sdl->sdl_family = AF_LINK;
 558         bcopy(ifp->if_xname, sdl->sdl_data, namelen);
 559         sdl->sdl_nlen = namelen;
 560         sdl->sdl_type = ifp->if_type;
 561         ifp->if_lladdr = ifa;
 562         ifa->ifa_ifp = ifp;
 563         ifa->ifa_addr = (struct sockaddr *)sdl;
 564         sdl = (struct sockaddr_dl *)(socksize + (caddr_t)sdl);
 565         ifa->ifa_netmask = (struct sockaddr *)sdl;
 566         sdl->sdl_len = masklen;
 567         while (namelen != 0)
 568                 sdl->sdl_data[--namelen] = 0xff;
 569         ifa_iflink(ifa, ifp, 0 /* Insert head */);
 570
 571         /*
 572          * Make if_data available on all CPUs, since they could
 573          * be updated by hardware interrupt routing, which could
 574          * be bound to any CPU.
 575          */
 576         ifp->if_data_pcpu = kmalloc(ncpus * sizeof(struct ifdata_pcpu),
 577                                     M_DEVBUF,
 578                                     M_WAITOK | M_ZERO | M_CACHEALIGN);
 579
 580         if (ifp->if_mapsubq == NULL)
 581                 ifp->if_mapsubq = ifq_mapsubq_default;
 582
 583         ifq = &ifp->if_snd;
 584         ifq->altq_type = 0;
 585         ifq->altq_disc = NULL;
 586         ifq->altq_flags &= ALTQF_CANTCHANGE;
 587         ifq->altq_tbr = NULL;
 588         ifq->altq_ifp = ifp;
 589
 590         if (ifq->altq_subq_cnt <= 0)
 591                 ifq->altq_subq_cnt = 1;
 592         ifq->altq_subq =
 593                 kmalloc(ifq->altq_subq_cnt * sizeof(struct ifaltq_subque),
 594                         M_DEVBUF,
 595                         M_WAITOK | M_ZERO | M_CACHEALIGN);
 596
 597         if (ifq->altq_maxlen == 0) {
 598                 if_printf(ifp, "driver didn't set altq_maxlen\n");
 599                 ifq_set_maxlen(ifq, ifqmaxlen);
 600         }
 601
 602         /* Allow user to override driver's setting. */
 603         ksnprintf(qlenname, sizeof(qlenname), "net.%s.qlenmax", ifp->if_xname);
 604         qlen = -1;
 605         TUNABLE_INT_FETCH(qlenname, &qlen);
 606         if (qlen > 0) {
 607                 if_printf(ifp, "qlenmax -> %d\n", qlen);
 608                 ifq_set_maxlen(ifq, qlen);
 609         }
 610
 611         for (q = 0; q < ifq->altq_subq_cnt; ++q) {
 612                 struct ifaltq_subque *ifsq = &ifq->altq_subq[q];
 613
 614                 ALTQ_SQ_LOCK_INIT(ifsq);
 615                 ifsq->ifsq_index = q;
 616
 617                 ifsq->ifsq_altq = ifq;
 618                 ifsq->ifsq_ifp = ifp;
 619
 620                 ifsq->ifsq_maxlen = ifq->altq_maxlen;
 621                 ifsq->ifsq_maxbcnt = ifsq->ifsq_maxlen * MCLBYTES;
 622                 ifsq->ifsq_prepended = NULL;
 623                 ifsq->ifsq_started = 0;
 624                 ifsq->ifsq_hw_oactive = 0;
 625                 ifsq_set_cpuid(ifsq, 0);
 626                 if (ifp->if_serializer != NULL)
 627                         ifsq_set_hw_serialize(ifsq, ifp->if_serializer);
 628
 629                 /* XXX: netisr_ncpus */
 630                 ifsq->ifsq_stage =
 631                         kmalloc(ncpus * sizeof(struct ifsubq_stage),
 632                                 M_DEVBUF,
 633                                 M_WAITOK | M_ZERO | M_CACHEALIGN);
 634                 for (i = 0; i < ncpus; ++i)
 635                         ifsq->ifsq_stage[i].stg_subq = ifsq;
 636
 637                 /*
 638                  * Allocate one if_start message for each CPU, since
 639                  * the hardware TX ring could be assigned to any CPU.
 640                  *
 641                  * NOTE:
 642                  * If the hardware TX ring polling CPU and the hardware
 643                  * TX ring interrupt CPU are same, one if_start message
 644                  * should be enough.
 645                  */
 646                 ifsq->ifsq_ifstart_nmsg =
 647                     kmalloc(ncpus * sizeof(struct netmsg_base),
 648                     M_LWKTMSG, M_WAITOK);
 649                 for (i = 0; i < ncpus; ++i) {
 650                         netmsg_init(&ifsq->ifsq_ifstart_nmsg[i], NULL,
 651                             &netisr_adone_rport, 0, ifsq_ifstart_dispatch);
 652                         ifsq->ifsq_ifstart_nmsg[i].lmsg.u.ms_resultp = ifsq;
 653                 }
 654         }
 655         ifq_set_classic(ifq);
 656
 657         /*
 658          * Increase mbuf cluster/jcluster limits for the mbufs that
 659          * could sit on the device queues for quite some time.
 660          */
 661         if (ifp->if_nmbclusters > 0)
 662                 mcl_inclimit(ifp->if_nmbclusters);
 663         if (ifp->if_nmbjclusters > 0)
 664                 mjcl_inclimit(ifp->if_nmbjclusters);
 665
 666         /*
 667          * Install this ifp into ifindex2inet, ifnet queue and ifnet
 668          * array after it is setup.
 669          *
 670          * Protect ifindex2ifnet, ifnet queue and ifnet array changes
 671          * by ifnet lock, so that non-netisr threads could get a
 672          * consistent view.
 673          */
 674         ifnet_lock();
 675
 676         /* Don't update if_index until ifindex2ifnet is setup */
 677         ifp->if_index = if_index + 1;
 678         sdl_addr->sdl_index = ifp->if_index;
 679
 680         /*
 681          * Install this ifp into ifindex2ifnet
 682          */
 683         if (ifindex2ifnet == NULL || ifp->if_index >= if_indexlim) {
 684                 unsigned int n;
 685                 struct ifnet **q;
 686
 687                 /*
 688                  * Grow ifindex2ifnet
 689                  */
 690                 if_indexlim <<= 1;
 691                 n = if_indexlim * sizeof(*q);
 692                 q = kmalloc(n, M_IFADDR, M_WAITOK | M_ZERO);
 693                 if (ifindex2ifnet != NULL) {
 694                         bcopy(ifindex2ifnet, q, n/2);
 695                         /* Free old ifindex2ifnet after sync all netisrs */
 696                         old_ifindex2ifnet = ifindex2ifnet;
 697                 }
 698                 ifindex2ifnet = q;
 699         }
 700         ifindex2ifnet[ifp->if_index] = ifp;
 701         /*
 702          * Update if_index after this ifp is installed into ifindex2ifnet,
 703          * so that netisrs could get a consistent view of ifindex2ifnet.
 704          */
 705         cpu_sfence();
 706         if_index = ifp->if_index;
 707
 708         /*
 709          * Install this ifp into ifnet array.
 710          */
 711         /* Free old ifnet array after sync all netisrs */
 712         old_ifnet_array = ifnet_array;
 713         ifnet_array = ifnet_array_add(ifp, old_ifnet_array);
 714
 715         /*
 716          * Install this ifp into ifnet queue.
 717          */
 718         TAILQ_INSERT_TAIL(&ifnetlist, ifp, if_link);
 719
 720         ifnet_unlock();
 721
 722         /*
 723          * Sync all netisrs so that the old ifindex2ifnet and ifnet array
 724          * are no longer accessed and we can free them safely later on.
 725          */
 726         netmsg_service_sync();
 727         if (old_ifindex2ifnet != NULL)
 728                 kfree(old_ifindex2ifnet, M_IFADDR);
 729         ifnet_array_free(old_ifnet_array);
 730
 731         if (!SLIST_EMPTY(&domains))
 732                 if_attachdomain1(ifp);
 733
 734         /* Announce the interface. */
 735         EVENTHANDLER_INVOKE(ifnet_attach_event, ifp);
 736         devctl_notify("IFNET", ifp->if_xname, "ATTACH", NULL);
 737         rt_ifannouncemsg(ifp, IFAN_ARRIVAL);
 738 }
 739
 740 static void
 741 if_attachdomain(void *dummy)
 742 {
 743         struct ifnet *ifp;
 744
 745         ifnet_lock();
 746         TAILQ_FOREACH(ifp, &ifnetlist, if_list)
 747                 if_attachdomain1(ifp);
 748         ifnet_unlock();
 749 }
 750 SYSINIT(domainifattach, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_FIRST,
 751         if_attachdomain, NULL);
 752
 753 static void
 754 if_attachdomain1(struct ifnet *ifp)
 755 {
 756         struct domain *dp;
 757
 758         crit_enter();
 759
 760         /* address family dependent data region */
 761         bzero(ifp->if_afdata, sizeof(ifp->if_afdata));
 762         SLIST_FOREACH(dp, &domains, dom_next)
 763                 if (dp->dom_ifattach)
 764                         ifp->if_afdata[dp->dom_family] =
 765                                 (*dp->dom_ifattach)(ifp);
 766         crit_exit();
 767 }
 768
 769 /*
 770  * Purge all addresses whose type is _not_ AF_LINK
 771  */
 772 static void
 773 if_purgeaddrs_nolink_dispatch(netmsg_t nmsg)
 774 {
 775         struct ifnet *ifp = nmsg->lmsg.u.ms_resultp;
 776         struct ifaddr_container *ifac, *next;
 777
 778         ASSERT_NETISR0;
 779
 780         /*
 781          * The ifaddr processing in the following loop will block,
 782          * however, this function is called in netisr0, in which
 783          * ifaddr list changes happen, so we don't care about the
 784          * blockness of the ifaddr processing here.
 785          */
 786         TAILQ_FOREACH_MUTABLE(ifac, &ifp->if_addrheads[mycpuid],
 787                               ifa_link, next) {
 788                 struct ifaddr *ifa = ifac->ifa;
 789
 790                 /* Ignore marker */
 791                 if (ifa->ifa_addr->sa_family == AF_UNSPEC)
 792                         continue;
 793
 794                 /* Leave link ifaddr as it is */
 795                 if (ifa->ifa_addr->sa_family == AF_LINK)
 796                         continue;
 797 #ifdef INET
 798                 /* XXX: Ugly!! ad hoc just for INET */
 799                 if (ifa->ifa_addr->sa_family == AF_INET) {
 800                         struct ifaliasreq ifr;
 801                         struct sockaddr_in saved_addr, saved_dst;
 802 #ifdef IFADDR_DEBUG_VERBOSE
 803                         int i;
 804
 805                         kprintf("purge in4 addr %p: ", ifa);
 806                         for (i = 0; i < ncpus; ++i) {
 807                                 kprintf("%d ",
 808                                     ifa->ifa_containers[i].ifa_refcnt);
 809                         }
 810                         kprintf("\n");
 811 #endif
 812
 813                         /* Save information for panic. */
 814                         memcpy(&saved_addr, ifa->ifa_addr, sizeof(saved_addr));
 815                         if (ifa->ifa_dstaddr != NULL) {
 816                                 memcpy(&saved_dst, ifa->ifa_dstaddr,
 817                                     sizeof(saved_dst));
 818                         } else {
 819                                 memset(&saved_dst, 0, sizeof(saved_dst));
 820                         }
 821
 822                         bzero(&ifr, sizeof ifr);
 823                         ifr.ifra_addr = *ifa->ifa_addr;
 824                         if (ifa->ifa_dstaddr)
 825                                 ifr.ifra_broadaddr = *ifa->ifa_dstaddr;
 826                         if (in_control(SIOCDIFADDR, (caddr_t)&ifr, ifp,
 827                                        NULL) == 0)
 828                                 continue;
 829
 830                         /* MUST NOT HAPPEN */
 831                         panic("%s: in_control failed %x, dst %x", ifp->if_xname,
 832                             ntohl(saved_addr.sin_addr.s_addr),
 833                             ntohl(saved_dst.sin_addr.s_addr));
 834                 }
 835 #endif /* INET */
 836 #ifdef INET6
 837                 if (ifa->ifa_addr->sa_family == AF_INET6) {
 838 #ifdef IFADDR_DEBUG_VERBOSE
 839                         int i;
 840
 841                         kprintf("purge in6 addr %p: ", ifa);
 842                         for (i = 0; i < ncpus; ++i) {
 843                                 kprintf("%d ",
 844                                     ifa->ifa_containers[i].ifa_refcnt);
 845                         }
 846                         kprintf("\n");
 847 #endif
 848
 849                         in6_purgeaddr(ifa);
 850                         /* ifp_addrhead is already updated */
 851                         continue;
 852                 }
 853 #endif /* INET6 */
 854                 if_printf(ifp, "destroy ifaddr family %d\n",
 855                     ifa->ifa_addr->sa_family);
 856                 ifa_ifunlink(ifa, ifp);
 857                 ifa_destroy(ifa);
 858         }
 859
 860         netisr_replymsg(&nmsg->base, 0);
 861 }
 862
 863 void
 864 if_purgeaddrs_nolink(struct ifnet *ifp)
 865 {
 866         struct netmsg_base nmsg;
 867
 868         netmsg_init(&nmsg, NULL, &curthread->td_msgport, 0,
 869             if_purgeaddrs_nolink_dispatch);
 870         nmsg.lmsg.u.ms_resultp = ifp;
 871         netisr_domsg(&nmsg, 0);
 872 }
 873
 874 static void
 875 ifq_stage_detach_handler(netmsg_t nmsg)
 876 {
 877         struct ifaltq *ifq = nmsg->lmsg.u.ms_resultp;
 878         int q;
 879
 880         for (q = 0; q < ifq->altq_subq_cnt; ++q) {
 881                 struct ifaltq_subque *ifsq = &ifq->altq_subq[q];
 882                 struct ifsubq_stage *stage = ifsq_get_stage(ifsq, mycpuid);
 883
 884                 if (stage->stg_flags & IFSQ_STAGE_FLAG_QUED)
 885                         ifsq_stage_remove(&ifsubq_stage_heads[mycpuid], stage);
 886         }
 887         lwkt_replymsg(&nmsg->lmsg, 0);
 888 }
 889
 890 static void
 891 ifq_stage_detach(struct ifaltq *ifq)
 892 {
 893         struct netmsg_base base;
 894         int cpu;
 895
 896         netmsg_init(&base, NULL, &curthread->td_msgport, 0,
 897             ifq_stage_detach_handler);
 898         base.lmsg.u.ms_resultp = ifq;
 899
 900         /* XXX netisr_ncpus */
 901         for (cpu = 0; cpu < ncpus; ++cpu)
 902                 lwkt_domsg(netisr_cpuport(cpu), &base.lmsg, 0);
 903 }
 904
 905 struct netmsg_if_rtdel {
 906         struct netmsg_base      base;
 907         struct ifnet            *ifp;
 908 };
 909
 910 static void
 911 if_rtdel_dispatch(netmsg_t msg)
 912 {
 913         struct netmsg_if_rtdel *rmsg = (void *)msg;
 914         int i, cpu;
 915
 916         cpu = mycpuid;
 917         ASSERT_NETISR_NCPUS(cpu);
 918
 919         for (i = 1; i <= AF_MAX; i++) {
 920                 struct radix_node_head  *rnh;
 921
 922                 if ((rnh = rt_tables[cpu][i]) == NULL)
 923                         continue;
 924                 rnh->rnh_walktree(rnh, if_rtdel, rmsg->ifp);
 925         }
 926         netisr_forwardmsg(&msg->base, cpu + 1);
 927 }
 928
 929 /*
 930  * Detach an interface, removing it from the
 931  * list of "active" interfaces.
 932  */
 933 void
 934 if_detach(struct ifnet *ifp)
 935 {
 936         struct ifnet_array *old_ifnet_array;
 937         struct ifg_list *ifgl;
 938         struct netmsg_if_rtdel msg;
 939         struct domain *dp;
 940         int q;
 941
 942         /* Announce that the interface is gone. */
 943         EVENTHANDLER_INVOKE(ifnet_detach_event, ifp);
 944         rt_ifannouncemsg(ifp, IFAN_DEPARTURE);
 945         devctl_notify("IFNET", ifp->if_xname, "DETACH", NULL);
 946
 947         /*
 948          * Remove this ifp from ifindex2inet, ifnet queue and ifnet
 949          * array before it is whacked.
 950          *
 951          * Protect ifindex2ifnet, ifnet queue and ifnet array changes
 952          * by ifnet lock, so that non-netisr threads could get a
 953          * consistent view.
 954          */
 955         ifnet_lock();
 956
 957         /*
 958          * Remove this ifp from ifindex2ifnet and maybe decrement if_index.
 959          */
 960         ifindex2ifnet[ifp->if_index] = NULL;
 961         while (if_index > 0 && ifindex2ifnet[if_index] == NULL)
 962                 if_index--;
 963
 964         /*
 965          * Remove this ifp from ifnet queue.
 966          */
 967         TAILQ_REMOVE(&ifnetlist, ifp, if_link);
 968
 969         /*
 970          * Remove this ifp from ifnet array.
 971          */
 972         /* Free old ifnet array after sync all netisrs */
 973         old_ifnet_array = ifnet_array;
 974         ifnet_array = ifnet_array_del(ifp, old_ifnet_array);
 975
 976         ifnet_unlock();
 977
 978         ifgroup_lockmgr(LK_EXCLUSIVE);
 979         while ((ifgl = TAILQ_FIRST(&ifp->if_groups)) != NULL)
 980                 if_delgroup_locked(ifp, ifgl->ifgl_group->ifg_group);
 981         ifgroup_lockmgr(LK_RELEASE);
 982
 983         /*
 984          * Sync all netisrs so that the old ifnet array is no longer
 985          * accessed and we can free it safely later on.
 986          */
 987         netmsg_service_sync();
 988         ifnet_array_free(old_ifnet_array);
 989
 990         /*
 991          * Remove routes and flush queues.
 992          */
 993         crit_enter();
 994 #ifdef IFPOLL_ENABLE
 995         if (ifp->if_flags & IFF_NPOLLING)
 996                 ifpoll_deregister(ifp);
 997 #endif
 998         if_down(ifp);
 999
1000         /* Decrease the mbuf clusters/jclusters limits increased by us */
1001         if (ifp->if_nmbclusters > 0)
1002                 mcl_inclimit(-ifp->if_nmbclusters);
1003         if (ifp->if_nmbjclusters > 0)
1004                 mjcl_inclimit(-ifp->if_nmbjclusters);
1005
1006 #ifdef ALTQ
1007         if (ifq_is_enabled(&ifp->if_snd))
1008                 altq_disable(&ifp->if_snd);
1009         if (ifq_is_attached(&ifp->if_snd))
1010                 altq_detach(&ifp->if_snd);
1011 #endif
1012
1013         /*
1014          * Clean up all addresses.
1015          */
1016         ifp->if_lladdr = NULL;
1017
1018         if_purgeaddrs_nolink(ifp);
1019         if (!TAILQ_EMPTY(&ifp->if_addrheads[mycpuid])) {
1020                 struct ifaddr *ifa;
1021
1022                 ifa = TAILQ_FIRST(&ifp->if_addrheads[mycpuid])->ifa;
1023                 KASSERT(ifa->ifa_addr->sa_family == AF_LINK,
1024                         ("non-link ifaddr is left on if_addrheads"));
1025
1026                 ifa_ifunlink(ifa, ifp);
1027                 ifa_destroy(ifa);
1028                 KASSERT(TAILQ_EMPTY(&ifp->if_addrheads[mycpuid]),
1029                         ("there are still ifaddrs left on if_addrheads"));
1030         }
1031
1032 #ifdef INET
1033         /*
1034          * Remove all IPv4 kernel structures related to ifp.
1035          */
1036         in_ifdetach(ifp);
1037 #endif
1038
1039 #ifdef INET6
1040         /*
1041          * Remove all IPv6 kernel structs related to ifp.  This should be done
1042          * before removing routing entries below, since IPv6 interface direct
1043          * routes are expected to be removed by the IPv6-specific kernel API.
1044          * Otherwise, the kernel will detect some inconsistency and bark it.
1045          */
1046         in6_ifdetach(ifp);
1047 #endif
1048
1049         /*
1050          * Delete all remaining routes using this interface
1051          */
1052         netmsg_init(&msg.base, NULL, &curthread->td_msgport, MSGF_PRIORITY,
1053             if_rtdel_dispatch);
1054         msg.ifp = ifp;
1055         netisr_domsg_global(&msg.base);
1056
1057         SLIST_FOREACH(dp, &domains, dom_next) {
1058                 if (dp->dom_ifdetach && ifp->if_afdata[dp->dom_family])
1059                         (*dp->dom_ifdetach)(ifp,
1060                                 ifp->if_afdata[dp->dom_family]);
1061         }
1062
1063         kfree(ifp->if_addrheads, M_IFADDR);
1064
1065         lwkt_synchronize_ipiqs("if_detach");
1066         ifq_stage_detach(&ifp->if_snd);
1067
1068         for (q = 0; q < ifp->if_snd.altq_subq_cnt; ++q) {
1069                 struct ifaltq_subque *ifsq = &ifp->if_snd.altq_subq[q];
1070
1071                 kfree(ifsq->ifsq_ifstart_nmsg, M_LWKTMSG);
1072                 kfree(ifsq->ifsq_stage, M_DEVBUF);
1073         }
1074         kfree(ifp->if_snd.altq_subq, M_DEVBUF);
1075
1076         kfree(ifp->if_data_pcpu, M_DEVBUF);
1077
1078         crit_exit();
1079 }
1080
1081 int
1082 ifgroup_lockmgr(u_int flags)
1083 {
1084         return lockmgr(&ifgroup_lock, flags);
1085 }
1086
1087 /*
1088  * Create an empty interface group.
1089  */
1090 static struct ifg_group *
1091 if_creategroup(const char *groupname)
1092 {
1093         struct ifg_group *ifg;
1094
1095         ifg = kmalloc(sizeof(*ifg), M_IFNET, M_WAITOK);
1096         strlcpy(ifg->ifg_group, groupname, sizeof(ifg->ifg_group));
1097         ifg->ifg_refcnt = 0;
1098         ifg->ifg_carp_demoted = 0;
1099         TAILQ_INIT(&ifg->ifg_members);
1100
1101         ifgroup_lockmgr(LK_EXCLUSIVE);
1102         TAILQ_INSERT_TAIL(&ifg_head, ifg, ifg_next);
1103         ifgroup_lockmgr(LK_RELEASE);
1104
1105         EVENTHANDLER_INVOKE(group_attach_event, ifg);
1106
1107         return (ifg);
1108 }
1109
1110 /*
1111  * Destroy an empty interface group.
1112  */
1113 static int
1114 if_destroygroup(struct ifg_group *ifg)
1115 {
1116         KASSERT(ifg->ifg_refcnt == 0,
1117                 ("trying to delete a non-empty interface group"));
1118
1119         ifgroup_lockmgr(LK_EXCLUSIVE);
1120         TAILQ_REMOVE(&ifg_head, ifg, ifg_next);
1121         ifgroup_lockmgr(LK_RELEASE);
1122
1123         EVENTHANDLER_INVOKE(group_detach_event, ifg);
1124         kfree(ifg, M_IFNET);
1125
1126         return (0);
1127 }
1128
1129 /*
1130  * Add the interface to a group.
1131  * The target group will be created if it doesn't exist.
1132  */
1133 int
1134 if_addgroup(struct ifnet *ifp, const char *groupname)
1135 {
1136         struct ifg_list *ifgl;
1137         struct ifg_group *ifg;
1138         struct ifg_member *ifgm;
1139
1140         if (groupname[0] &&
1141             groupname[strlen(groupname) - 1] >= '0' &&
1142             groupname[strlen(groupname) - 1] <= '9')
1143                 return (EINVAL);
1144
1145         ifgroup_lockmgr(LK_SHARED);
1146
1147         TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) {
1148                 if (strcmp(ifgl->ifgl_group->ifg_group, groupname) == 0) {
1149                         ifgroup_lockmgr(LK_RELEASE);
1150                         return (EEXIST);
1151                 }
1152         }
1153
1154         TAILQ_FOREACH(ifg, &ifg_head, ifg_next) {
1155                 if (strcmp(ifg->ifg_group, groupname) == 0)
1156                         break;
1157         }
1158
1159         ifgroup_lockmgr(LK_RELEASE);
1160
1161         if (ifg == NULL)
1162                 ifg = if_creategroup(groupname);
1163
1164         ifgl = kmalloc(sizeof(*ifgl), M_IFNET, M_WAITOK);
1165         ifgm = kmalloc(sizeof(*ifgm), M_IFNET, M_WAITOK);
1166         ifgl->ifgl_group = ifg;
1167         ifgm->ifgm_ifp = ifp;
1168         ifg->ifg_refcnt++;
1169
1170         ifgroup_lockmgr(LK_EXCLUSIVE);
1171         TAILQ_INSERT_TAIL(&ifg->ifg_members, ifgm, ifgm_next);
1172         TAILQ_INSERT_TAIL(&ifp->if_groups, ifgl, ifgl_next);
1173         ifgroup_lockmgr(LK_RELEASE);
1174
1175         EVENTHANDLER_INVOKE(group_change_event, groupname);
1176
1177         return (0);
1178 }
1179
1180 /*
1181  * Remove the interface from a group.
1182  * The group will be destroyed if it becomes empty.
1183  *
1184  * The 'ifgroup_lock' must be hold exclusively when calling this.
1185  */
1186 static int
1187 if_delgroup_locked(struct ifnet *ifp, const char *groupname)
1188 {
1189         struct ifg_list *ifgl;
1190         struct ifg_member *ifgm;
1191
1192         KKASSERT(lockstatus(&ifgroup_lock, curthread) == LK_EXCLUSIVE);
1193
1194         TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) {
1195                 if (strcmp(ifgl->ifgl_group->ifg_group, groupname) == 0)
1196                         break;
1197         }
1198         if (ifgl == NULL)
1199                 return (ENOENT);
1200
1201         TAILQ_REMOVE(&ifp->if_groups, ifgl, ifgl_next);
1202
1203         TAILQ_FOREACH(ifgm, &ifgl->ifgl_group->ifg_members, ifgm_next) {
1204                 if (ifgm->ifgm_ifp == ifp)
1205                         break;
1206         }
1207
1208         if (ifgm != NULL) {
1209                 TAILQ_REMOVE(&ifgl->ifgl_group->ifg_members, ifgm, ifgm_next);
1210
1211                 ifgroup_lockmgr(LK_RELEASE);
1212                 EVENTHANDLER_INVOKE(group_change_event, groupname);
1213                 ifgroup_lockmgr(LK_EXCLUSIVE);
1214
1215                 kfree(ifgm, M_IFNET);
1216                 ifgl->ifgl_group->ifg_refcnt--;
1217         }
1218
1219         if (ifgl->ifgl_group->ifg_refcnt == 0) {
1220                 ifgroup_lockmgr(LK_RELEASE);
1221                 if_destroygroup(ifgl->ifgl_group);
1222                 ifgroup_lockmgr(LK_EXCLUSIVE);
1223         }
1224
1225         kfree(ifgl, M_IFNET);
1226
1227         return (0);
1228 }
1229
1230 int
1231 if_delgroup(struct ifnet *ifp, const char *groupname)
1232 {
1233         int error;
1234
1235         ifgroup_lockmgr(LK_EXCLUSIVE);
1236         error = if_delgroup_locked(ifp, groupname);
1237         ifgroup_lockmgr(LK_RELEASE);
1238
1239         return (error);
1240 }
1241
1242 /*
1243  * Store all the groups that the interface belongs to in memory
1244  * pointed to by data.
1245  */
1246 static int
1247 if_getgroups(struct ifgroupreq *ifgr, struct ifnet *ifp)
1248 {
1249         struct ifg_list *ifgl;
1250         struct ifg_req *ifgrq, *p;
1251         int len, error;
1252
1253         len = 0;
1254         ifgroup_lockmgr(LK_SHARED);
1255         TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next)
1256                 len += sizeof(struct ifg_req);
1257         ifgroup_lockmgr(LK_RELEASE);
1258
1259         if (ifgr->ifgr_len == 0) {
1260                 /*
1261                  * Caller is asking how much memory should be allocated in
1262                  * the next request in order to hold all the groups.
1263                  */
1264                 ifgr->ifgr_len = len;
1265                 return (0);
1266         } else if (ifgr->ifgr_len != len) {
1267                 return (EINVAL);
1268         }
1269
1270         ifgrq = kmalloc(len, M_TEMP, M_INTWAIT | M_NULLOK | M_ZERO);
1271         if (ifgrq == NULL)
1272                 return (ENOMEM);
1273
1274         ifgroup_lockmgr(LK_SHARED);
1275         p = ifgrq;
1276         TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) {
1277                 if (len < sizeof(struct ifg_req)) {
1278                         ifgroup_lockmgr(LK_RELEASE);
1279                         error = EINVAL;
1280                         goto failed;
1281                 }
1282
1283                 strlcpy(p->ifgrq_group, ifgl->ifgl_group->ifg_group,
1284                         sizeof(ifgrq->ifgrq_group));
1285                 len -= sizeof(struct ifg_req);
1286                 p++;
1287         }
1288         ifgroup_lockmgr(LK_RELEASE);
1289
1290         error = copyout(ifgrq, ifgr->ifgr_groups, ifgr->ifgr_len);
1291 failed:
1292         kfree(ifgrq, M_TEMP);
1293         return error;
1294 }
1295
1296 /*
1297  * Store all the members of a group in memory pointed to by data.
1298  */
1299 static int
1300 if_getgroupmembers(struct ifgroupreq *ifgr)
1301 {
1302         struct ifg_group *ifg;
1303         struct ifg_member *ifgm;
1304         struct ifg_req *ifgrq, *p;
1305         int len, error;
1306
1307         ifgroup_lockmgr(LK_SHARED);
1308
1309         TAILQ_FOREACH(ifg, &ifg_head, ifg_next) {
1310                 if (strcmp(ifg->ifg_group, ifgr->ifgr_name) == 0)
1311                         break;
1312         }
1313         if (ifg == NULL) {
1314                 ifgroup_lockmgr(LK_RELEASE);
1315                 return (ENOENT);
1316         }
1317
1318         len = 0;
1319         TAILQ_FOREACH(ifgm, &ifg->ifg_members, ifgm_next)
1320                 len += sizeof(struct ifg_req);
1321
1322         ifgroup_lockmgr(LK_RELEASE);
1323
1324         if (ifgr->ifgr_len == 0) {
1325                 ifgr->ifgr_len = len;
1326                 return (0);
1327         } else if (ifgr->ifgr_len != len) {
1328                 return (EINVAL);
1329         }
1330
1331         ifgrq = kmalloc(len, M_TEMP, M_INTWAIT | M_NULLOK | M_ZERO);
1332         if (ifgrq == NULL)
1333                 return (ENOMEM);
1334
1335         ifgroup_lockmgr(LK_SHARED);
1336         p = ifgrq;
1337         TAILQ_FOREACH(ifgm, &ifg->ifg_members, ifgm_next) {
1338                 if (len < sizeof(struct ifg_req)) {
1339                         ifgroup_lockmgr(LK_RELEASE);
1340                         error = EINVAL;
1341                         goto failed;
1342                 }
1343
1344                 strlcpy(p->ifgrq_member, ifgm->ifgm_ifp->if_xname,
1345                         sizeof(p->ifgrq_member));
1346                 len -= sizeof(struct ifg_req);
1347                 p++;
1348         }
1349         ifgroup_lockmgr(LK_RELEASE);
1350
1351         error = copyout(ifgrq, ifgr->ifgr_groups, ifgr->ifgr_len);
1352 failed:
1353         kfree(ifgrq, M_TEMP);
1354         return error;
1355 }
1356
1357 static int
1358 ifa_maintain_loopback_route(int cmd, struct ifaddr *ifa, struct sockaddr *ia)
1359 {
1360         struct sockaddr_dl null_sdl;
1361         struct rt_addrinfo info;
1362         struct ifaddr *rti_ifa;
1363         struct ifnet *ifp;
1364         int error;
1365
1366         /* RTM_CHANGE is unsupported in rtrequest1() yet. */
1367         KKASSERT(cmd == RTM_DELETE || cmd == RTM_ADD);
1368
1369         rti_ifa = NULL;
1370         ifp = ifa->ifa_ifp;
1371
1372         bzero(&null_sdl, sizeof(null_sdl));
1373         null_sdl.sdl_len = sizeof(null_sdl);
1374         null_sdl.sdl_family = AF_LINK;
1375         null_sdl.sdl_index = ifp->if_index;
1376         null_sdl.sdl_type = ifp->if_type;
1377
1378         bzero(&info, sizeof(info));
1379         if (cmd != RTM_DELETE)
1380                 info.rti_ifp = loif;
1381         if (cmd == RTM_ADD) {
1382                 /*
1383                  * Explicitly specify the loopback IFA.
1384                  */
1385                 rti_ifa = ifaof_ifpforaddr(ifa->ifa_addr, info.rti_ifp);
1386                 if (rti_ifa != NULL) {
1387                         /*
1388                          * The loopback IFA wouldn't disappear, but ref it
1389                          * for safety.
1390                          */
1391                         IFAREF(rti_ifa);
1392                         info.rti_ifa = rti_ifa;
1393                 }
1394         }
1395         info.rti_info[RTAX_DST] = ia;
1396         info.rti_info[RTAX_GATEWAY] = (struct sockaddr *)&null_sdl;
1397         /*
1398          * Manually set RTF_LOCAL so that the IFA and IFP wouldn't be
1399          * overrided to be the owner of the destination address (ia)
1400          * by in_addroute().
1401          */
1402         info.rti_flags = ifa->ifa_flags | RTF_HOST | RTF_LOCAL;
1403
1404         error = rtrequest1_global(cmd, &info, NULL, NULL, RTREQ_PRIO_NORM);
1405
1406         if (rti_ifa != NULL)
1407                 IFAFREE(rti_ifa);
1408
1409         if (error == 0 ||
1410             (cmd == RTM_ADD && error == EEXIST) ||
1411             (cmd == RTM_DELETE && (error == ESRCH || error == ENOENT)))
1412                 return (error);
1413
1414         log(LOG_DEBUG, "%s: %s failed for interface %s: %d\n",
1415             __func__, (cmd == RTM_ADD ? "insertion" : "deletion"),
1416             ifp->if_xname, error);
1417         return (error);
1418 }
1419
1420 int
1421 ifa_add_loopback_route(struct ifaddr *ifa, struct sockaddr *ia)
1422 {
1423         return ifa_maintain_loopback_route(RTM_ADD, ifa, ia);
1424 }
1425
1426 int
1427 ifa_del_loopback_route(struct ifaddr *ifa, struct sockaddr *ia)
1428 {
1429         return ifa_maintain_loopback_route(RTM_DELETE, ifa, ia);
1430 }
1431
1432 /*
1433  * Delete Routes for a Network Interface
1434  *
1435  * Called for each routing entry via the rnh->rnh_walktree() call above
1436  * to delete all route entries referencing a detaching network interface.
1437  *
1438  * Arguments:
1439  *      rn      pointer to node in the routing table
1440  *      arg     argument passed to rnh->rnh_walktree() - detaching interface
1441  *
1442  * Returns:
1443  *      0       successful
1444  *      errno   failed - reason indicated
1445  *
1446  */
1447 static int
1448 if_rtdel(struct radix_node *rn, void *arg)
1449 {
1450         struct rtentry  *rt = (struct rtentry *)rn;
1451         struct ifnet    *ifp = arg;
1452         int             err;
1453
1454         if (rt->rt_ifp == ifp) {
1455
1456                 /*
1457                  * Protect (sorta) against walktree recursion problems
1458                  * with cloned routes
1459                  */
1460                 if (!(rt->rt_flags & RTF_UP))
1461                         return (0);
1462
1463                 err = rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway,
1464                                 rt_mask(rt), rt->rt_flags,
1465                                 NULL);
1466                 if (err) {
1467                         log(LOG_WARNING, "if_rtdel: error %d\n", err);
1468                 }
1469         }
1470
1471         return (0);
1472 }
1473
1474 static __inline boolean_t
1475 ifa_match_withmask(const struct ifaddr *ifa, const struct sockaddr *addr)
1476 {
1477         const char *cp, *cp2, *cp3, *cplim;
1478
1479         KKASSERT(ifa->ifa_addr->sa_family == addr->sa_family);
1480
1481         cp = addr->sa_data;
1482         cp2 = ifa->ifa_addr->sa_data;
1483         cp3 = ifa->ifa_netmask->sa_data;
1484         cplim = (const char *)ifa->ifa_netmask + ifa->ifa_netmask->sa_len;
1485
1486         while (cp3 < cplim) {
1487                 if ((*cp++ ^ *cp2++) & *cp3++)
1488                         return (FALSE);
1489         }
1490
1491         return (TRUE);
1492 }
1493
1494 static __inline boolean_t
1495 ifa_prefer(const struct ifaddr *cur_ifa, const struct ifaddr *old_ifa)
1496 {
1497         if (old_ifa == NULL)
1498                 return (TRUE);
1499
1500         if ((old_ifa->ifa_ifp->if_flags & IFF_UP) == 0 &&
1501             (cur_ifa->ifa_ifp->if_flags & IFF_UP))
1502                 return (TRUE);
1503         if ((old_ifa->ifa_flags & IFA_ROUTE) == 0 &&
1504             (cur_ifa->ifa_flags & IFA_ROUTE))
1505                 return (TRUE);
1506
1507         return (FALSE);
1508 }
1509
1510 /*
1511  * Locate an interface based on a complete address.
1512  */
1513 struct ifaddr *
1514 ifa_ifwithaddr(struct sockaddr *addr)
1515 {
1516         const struct ifnet_array *arr;
1517         int i;
1518
1519         arr = ifnet_array_get();
1520         for (i = 0; i < arr->ifnet_count; ++i) {
1521                 struct ifnet *ifp = arr->ifnet_arr[i];
1522                 struct ifaddr_container *ifac;
1523
1524                 TAILQ_FOREACH(ifac, &ifp->if_addrheads[mycpuid], ifa_link) {
1525                         struct ifaddr *ifa = ifac->ifa;
1526
1527                         if (ifa->ifa_addr->sa_family != addr->sa_family)
1528                                 continue;
1529                         if (sa_equal(addr, ifa->ifa_addr))
1530                                 return (ifa);
1531                         if ((ifp->if_flags & IFF_BROADCAST) &&
1532                             ifa->ifa_broadaddr &&
1533                             /* IPv6 doesn't have broadcast */
1534                             ifa->ifa_broadaddr->sa_len != 0 &&
1535                             sa_equal(ifa->ifa_broadaddr, addr))
1536                                 return (ifa);
1537                 }
1538         }
1539         return (NULL);
1540 }
1541
1542 /*
1543  * Locate the point-to-point interface with a given destination address.
1544  */
1545 struct ifaddr *
1546 ifa_ifwithdstaddr(struct sockaddr *addr)
1547 {
1548         const struct ifnet_array *arr;
1549         int i;
1550
1551         arr = ifnet_array_get();
1552         for (i = 0; i < arr->ifnet_count; ++i) {
1553                 struct ifnet *ifp = arr->ifnet_arr[i];
1554                 struct ifaddr_container *ifac;
1555
1556                 if (!(ifp->if_flags & IFF_POINTOPOINT))
1557                         continue;
1558
1559                 TAILQ_FOREACH(ifac, &ifp->if_addrheads[mycpuid], ifa_link) {
1560                         struct ifaddr *ifa = ifac->ifa;
1561
1562                         if (ifa->ifa_addr->sa_family != addr->sa_family)
1563                                 continue;
1564                         if (ifa->ifa_dstaddr &&
1565                             sa_equal(addr, ifa->ifa_dstaddr))
1566                                 return (ifa);
1567                 }
1568         }
1569         return (NULL);
1570 }
1571
1572 /*
1573  * Find an interface on a specific network.  If many, choice
1574  * is most specific found.
1575  */
1576 struct ifaddr *
1577 ifa_ifwithnet(struct sockaddr *addr)
1578 {
1579         struct ifaddr *ifa_maybe = NULL;
1580         u_int af = addr->sa_family;
1581         const struct ifnet_array *arr;
1582         int i;
1583
1584         /*
1585          * AF_LINK addresses can be looked up directly by their index number,
1586          * so do that if we can.
1587          */
1588         if (af == AF_LINK) {
1589                 struct sockaddr_dl *sdl = (struct sockaddr_dl *)addr;
1590
1591                 if (sdl->sdl_index && sdl->sdl_index <= if_index)
1592                         return (ifindex2ifnet[sdl->sdl_index]->if_lladdr);
1593         }
1594
1595         /*
1596          * Scan though each interface, looking for ones that have
1597          * addresses in this address family.
1598          */
1599         arr = ifnet_array_get();
1600         for (i = 0; i < arr->ifnet_count; ++i) {
1601                 struct ifnet *ifp = arr->ifnet_arr[i];
1602                 struct ifaddr_container *ifac;
1603
1604                 TAILQ_FOREACH(ifac, &ifp->if_addrheads[mycpuid], ifa_link) {
1605                         struct ifaddr *ifa = ifac->ifa;
1606
1607                         if (ifa->ifa_addr->sa_family != af)
1608                                 continue;
1609                         if (af == AF_INET && ifp->if_flags & IFF_POINTOPOINT) {
1610                                 /*
1611                                  * This is a bit broken as it doesn't
1612                                  * take into account that the remote end may
1613                                  * be a single node in the network we are
1614                                  * looking for.
1615                                  * The trouble is that we don't know the
1616                                  * netmask for the remote end.
1617                                  */
1618                                 if (ifa->ifa_dstaddr != NULL &&
1619                                     sa_equal(addr, ifa->ifa_dstaddr))
1620                                         return (ifa);
1621                         } else {
1622                                 /*
1623                                  * If we have a special address handler,
1624                                  * then use it instead of the generic one.
1625                                  */
1626                                 if (ifa->ifa_claim_addr) {
1627                                         if ((*ifa->ifa_claim_addr)(ifa, addr)) {
1628                                                 return (ifa);
1629                                         } else {
1630                                                 continue;
1631                                         }
1632                                 }
1633
1634                                 if (ifa->ifa_netmask == NULL ||
1635                                     !ifa_match_withmask(ifa, addr))
1636                                         continue;
1637
1638                                 /*
1639                                  * If the netmask of what we just found
1640                                  * is more specific than what we had before
1641                                  * (if we had one) then remember the new one
1642                                  * before continuing to search for an even
1643                                  * better one.  If the netmasks are equal,
1644                                  * we prefer the this ifa based on the result
1645                                  * of ifa_prefer().
1646                                  */
1647                                 if (ifa_maybe == NULL ||
1648                                     rn_refines(ifa->ifa_netmask,
1649                                                ifa_maybe->ifa_netmask) ||
1650                                     (sa_equal(ifa_maybe->ifa_netmask,
1651                                               ifa->ifa_netmask) &&
1652                                      ifa_prefer(ifa, ifa_maybe)))
1653                                         ifa_maybe = ifa;
1654                         }
1655                 }
1656         }
1657
1658         return (ifa_maybe);
1659 }
1660
1661 /*
1662  * Find an interface address specific to an interface best matching
1663  * a given address.
1664  */
1665 struct ifaddr *
1666 ifaof_ifpforaddr(struct sockaddr *addr, struct ifnet *ifp)
1667 {
1668         struct ifaddr_container *ifac;
1669         struct ifaddr *ifa_maybe = NULL;
1670         u_int af = addr->sa_family;
1671
1672         if (af >= AF_MAX)
1673                 return (NULL);
1674
1675         TAILQ_FOREACH(ifac, &ifp->if_addrheads[mycpuid], ifa_link) {
1676                 struct ifaddr *ifa = ifac->ifa;
1677
1678                 if (ifa->ifa_addr->sa_family != af)
1679                         continue;
1680                 if (ifa_maybe == NULL)
1681                         ifa_maybe = ifa;
1682                 if (ifa->ifa_netmask == NULL) {
1683                         if (sa_equal(addr, ifa->ifa_addr) ||
1684                             (ifa->ifa_dstaddr != NULL &&
1685                              sa_equal(addr, ifa->ifa_dstaddr)))
1686                                 return (ifa);
1687                         continue;
1688                 }
1689                 if (ifp->if_flags & IFF_POINTOPOINT) {
1690                         if (sa_equal(addr, ifa->ifa_dstaddr))
1691                                 return (ifa);
1692                 } else {
1693                         if (ifa_match_withmask(ifa, addr))
1694                                 return (ifa);
1695                 }
1696         }
1697
1698         return (ifa_maybe);
1699 }
1700
1701 struct netmsg_if {
1702         struct netmsg_base      base;
1703         struct ifnet            *ifp;
1704 };
1705
1706 /*
1707  * Mark an interface down and notify protocols of the transition.
1708  */
1709 static void
1710 if_down_dispatch(netmsg_t nmsg)
1711 {
1712         struct netmsg_if *msg = (struct netmsg_if *)nmsg;
1713         struct ifnet *ifp = msg->ifp;
1714         struct ifaddr_container *ifac;
1715         struct domain *dp;
1716
1717         ASSERT_NETISR0;
1718
1719         ifp->if_flags &= ~IFF_UP;
1720         getmicrotime(&ifp->if_lastchange);
1721         rt_ifmsg(ifp);
1722
1723         /*
1724          * The ifaddr processing in the following loop will block,
1725          * however, this function is called in netisr0, in which
1726          * ifaddr list changes happen, so we don't care about the
1727          * blockness of the ifaddr processing here.
1728          */
1729         TAILQ_FOREACH(ifac, &ifp->if_addrheads[mycpuid], ifa_link) {
1730                 struct ifaddr *ifa = ifac->ifa;
1731
1732                 /* Ignore marker */
1733                 if (ifa->ifa_addr->sa_family == AF_UNSPEC)
1734                         continue;
1735
1736                 kpfctlinput(PRC_IFDOWN, ifa->ifa_addr);
1737         }
1738
1739         SLIST_FOREACH(dp, &domains, dom_next)
1740                 if (dp->dom_if_down != NULL)
1741                         dp->dom_if_down(ifp);
1742
1743         ifq_purge_all(&ifp->if_snd);
1744         netisr_replymsg(&nmsg->base, 0);
1745 }
1746
1747 /*
1748  * Mark an interface up and notify protocols of the transition.
1749  */
1750 static void
1751 if_up_dispatch(netmsg_t nmsg)
1752 {
1753         struct netmsg_if *msg = (struct netmsg_if *)nmsg;
1754         struct ifnet *ifp = msg->ifp;
1755         struct ifaddr_container *ifac;
1756         struct domain *dp;
1757
1758         ASSERT_NETISR0;
1759
1760         ifq_purge_all(&ifp->if_snd);
1761         ifp->if_flags |= IFF_UP;
1762         getmicrotime(&ifp->if_lastchange);
1763         rt_ifmsg(ifp);
1764
1765         /*
1766          * The ifaddr processing in the following loop will block,
1767          * however, this function is called in netisr0, in which
1768          * ifaddr list changes happen, so we don't care about the
1769          * blockness of the ifaddr processing here.
1770          */
1771         TAILQ_FOREACH(ifac, &ifp->if_addrheads[mycpuid], ifa_link) {
1772                 struct ifaddr *ifa = ifac->ifa;
1773
1774                 /* Ignore marker */
1775                 if (ifa->ifa_addr->sa_family == AF_UNSPEC)
1776                         continue;
1777
1778                 kpfctlinput(PRC_IFUP, ifa->ifa_addr);
1779         }
1780
1781         SLIST_FOREACH(dp, &domains, dom_next)
1782                 if (dp->dom_if_up != NULL)
1783                         dp->dom_if_up(ifp);
1784
1785         netisr_replymsg(&nmsg->base, 0);
1786 }
1787
1788 /*
1789  * Mark an interface down and notify protocols of the transition.  An
1790  * interface going down is also considered to be a synchronizing event.
1791  * We must ensure that all packet processing related to the interface
1792  * has completed before we return so e.g. the caller can free the ifnet
1793  * structure that the mbufs may be referencing.
1794  *
1795  * NOTE: must be called at splnet or eqivalent.
1796  */
1797 void
1798 if_down(struct ifnet *ifp)
1799 {
1800         struct netmsg_if msg;
1801
1802         EVENTHANDLER_INVOKE(ifnet_event, ifp, IFNET_EVENT_DOWN);
1803         netmsg_init(&msg.base, NULL, &curthread->td_msgport, 0,
1804             if_down_dispatch);
1805         msg.ifp = ifp;
1806         netisr_domsg(&msg.base, 0);
1807         netmsg_service_sync();
1808 }
1809
1810 /*
1811  * Mark an interface up and notify protocols of
1812  * the transition.
1813  * NOTE: must be called at splnet or eqivalent.
1814  */
1815 void
1816 if_up(struct ifnet *ifp)
1817 {
1818         struct netmsg_if msg;
1819
1820         netmsg_init(&msg.base, NULL, &curthread->td_msgport, 0,
1821             if_up_dispatch);
1822         msg.ifp = ifp;
1823         netisr_domsg(&msg.base, 0);
1824         EVENTHANDLER_INVOKE(ifnet_event, ifp, IFNET_EVENT_UP);
1825 }
1826
1827 /*
1828  * Process a link state change.
1829  * NOTE: must be called at splsoftnet or equivalent.
1830  */
1831 void
1832 if_link_state_change(struct ifnet *ifp)
1833 {
1834         int link_state = ifp->if_link_state;
1835
1836         rt_ifmsg(ifp);
1837         devctl_notify("IFNET", ifp->if_xname,
1838             (link_state == LINK_STATE_UP) ? "LINK_UP" : "LINK_DOWN", NULL);
1839
1840         EVENTHANDLER_INVOKE(ifnet_link_event, ifp, link_state);
1841 }
1842
1843 /*
1844  * Handle interface watchdog timer routines.  Called
1845  * from softclock, we decrement timers (if set) and
1846  * call the appropriate interface routine on expiration.
1847  */
1848 static void
1849 if_slowtimo_dispatch(netmsg_t nmsg)
1850 {
1851         struct globaldata *gd = mycpu;
1852         const struct ifnet_array *arr;
1853         int i;
1854
1855         ASSERT_NETISR0;
1856
1857         crit_enter_gd(gd);
1858         lwkt_replymsg(&nmsg->lmsg, 0);  /* reply ASAP */
1859         crit_exit_gd(gd);
1860
1861         arr = ifnet_array_get();
1862         for (i = 0; i < arr->ifnet_count; ++i) {
1863                 struct ifnet *ifp = arr->ifnet_arr[i];
1864
1865                 crit_enter_gd(gd);
1866
1867                 if (if_stats_compat) {
1868                         IFNET_STAT_GET(ifp, ipackets, ifp->if_ipackets);
1869                         IFNET_STAT_GET(ifp, ierrors, ifp->if_ierrors);
1870                         IFNET_STAT_GET(ifp, opackets, ifp->if_opackets);
1871                         IFNET_STAT_GET(ifp, oerrors, ifp->if_oerrors);
1872                         IFNET_STAT_GET(ifp, collisions, ifp->if_collisions);
1873                         IFNET_STAT_GET(ifp, ibytes, ifp->if_ibytes);
1874                         IFNET_STAT_GET(ifp, obytes, ifp->if_obytes);
1875                         IFNET_STAT_GET(ifp, imcasts, ifp->if_imcasts);
1876                         IFNET_STAT_GET(ifp, omcasts, ifp->if_omcasts);
1877                         IFNET_STAT_GET(ifp, iqdrops, ifp->if_iqdrops);
1878                         IFNET_STAT_GET(ifp, noproto, ifp->if_noproto);
1879                         IFNET_STAT_GET(ifp, oqdrops, ifp->if_oqdrops);
1880                 }
1881
1882                 if (ifp->if_timer == 0 || --ifp->if_timer) {
1883                         crit_exit_gd(gd);
1884                         continue;
1885                 }
1886                 if (ifp->if_watchdog) {
1887                         if (ifnet_tryserialize_all(ifp)) {
1888                                 (*ifp->if_watchdog)(ifp);
1889                                 ifnet_deserialize_all(ifp);
1890                         } else {
1891                                 /* try again next timeout */
1892                                 ++ifp->if_timer;
1893                         }
1894                 }
1895
1896                 crit_exit_gd(gd);
1897         }
1898
1899         callout_reset(&if_slowtimo_timer, hz / IFNET_SLOWHZ, if_slowtimo, NULL);
1900 }
1901
1902 static void
1903 if_slowtimo(void *arg __unused)
1904 {
1905         struct lwkt_msg *lmsg = &if_slowtimo_netmsg.lmsg;
1906
1907         KASSERT(mycpuid == 0, ("not on cpu0"));
1908         crit_enter();
1909         if (lmsg->ms_flags & MSGF_DONE)
1910                 lwkt_sendmsg_oncpu(netisr_cpuport(0), lmsg);
1911         crit_exit();
1912 }
1913
1914 /*
1915  * Map interface name to
1916  * interface structure pointer.
1917  */
1918 struct ifnet *
1919 ifunit(const char *name)
1920 {
1921         struct ifnet *ifp;
1922
1923         /*
1924          * Search all the interfaces for this name/number
1925          */
1926         KASSERT(mtx_owned(&ifnet_mtx), ("ifnet is not locked"));
1927
1928         TAILQ_FOREACH(ifp, &ifnetlist, if_link) {
1929                 if (strncmp(ifp->if_xname, name, IFNAMSIZ) == 0)
1930                         break;
1931         }
1932         return (ifp);
1933 }
1934
1935 struct ifnet *
1936 ifunit_netisr(const char *name)
1937 {
1938         const struct ifnet_array *arr;
1939         int i;
1940
1941         /*
1942          * Search all the interfaces for this name/number
1943          */
1944
1945         arr = ifnet_array_get();
1946         for (i = 0; i < arr->ifnet_count; ++i) {
1947                 struct ifnet *ifp = arr->ifnet_arr[i];
1948
1949                 if (strncmp(ifp->if_xname, name, IFNAMSIZ) == 0)
1950                         return ifp;
1951         }
1952         return NULL;
1953 }
1954
1955 /*
1956  * Interface ioctls.
1957  */
1958 int
1959 ifioctl(struct socket *so, u_long cmd, caddr_t data, struct ucred *cred)
1960 {
1961         struct ifnet *ifp;
1962         struct ifgroupreq *ifgr;
1963         struct ifreq *ifr;
1964         struct ifstat *ifs;
1965         int error, do_ifup = 0;
1966         short oif_flags;
1967         int new_flags;
1968         size_t namelen, onamelen;
1969         size_t descrlen;
1970         char *descrbuf, *odescrbuf;
1971         char new_name[IFNAMSIZ];
1972         struct ifaddr *ifa;
1973         struct sockaddr_dl *sdl;
1974
1975         switch (cmd) {
1976         case SIOCGIFCONF:
1977                 return (ifconf(cmd, data, cred));
1978         default:
1979                 break;
1980         }
1981
1982         ifr = (struct ifreq *)data;
1983
1984         switch (cmd) {
1985         case SIOCIFCREATE:
1986         case SIOCIFCREATE2:
1987                 error = caps_priv_check(cred, SYSCAP_RESTRICTEDROOT);
1988                 if (error)
1989                         return (error);
1990                 return (if_clone_create(ifr->ifr_name, sizeof(ifr->ifr_name),
1991                         (cmd == SIOCIFCREATE2 ? ifr->ifr_data : NULL), NULL));
1992         case SIOCIFDESTROY:
1993                 error = caps_priv_check(cred, SYSCAP_RESTRICTEDROOT);
1994                 if (error)
1995                         return (error);
1996                 return (if_clone_destroy(ifr->ifr_name));
1997         case SIOCIFGCLONERS:
1998                 return (if_clone_list((struct if_clonereq *)data));
1999         case SIOCGIFGMEMB:
2000                 return (if_getgroupmembers((struct ifgroupreq *)data));
2001         default:
2002                 break;
2003         }
2004
2005         /*
2006          * Nominal ioctl through interface, lookup the ifp and obtain a
2007          * lock to serialize the ifconfig ioctl operation.
2008          */
2009         ifnet_lock();
2010
2011         ifp = ifunit(ifr->ifr_name);
2012         if (ifp == NULL) {
2013                 ifnet_unlock();
2014                 return (ENXIO);
2015         }
2016         error = 0;
2017
2018         switch (cmd) {
2019         case SIOCGIFINDEX:
2020                 ifr->ifr_index = ifp->if_index;
2021                 break;
2022
2023         case SIOCGIFFLAGS:
2024                 ifr->ifr_flags = ifp->if_flags;
2025                 ifr->ifr_flagshigh = ifp->if_flags >> 16;
2026                 break;
2027
2028         case SIOCGIFCAP:
2029                 ifr->ifr_reqcap = ifp->if_capabilities;
2030                 ifr->ifr_curcap = ifp->if_capenable;
2031                 break;
2032
2033         case SIOCGIFMETRIC:
2034                 ifr->ifr_metric = ifp->if_metric;
2035                 break;
2036
2037         case SIOCGIFMTU:
2038                 ifr->ifr_mtu = ifp->if_mtu;
2039                 break;
2040
2041         case SIOCGIFTSOLEN:
2042                 ifr->ifr_tsolen = ifp->if_tsolen;
2043                 break;
2044
2045         case SIOCGIFDATA:
2046                 error = copyout((caddr_t)&ifp->if_data, ifr->ifr_data,
2047                                 sizeof(ifp->if_data));
2048                 break;
2049
2050         case SIOCGIFPHYS:
2051                 ifr->ifr_phys = ifp->if_physical;
2052                 break;
2053
2054         case SIOCGIFPOLLCPU:
2055                 ifr->ifr_pollcpu = -1;
2056                 break;
2057
2058         case SIOCSIFPOLLCPU:
2059                 break;
2060
2061         case SIOCGIFDESCR:
2062                 error = 0;
2063                 ifnet_lock();
2064                 if (ifp->if_description == NULL) {
2065                         ifr->ifr_buffer.length = 0;
2066                         error = ENOMSG;
2067                 } else {
2068                         /* space for terminating nul */
2069                         descrlen = strlen(ifp->if_description) + 1;
2070                         if (ifr->ifr_buffer.length < descrlen)
2071                                 error = ENAMETOOLONG;
2072                         else
2073                                 error = copyout(ifp->if_description,
2074                                     ifr->ifr_buffer.buffer, descrlen);
2075                         ifr->ifr_buffer.length = descrlen;
2076                 }
2077                 ifnet_unlock();
2078                 break;
2079
2080         case SIOCSIFDESCR:
2081                 error = caps_priv_check(cred, SYSCAP_RESTRICTEDROOT);
2082                 if (error)
2083                         break;
2084
2085                 /*
2086                  * Copy only (length-1) bytes to make sure that
2087                  * if_description is always nul terminated.  The
2088                  * length parameter is supposed to count the
2089                  * terminating nul in.
2090                  */
2091                 if (ifr->ifr_buffer.length > ifdescr_maxlen)
2092                         return (ENAMETOOLONG);
2093                 else if (ifr->ifr_buffer.length == 0)
2094                         descrbuf = NULL;
2095                 else {
2096                         descrbuf = kmalloc(ifr->ifr_buffer.length, M_IFDESCR,
2097                             M_WAITOK | M_ZERO);
2098                         error = copyin(ifr->ifr_buffer.buffer, descrbuf,
2099                             ifr->ifr_buffer.length - 1);
2100                         if (error) {
2101                                 kfree(descrbuf, M_IFDESCR);
2102                                 break;
2103                         }
2104                 }
2105
2106                 ifnet_lock();
2107                 odescrbuf = ifp->if_description;
2108                 ifp->if_description = descrbuf;
2109                 ifnet_unlock();
2110
2111                 if (odescrbuf)
2112                         kfree(odescrbuf, M_IFDESCR);
2113
2114         case SIOCSIFFLAGS:
2115                 error = caps_priv_check(cred, SYSCAP_RESTRICTEDROOT);
2116                 if (error)
2117                         break;
2118                 new_flags = (ifr->ifr_flags & 0xffff) |
2119                     (ifr->ifr_flagshigh << 16);
2120                 if (ifp->if_flags & IFF_SMART) {
2121                         /* Smart drivers twiddle their own routes */
2122                 } else if (ifp->if_flags & IFF_UP &&
2123                     (new_flags & IFF_UP) == 0) {
2124                         if_down(ifp);
2125                 } else if (new_flags & IFF_UP &&
2126                     (ifp->if_flags & IFF_UP) == 0) {
2127                         do_ifup = 1;
2128                 }
2129
2130 #ifdef IFPOLL_ENABLE
2131                 if ((new_flags ^ ifp->if_flags) & IFF_NPOLLING) {
2132                         if (new_flags & IFF_NPOLLING)
2133                                 ifpoll_register(ifp);
2134                         else
2135                                 ifpoll_deregister(ifp);
2136                 }
2137 #endif
2138
2139                 ifp->if_flags = (ifp->if_flags & IFF_CANTCHANGE) |
2140                         (new_flags &~ IFF_CANTCHANGE);
2141                 if (new_flags & IFF_PPROMISC) {
2142                         /* Permanently promiscuous mode requested */
2143                         ifp->if_flags |= IFF_PROMISC;
2144                 } else if (ifp->if_pcount == 0) {
2145                         ifp->if_flags &= ~IFF_PROMISC;
2146                 }
2147                 if (ifp->if_ioctl) {
2148                         ifnet_serialize_all(ifp);
2149                         ifp->if_ioctl(ifp, cmd, data, cred);
2150                         ifnet_deserialize_all(ifp);
2151                 }
2152                 if (do_ifup)
2153                         if_up(ifp);
2154                 getmicrotime(&ifp->if_lastchange);
2155                 break;
2156
2157         case SIOCSIFCAP:
2158                 error = caps_priv_check(cred, SYSCAP_RESTRICTEDROOT);
2159                 if (error)
2160                         break;
2161                 if (ifr->ifr_reqcap & ~ifp->if_capabilities) {
2162                         error = EINVAL;
2163                         break;
2164                 }
2165                 ifnet_serialize_all(ifp);
2166                 ifp->if_ioctl(ifp, cmd, data, cred);
2167                 ifnet_deserialize_all(ifp);
2168                 break;
2169
2170         case SIOCSIFNAME:
2171                 error = caps_priv_check(cred, SYSCAP_RESTRICTEDROOT);
2172                 if (error)
2173                         break;
2174                 error = copyinstr(ifr->ifr_data, new_name, IFNAMSIZ, NULL);
2175                 if (error)
2176                         break;
2177                 if (new_name[0] == '\0') {
2178                         error = EINVAL;
2179                         break;
2180                 }
2181                 if (ifunit(new_name) != NULL) {
2182                         error = EEXIST;
2183                         break;
2184                 }
2185
2186                 EVENTHANDLER_INVOKE(ifnet_detach_event, ifp);
2187
2188                 /* Announce the departure of the interface. */
2189                 rt_ifannouncemsg(ifp, IFAN_DEPARTURE);
2190
2191                 strlcpy(ifp->if_xname, new_name, sizeof(ifp->if_xname));
2192                 ifa = TAILQ_FIRST(&ifp->if_addrheads[mycpuid])->ifa;
2193                 sdl = (struct sockaddr_dl *)ifa->ifa_addr;
2194                 namelen = strlen(new_name);
2195                 onamelen = sdl->sdl_nlen;
2196                 /*
2197                  * Move the address if needed.  This is safe because we
2198                  * allocate space for a name of length IFNAMSIZ when we
2199                  * create this in if_attach().
2200                  */
2201                 if (namelen != onamelen) {
2202                         bcopy(sdl->sdl_data + onamelen,
2203                             sdl->sdl_data + namelen, sdl->sdl_alen);
2204                 }
2205                 bcopy(new_name, sdl->sdl_data, namelen);
2206                 sdl->sdl_nlen = namelen;
2207                 sdl = (struct sockaddr_dl *)ifa->ifa_netmask;
2208                 bzero(sdl->sdl_data, onamelen);
2209                 while (namelen != 0)
2210                         sdl->sdl_data[--namelen] = 0xff;
2211
2212                 EVENTHANDLER_INVOKE(ifnet_attach_event, ifp);
2213
2214                 /* Announce the return of the interface. */
2215                 rt_ifannouncemsg(ifp, IFAN_ARRIVAL);
2216                 break;
2217
2218         case SIOCSIFMETRIC:
2219                 error = caps_priv_check(cred, SYSCAP_RESTRICTEDROOT);
2220                 if (error)
2221                         break;
2222                 ifp->if_metric = ifr->ifr_metric;
2223                 getmicrotime(&ifp->if_lastchange);
2224                 break;
2225
2226         case SIOCSIFPHYS:
2227                 error = caps_priv_check(cred, SYSCAP_RESTRICTEDROOT);
2228                 if (error)
2229                         break;
2230                 if (ifp->if_ioctl == NULL) {
2231                         error = EOPNOTSUPP;
2232                         break;
2233                 }
2234                 ifnet_serialize_all(ifp);
2235                 error = ifp->if_ioctl(ifp, cmd, data, cred);
2236                 ifnet_deserialize_all(ifp);
2237                 if (error == 0)
2238                         getmicrotime(&ifp->if_lastchange);
2239                 break;
2240
2241         case SIOCSIFMTU:
2242         {
2243                 u_long oldmtu = ifp->if_mtu;
2244
2245                 error = caps_priv_check(cred, SYSCAP_RESTRICTEDROOT);
2246                 if (error)
2247                         break;
2248                 if (ifp->if_ioctl == NULL) {
2249                         error = EOPNOTSUPP;
2250                         break;
2251                 }
2252                 if (ifr->ifr_mtu < IF_MINMTU || ifr->ifr_mtu > IF_MAXMTU) {
2253                         error = EINVAL;
2254                         break;
2255                 }
2256                 ifnet_serialize_all(ifp);
2257                 error = ifp->if_ioctl(ifp, cmd, data, cred);
2258                 ifnet_deserialize_all(ifp);
2259                 if (error == 0) {
2260                         getmicrotime(&ifp->if_lastchange);
2261                         rt_ifmsg(ifp);
2262                 }
2263                 /*
2264                  * If the link MTU changed, do network layer specific procedure.
2265                  */
2266                 if (ifp->if_mtu != oldmtu) {
2267 #ifdef INET6
2268                         nd6_setmtu(ifp);
2269 #endif
2270                 }
2271                 break;
2272         }
2273
2274         case SIOCSIFTSOLEN:
2275                 error = caps_priv_check(cred, SYSCAP_RESTRICTEDROOT);
2276                 if (error)
2277                         break;
2278
2279                 /* XXX need driver supplied upper limit */
2280                 if (ifr->ifr_tsolen <= 0) {
2281                         error = EINVAL;
2282                         break;
2283                 }
2284                 ifp->if_tsolen = ifr->ifr_tsolen;
2285                 break;
2286
2287         case SIOCADDMULTI:
2288         case SIOCDELMULTI:
2289                 error = caps_priv_check(cred, SYSCAP_RESTRICTEDROOT);
2290                 if (error)
2291                         break;
2292
2293                 /* Don't allow group membership on non-multicast interfaces. */
2294                 if ((ifp->if_flags & IFF_MULTICAST) == 0) {
2295                         error = EOPNOTSUPP;
2296                         break;
2297                 }
2298
2299                 /* Don't let users screw up protocols' entries. */
2300                 if (ifr->ifr_addr.sa_family != AF_LINK) {
2301                         error = EINVAL;
2302                         break;
2303                 }
2304
2305                 if (cmd == SIOCADDMULTI) {
2306                         struct ifmultiaddr *ifma;
2307                         error = if_addmulti(ifp, &ifr->ifr_addr, &ifma);
2308                 } else {
2309                         error = if_delmulti(ifp, &ifr->ifr_addr);
2310                 }
2311                 if (error == 0)
2312                         getmicrotime(&ifp->if_lastchange);
2313                 break;
2314
2315         case SIOCSIFPHYADDR:
2316         case SIOCDIFPHYADDR:
2317 #ifdef INET6
2318         case SIOCSIFPHYADDR_IN6:
2319 #endif
2320         case SIOCSLIFPHYADDR:
2321         case SIOCSIFMEDIA:
2322         case SIOCSIFGENERIC:
2323                 error = caps_priv_check(cred, SYSCAP_RESTRICTEDROOT);
2324                 if (error)
2325                         break;
2326                 if (ifp->if_ioctl == NULL) {
2327                         error = EOPNOTSUPP;
2328                         break;
2329                 }
2330                 ifnet_serialize_all(ifp);
2331                 error = ifp->if_ioctl(ifp, cmd, data, cred);
2332                 ifnet_deserialize_all(ifp);
2333                 if (error == 0)
2334                         getmicrotime(&ifp->if_lastchange);
2335                 break;
2336
2337         case SIOCGIFSTATUS:
2338                 ifs = (struct ifstat *)data;
2339                 ifs->ascii[0] = '\0';
2340                 /* fall through */
2341         case SIOCGIFPSRCADDR:
2342         case SIOCGIFPDSTADDR:
2343         case SIOCGLIFPHYADDR:
2344         case SIOCGIFMEDIA:
2345         case SIOCGIFXMEDIA:
2346         case SIOCGIFGENERIC:
2347                 if (ifp->if_ioctl == NULL) {
2348                         error = EOPNOTSUPP;
2349                         break;
2350                 }
2351                 ifnet_serialize_all(ifp);
2352                 error = ifp->if_ioctl(ifp, cmd, data, cred);
2353                 ifnet_deserialize_all(ifp);
2354                 break;
2355
2356         case SIOCSIFLLADDR:
2357                 error = caps_priv_check(cred, SYSCAP_RESTRICTEDROOT);
2358                 if (error)
2359                         break;
2360                 error = if_setlladdr(ifp, ifr->ifr_addr.sa_data,
2361                                      ifr->ifr_addr.sa_len);
2362                 EVENTHANDLER_INVOKE(iflladdr_event, ifp);
2363                 break;
2364
2365         case SIOCAIFGROUP:
2366                 ifgr = (struct ifgroupreq *)ifr;
2367                 error = caps_priv_check(cred, SYSCAP_NONET_IFCONFIG);
2368                 if (error)
2369                         return (error);
2370                 if ((error = if_addgroup(ifp, ifgr->ifgr_group)))
2371                         return (error);
2372                 break;
2373
2374         case SIOCDIFGROUP:
2375                 ifgr = (struct ifgroupreq *)ifr;
2376                 error = caps_priv_check(cred, SYSCAP_NONET_IFCONFIG);
2377                 if (error)
2378                         return (error);
2379                 if ((error = if_delgroup(ifp, ifgr->ifgr_group)))
2380                         return (error);
2381                 break;
2382
2383         case SIOCGIFGROUP:
2384                 ifgr = (struct ifgroupreq *)ifr;
2385                 if ((error = if_getgroups(ifgr, ifp)))
2386                         return (error);
2387                 break;
2388
2389         default:
2390                 oif_flags = ifp->if_flags;
2391                 if (so->so_proto == 0) {
2392                         error = EOPNOTSUPP;
2393                         break;
2394                 }
2395                 error = so_pru_control_direct(so, cmd, data, ifp);
2396
2397                 /*
2398                  * If the socket control method returns EOPNOTSUPP, pass the
2399                  * request directly to the interface.
2400                  *
2401                  * Exclude the SIOCSIF{ADDR,BRDADDR,DSTADDR,NETMASK} ioctls,
2402                  * because drivers may trust these ioctls to come from an
2403                  * already privileged layer and thus do not perform credentials
2404                  * checks or input validation.
2405                  */
2406                 if (error == EOPNOTSUPP &&
2407                     ifp->if_ioctl != NULL &&
2408                     cmd != SIOCSIFADDR &&
2409                     cmd != SIOCSIFBRDADDR &&
2410                     cmd != SIOCSIFDSTADDR &&
2411                     cmd != SIOCSIFNETMASK) {
2412                         ifnet_serialize_all(ifp);
2413                         error = ifp->if_ioctl(ifp, cmd, data, cred);
2414                         ifnet_deserialize_all(ifp);
2415                 }
2416
2417                 if ((oif_flags ^ ifp->if_flags) & IFF_UP) {
2418 #ifdef INET6
2419                         DELAY(100);/* XXX: temporary workaround for fxp issue*/
2420                         if (ifp->if_flags & IFF_UP) {
2421                                 crit_enter();
2422                                 in6_if_up(ifp);
2423                                 crit_exit();
2424                         }
2425 #endif
2426                 }
2427                 break;
2428         }
2429
2430         ifnet_unlock();
2431         return (error);
2432 }
2433
2434 /*
2435  * Set/clear promiscuous mode on interface ifp based on the truth value
2436  * of pswitch.  The calls are reference counted so that only the first
2437  * "on" request actually has an effect, as does the final "off" request.
2438  * Results are undefined if the "off" and "on" requests are not matched.
2439  */
2440 int
2441 ifpromisc(struct ifnet *ifp, int pswitch)
2442 {
2443         struct ifreq ifr;
2444         int error;
2445         int oldflags;
2446
2447         oldflags = ifp->if_flags;
2448         if (ifp->if_flags & IFF_PPROMISC) {
2449                 /* Do nothing if device is in permanently promiscuous mode */
2450                 ifp->if_pcount += pswitch ? 1 : -1;
2451                 return (0);
2452         }
2453         if (pswitch) {
2454                 /*
2455                  * If the device is not configured up, we cannot put it in
2456                  * promiscuous mode.
2457                  */
2458                 if ((ifp->if_flags & IFF_UP) == 0)
2459                         return (ENETDOWN);
2460                 if (ifp->if_pcount++ != 0)
2461                         return (0);
2462                 ifp->if_flags |= IFF_PROMISC;
2463                 log(LOG_INFO, "%s: promiscuous mode enabled\n",
2464                     ifp->if_xname);
2465         } else {
2466                 if (--ifp->if_pcount > 0)
2467                         return (0);
2468                 ifp->if_flags &= ~IFF_PROMISC;
2469                 log(LOG_INFO, "%s: promiscuous mode disabled\n",
2470                     ifp->if_xname);
2471         }
2472         ifr.ifr_flags = ifp->if_flags;
2473         ifr.ifr_flagshigh = ifp->if_flags >> 16;
2474         ifnet_serialize_all(ifp);
2475         error = ifp->if_ioctl(ifp, SIOCSIFFLAGS, (caddr_t)&ifr, NULL);
2476         ifnet_deserialize_all(ifp);
2477         if (error == 0)
2478                 rt_ifmsg(ifp);
2479         else
2480                 ifp->if_flags = oldflags;
2481         return error;
2482 }
2483
2484 /*
2485  * Return interface configuration
2486  * of system.  List may be used
2487  * in later ioctl's (above) to get
2488  * other information.
2489  */
2490 static int
2491 ifconf(u_long cmd, caddr_t data, struct ucred *cred)
2492 {
2493         struct ifconf *ifc = (struct ifconf *)data;
2494         struct ifnet *ifp;
2495         struct sockaddr *sa;
2496         struct ifreq ifr, *ifrp;
2497         int space = ifc->ifc_len, error = 0;
2498
2499         ifrp = ifc->ifc_req;
2500
2501         ifnet_lock();
2502         TAILQ_FOREACH(ifp, &ifnetlist, if_link) {
2503                 struct ifaddr_container *ifac, *ifac_mark;
2504                 struct ifaddr_marker mark;
2505                 struct ifaddrhead *head;
2506                 int addrs;
2507
2508                 if (space <= sizeof ifr)
2509                         break;
2510
2511                 /*
2512                  * Zero the stack declared structure first to prevent
2513                  * memory disclosure.
2514                  */
2515                 bzero(&ifr, sizeof(ifr));
2516                 if (strlcpy(ifr.ifr_name, ifp->if_xname, sizeof(ifr.ifr_name))
2517                     >= sizeof(ifr.ifr_name)) {
2518                         error = ENAMETOOLONG;
2519                         break;
2520                 }
2521
2522                 /*
2523                  * Add a marker, since copyout() could block and during that
2524                  * period the list could be changed.  Inserting the marker to
2525                  * the header of the list will not cause trouble for the code
2526                  * assuming that the first element of the list is AF_LINK; the
2527                  * marker will be moved to the next position w/o blocking.
2528                  */
2529                 ifa_marker_init(&mark, ifp);
2530                 ifac_mark = &mark.ifac;
2531                 head = &ifp->if_addrheads[mycpuid];
2532
2533                 addrs = 0;
2534                 TAILQ_INSERT_HEAD(head, ifac_mark, ifa_link);
2535                 while ((ifac = TAILQ_NEXT(ifac_mark, ifa_link)) != NULL) {
2536                         struct ifaddr *ifa = ifac->ifa;
2537
2538                         TAILQ_REMOVE(head, ifac_mark, ifa_link);
2539                         TAILQ_INSERT_AFTER(head, ifac, ifac_mark, ifa_link);
2540
2541                         /* Ignore marker */
2542                         if (ifa->ifa_addr->sa_family == AF_UNSPEC)
2543                                 continue;
2544
2545                         if (space <= sizeof ifr)
2546                                 break;
2547                         sa = ifa->ifa_addr;
2548                         if (cred->cr_prison && prison_if(cred, sa))
2549                                 continue;
2550                         addrs++;
2551                         /*
2552                          * Keep a reference on this ifaddr, so that it will
2553                          * not be destroyed when its address is copied to
2554                          * the userland, which could block.
2555                          */
2556                         IFAREF(ifa);
2557                         if (sa->sa_len <= sizeof(*sa)) {
2558                                 ifr.ifr_addr = *sa;
2559                                 error = copyout(&ifr, ifrp, sizeof ifr);
2560                                 ifrp++;
2561                         } else {
2562                                 if (space < (sizeof ifr) + sa->sa_len -
2563                                             sizeof(*sa)) {
2564                                         IFAFREE(ifa);
2565                                         break;
2566                                 }
2567                                 space -= sa->sa_len - sizeof(*sa);
2568                                 error = copyout(&ifr, ifrp,
2569                                                 sizeof ifr.ifr_name);
2570                                 if (error == 0)
2571                                         error = copyout(sa, &ifrp->ifr_addr,
2572                                                         sa->sa_len);
2573                                 ifrp = (struct ifreq *)
2574                                         (sa->sa_len + (caddr_t)&ifrp->ifr_addr);
2575                         }
2576                         IFAFREE(ifa);
2577                         if (error)
2578                                 break;
2579                         space -= sizeof ifr;
2580                 }
2581                 TAILQ_REMOVE(head, ifac_mark, ifa_link);
2582                 if (error)
2583                         break;
2584                 if (!addrs) {
2585                         bzero(&ifr.ifr_addr, sizeof ifr.ifr_addr);
2586                         error = copyout(&ifr, ifrp, sizeof ifr);
2587                         if (error)
2588                                 break;
2589                         space -= sizeof ifr;
2590                         ifrp++;
2591                 }
2592         }
2593         ifnet_unlock();
2594
2595         ifc->ifc_len -= space;
2596         return (error);
2597 }
2598
2599 /*
2600  * Just like if_promisc(), but for all-multicast-reception mode.
2601  */
2602 int
2603 if_allmulti(struct ifnet *ifp, int onswitch)
2604 {
2605         int error = 0;
2606         struct ifreq ifr;
2607
2608         crit_enter();
2609
2610         if (onswitch) {
2611                 if (ifp->if_amcount++ == 0) {
2612                         ifp->if_flags |= IFF_ALLMULTI;
2613                         ifr.ifr_flags = ifp->if_flags;
2614                         ifr.ifr_flagshigh = ifp->if_flags >> 16;
2615                         ifnet_serialize_all(ifp);
2616                         error = ifp->if_ioctl(ifp, SIOCSIFFLAGS, (caddr_t)&ifr,
2617                                               NULL);
2618                         ifnet_deserialize_all(ifp);
2619                 }
2620         } else {
2621                 if (ifp->if_amcount > 1) {
2622                         ifp->if_amcount--;
2623                 } else {
2624                         ifp->if_amcount = 0;
2625                         ifp->if_flags &= ~IFF_ALLMULTI;
2626                         ifr.ifr_flags = ifp->if_flags;
2627                         ifr.ifr_flagshigh = ifp->if_flags >> 16;
2628                         ifnet_serialize_all(ifp);
2629                         error = ifp->if_ioctl(ifp, SIOCSIFFLAGS, (caddr_t)&ifr,
2630                                               NULL);
2631                         ifnet_deserialize_all(ifp);
2632                 }
2633         }
2634
2635         crit_exit();
2636
2637         if (error == 0)
2638                 rt_ifmsg(ifp);
2639         return error;
2640 }
2641
2642 /*
2643  * Add a multicast listenership to the interface in question.
2644  * The link layer provides a routine which converts
2645  */
2646 int
2647 if_addmulti_serialized(struct ifnet *ifp, struct sockaddr *sa,
2648     struct ifmultiaddr **retifma)
2649 {
2650         struct sockaddr *llsa, *dupsa;
2651         int error;
2652         struct ifmultiaddr *ifma;
2653
2654         ASSERT_IFNET_SERIALIZED_ALL(ifp);
2655
2656         /*
2657          * If the matching multicast address already exists
2658          * then don't add a new one, just add a reference
2659          */
2660         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2661                 if (sa_equal(sa, ifma->ifma_addr)) {
2662                         ifma->ifma_refcount++;
2663                         if (retifma)
2664                                 *retifma = ifma;
2665                         return 0;
2666                 }
2667         }
2668
2669         /*
2670          * Give the link layer a chance to accept/reject it, and also
2671          * find out which AF_LINK address this maps to, if it isn't one
2672          * already.
2673          */
2674         if (ifp->if_resolvemulti) {
2675                 error = ifp->if_resolvemulti(ifp, &llsa, sa);
2676                 if (error)
2677                         return error;
2678         } else {
2679                 llsa = NULL;
2680         }
2681
2682         ifma = kmalloc(sizeof *ifma, M_IFMADDR, M_INTWAIT);
2683         dupsa = kmalloc(sa->sa_len, M_IFMADDR, M_INTWAIT);
2684         bcopy(sa, dupsa, sa->sa_len);
2685
2686         ifma->ifma_addr = dupsa;
2687         ifma->ifma_lladdr = llsa;
2688         ifma->ifma_ifp = ifp;
2689         ifma->ifma_refcount = 1;
2690         ifma->ifma_protospec = NULL;
2691         rt_newmaddrmsg(RTM_NEWMADDR, ifma);
2692
2693         TAILQ_INSERT_HEAD(&ifp->if_multiaddrs, ifma, ifma_link);
2694         if (retifma)
2695                 *retifma = ifma;
2696
2697         if (llsa != NULL) {
2698                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2699                         if (sa_equal(ifma->ifma_addr, llsa))
2700                                 break;
2701                 }
2702                 if (ifma) {
2703                         ifma->ifma_refcount++;
2704                 } else {
2705                         ifma = kmalloc(sizeof *ifma, M_IFMADDR, M_INTWAIT);
2706                         dupsa = kmalloc(llsa->sa_len, M_IFMADDR, M_INTWAIT);
2707                         bcopy(llsa, dupsa, llsa->sa_len);
2708                         ifma->ifma_addr = dupsa;
2709                         ifma->ifma_ifp = ifp;
2710                         ifma->ifma_refcount = 1;
2711                         TAILQ_INSERT_HEAD(&ifp->if_multiaddrs, ifma, ifma_link);
2712                 }
2713         }
2714         /*
2715          * We are certain we have added something, so call down to the
2716          * interface to let them know about it.
2717          */
2718         if (ifp->if_ioctl)
2719                 ifp->if_ioctl(ifp, SIOCADDMULTI, 0, NULL);
2720
2721         return 0;
2722 }
2723
2724 int
2725 if_addmulti(struct ifnet *ifp, struct sockaddr *sa,
2726     struct ifmultiaddr **retifma)
2727 {
2728         int error;
2729
2730         ifnet_serialize_all(ifp);
2731         error = if_addmulti_serialized(ifp, sa, retifma);
2732         ifnet_deserialize_all(ifp);
2733
2734         return error;
2735 }
2736
2737 /*
2738  * Remove a reference to a multicast address on this interface.  Yell
2739  * if the request does not match an existing membership.
2740  */
2741 static int
2742 if_delmulti_serialized(struct ifnet *ifp, struct sockaddr *sa)
2743 {
2744         struct ifmultiaddr *ifma;
2745
2746         ASSERT_IFNET_SERIALIZED_ALL(ifp);
2747
2748         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link)
2749                 if (sa_equal(sa, ifma->ifma_addr))
2750                         break;
2751         if (ifma == NULL)
2752                 return ENOENT;
2753
2754         if (ifma->ifma_refcount > 1) {
2755                 ifma->ifma_refcount--;
2756                 return 0;
2757         }
2758
2759         rt_newmaddrmsg(RTM_DELMADDR, ifma);
2760         sa = ifma->ifma_lladdr;
2761         TAILQ_REMOVE(&ifp->if_multiaddrs, ifma, ifma_link);
2762         /*
2763          * Make sure the interface driver is notified
2764          * in the case of a link layer mcast group being left.
2765          */
2766         if (ifma->ifma_addr->sa_family == AF_LINK && sa == NULL)
2767                 ifp->if_ioctl(ifp, SIOCDELMULTI, 0, NULL);
2768         kfree(ifma->ifma_addr, M_IFMADDR);
2769         kfree(ifma, M_IFMADDR);
2770         if (sa == NULL)
2771                 return 0;
2772
2773         /*
2774          * Now look for the link-layer address which corresponds to
2775          * this network address.  It had been squirreled away in
2776          * ifma->ifma_lladdr for this purpose (so we don't have
2777          * to call ifp->if_resolvemulti() again), and we saved that
2778          * value in sa above.  If some nasty deleted the
2779          * link-layer address out from underneath us, we can deal because
2780          * the address we stored was is not the same as the one which was
2781          * in the record for the link-layer address.  (So we don't complain
2782          * in that case.)
2783          */
2784         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link)
2785                 if (sa_equal(sa, ifma->ifma_addr))
2786                         break;
2787         if (ifma == NULL)
2788                 return 0;
2789
2790         if (ifma->ifma_refcount > 1) {
2791                 ifma->ifma_refcount--;
2792                 return 0;
2793         }
2794
2795         TAILQ_REMOVE(&ifp->if_multiaddrs, ifma, ifma_link);
2796         ifp->if_ioctl(ifp, SIOCDELMULTI, 0, NULL);
2797         kfree(ifma->ifma_addr, M_IFMADDR);
2798         kfree(sa, M_IFMADDR);
2799         kfree(ifma, M_IFMADDR);
2800
2801         return 0;
2802 }
2803
2804 int
2805 if_delmulti(struct ifnet *ifp, struct sockaddr *sa)
2806 {
2807         int error;
2808
2809         ifnet_serialize_all(ifp);
2810         error = if_delmulti_serialized(ifp, sa);
2811         ifnet_deserialize_all(ifp);
2812
2813         return error;
2814 }
2815
2816 /*
2817  * Delete all multicast group membership for an interface.
2818  * Should be used to quickly flush all multicast filters.
2819  */
2820 void
2821 if_delallmulti_serialized(struct ifnet *ifp)
2822 {
2823         struct ifmultiaddr *ifma, mark;
2824         struct sockaddr sa;
2825
2826         ASSERT_IFNET_SERIALIZED_ALL(ifp);
2827
2828         bzero(&sa, sizeof(sa));
2829         sa.sa_family = AF_UNSPEC;
2830         sa.sa_len = sizeof(sa);
2831
2832         bzero(&mark, sizeof(mark));
2833         mark.ifma_addr = &sa;
2834
2835         TAILQ_INSERT_HEAD(&ifp->if_multiaddrs, &mark, ifma_link);
2836         while ((ifma = TAILQ_NEXT(&mark, ifma_link)) != NULL) {
2837                 TAILQ_REMOVE(&ifp->if_multiaddrs, &mark, ifma_link);
2838                 TAILQ_INSERT_AFTER(&ifp->if_multiaddrs, ifma, &mark,
2839                     ifma_link);
2840
2841                 if (ifma->ifma_addr->sa_family == AF_UNSPEC)
2842                         continue;
2843
2844                 if_delmulti_serialized(ifp, ifma->ifma_addr);
2845         }
2846         TAILQ_REMOVE(&ifp->if_multiaddrs, &mark, ifma_link);
2847 }
2848
2849
2850 /*
2851  * Set the link layer address on an interface.
2852  *
2853  * At this time we only support certain types of interfaces,
2854  * and we don't allow the length of the address to change.
2855  */
2856 int
2857 if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len)
2858 {
2859         struct sockaddr_dl *sdl;
2860         struct ifreq ifr;
2861
2862         sdl = IF_LLSOCKADDR(ifp);
2863         if (sdl == NULL)
2864                 return (EINVAL);
2865         if (len != sdl->sdl_alen)       /* don't allow length to change */
2866                 return (EINVAL);
2867         switch (ifp->if_type) {
2868         case IFT_ETHER:                 /* these types use struct arpcom */
2869         case IFT_XETHER:
2870         case IFT_L2VLAN:
2871         case IFT_IEEE8023ADLAG:
2872                 bcopy(lladdr, ((struct arpcom *)ifp->if_softc)->ac_enaddr, len);
2873                 bcopy(lladdr, LLADDR(sdl), len);
2874                 break;
2875         default:
2876                 return (ENODEV);
2877         }
2878         /*
2879          * If the interface is already up, we need
2880          * to re-init it in order to reprogram its
2881          * address filter.
2882          */
2883         ifnet_serialize_all(ifp);
2884         if ((ifp->if_flags & IFF_UP) != 0) {
2885 #ifdef INET
2886                 struct ifaddr_container *ifac;
2887 #endif
2888
2889                 ifp->if_flags &= ~IFF_UP;
2890                 ifr.ifr_flags = ifp->if_flags;
2891                 ifr.ifr_flagshigh = ifp->if_flags >> 16;
2892                 ifp->if_ioctl(ifp, SIOCSIFFLAGS, (caddr_t)&ifr,
2893                               NULL);
2894                 ifp->if_flags |= IFF_UP;
2895                 ifr.ifr_flags = ifp->if_flags;
2896                 ifr.ifr_flagshigh = ifp->if_flags >> 16;
2897                 ifp->if_ioctl(ifp, SIOCSIFFLAGS, (caddr_t)&ifr,
2898                                  NULL);
2899 #ifdef INET
2900                 /*
2901                  * Also send gratuitous ARPs to notify other nodes about
2902                  * the address change.
2903                  */
2904                 TAILQ_FOREACH(ifac, &ifp->if_addrheads[mycpuid], ifa_link) {
2905                         struct ifaddr *ifa = ifac->ifa;
2906
2907                         if (ifa->ifa_addr != NULL &&
2908                             ifa->ifa_addr->sa_family == AF_INET)
2909                                 arp_gratuitous(ifp, ifa);
2910                 }
2911 #endif
2912         }
2913         ifnet_deserialize_all(ifp);
2914         return (0);
2915 }
2916
2917
2918 /*
2919  * Tunnel interfaces can nest, also they may cause infinite recursion
2920  * calls when misconfigured.  Introduce an upper limit to prevent infinite
2921  * recursions, as well as to constrain the nesting depth.
2922  *
2923  * Return 0, if tunnel nesting count is equal or less than limit.
2924  */
2925 int
2926 if_tunnel_check_nesting(struct ifnet *ifp, struct mbuf *m, uint32_t cookie,
2927                         int limit)
2928 {
2929         struct m_tag *mtag;
2930         int count;
2931
2932         count = 1;
2933         mtag = m_tag_locate(m, cookie, 0 /* type */, NULL);
2934         if (mtag != NULL)
2935                 count += *(int *)(mtag + 1);
2936         if (count > limit) {
2937                 log(LOG_NOTICE,
2938                     "%s: packet looped too many times (%d), limit %d\n",
2939                     ifp->if_xname, count, limit);
2940                 return (ELOOP);
2941         }
2942
2943         if (mtag == NULL) {
2944                 mtag = m_tag_alloc(cookie, 0, sizeof(int), M_NOWAIT);
2945                 if (mtag == NULL)
2946                         return (ENOMEM);
2947                 m_tag_prepend(m, mtag);
2948         }
2949
2950         *(int *)(mtag + 1) = count;
2951         return (0);
2952 }
2953
2954
2955 /*
2956  * Locate an interface based on a complete address.
2957  */
2958 struct ifnet *
2959 if_bylla(const void *lla, unsigned char lla_len)
2960 {
2961         const struct ifnet_array *arr;
2962         struct ifnet *ifp;
2963         struct sockaddr_dl *sdl;
2964         int i;
2965
2966         arr = ifnet_array_get();
2967         for (i = 0; i < arr->ifnet_count; ++i) {
2968                 ifp = arr->ifnet_arr[i];
2969                 if (ifp->if_addrlen != lla_len)
2970                         continue;
2971
2972                 sdl = IF_LLSOCKADDR(ifp);
2973                 if (memcmp(lla, LLADDR(sdl), lla_len) == 0)
2974                         return (ifp);
2975         }
2976         return (NULL);
2977 }
2978
2979 struct ifmultiaddr *
2980 ifmaof_ifpforaddr(struct sockaddr *sa, struct ifnet *ifp)
2981 {
2982         struct ifmultiaddr *ifma;
2983
2984         /* TODO: need ifnet_serialize_main */
2985         ifnet_serialize_all(ifp);
2986         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link)
2987                 if (sa_equal(ifma->ifma_addr, sa))
2988                         break;
2989         ifnet_deserialize_all(ifp);
2990
2991         return ifma;
2992 }
2993
2994 /*
2995  * This function locates the first real ethernet MAC from a network
2996  * card and loads it into node, returning 0 on success or ENOENT if
2997  * no suitable interfaces were found.  It is used by the uuid code to
2998  * generate a unique 6-byte number.
2999  */
3000 int
3001 if_getanyethermac(uint16_t *node, int minlen)
3002 {
3003         struct ifnet *ifp;
3004         struct sockaddr_dl *sdl;
3005
3006         ifnet_lock();
3007         TAILQ_FOREACH(ifp, &ifnetlist, if_link) {
3008                 if (ifp->if_type != IFT_ETHER)
3009                         continue;
3010                 sdl = IF_LLSOCKADDR(ifp);
3011                 if (sdl->sdl_alen < minlen)
3012                         continue;
3013                 bcopy(((struct arpcom *)ifp->if_softc)->ac_enaddr, node,
3014                       minlen);
3015                 ifnet_unlock();
3016                 return(0);
3017         }
3018         ifnet_unlock();
3019         return (ENOENT);
3020 }
3021
3022 /*
3023  * The name argument must be a pointer to storage which will last as
3024  * long as the interface does.  For physical devices, the result of
3025  * device_get_name(dev) is a good choice and for pseudo-devices a
3026  * static string works well.
3027  */
3028 void
3029 if_initname(struct ifnet *ifp, const char *name, int unit)
3030 {
3031         ifp->if_dname = name;
3032         ifp->if_dunit = unit;
3033         if (unit != IF_DUNIT_NONE)
3034                 ksnprintf(ifp->if_xname, IFNAMSIZ, "%s%d", name, unit);
3035         else
3036                 strlcpy(ifp->if_xname, name, IFNAMSIZ);
3037 }
3038
3039 int
3040 if_printf(struct ifnet *ifp, const char *fmt, ...)
3041 {
3042         __va_list ap;
3043         int retval;
3044
3045         retval = kprintf("%s: ", ifp->if_xname);
3046         __va_start(ap, fmt);
3047         retval += kvprintf(fmt, ap);
3048         __va_end(ap);
3049         return (retval);
3050 }
3051
3052 struct ifnet *
3053 if_alloc(uint8_t type)
3054 {
3055         struct ifnet *ifp;
3056         size_t size;
3057
3058         /*
3059          * XXX temporary hack until arpcom is setup in if_l2com
3060          */
3061         if (type == IFT_ETHER)
3062                 size = sizeof(struct arpcom);
3063         else
3064                 size = sizeof(struct ifnet);
3065
3066         ifp = kmalloc(size, M_IFNET, M_WAITOK|M_ZERO);
3067
3068         ifp->if_type = type;
3069
3070         if (if_com_alloc[type] != NULL) {
3071                 ifp->if_l2com = if_com_alloc[type](type, ifp);
3072                 if (ifp->if_l2com == NULL) {
3073                         kfree(ifp, M_IFNET);
3074                         return (NULL);
3075                 }
3076         }
3077         return (ifp);
3078 }
3079
3080 void
3081 if_free(struct ifnet *ifp)
3082 {
3083         if (ifp->if_description != NULL)
3084                 kfree(ifp->if_description, M_IFDESCR);
3085         kfree(ifp, M_IFNET);
3086 }
3087
3088 void
3089 ifq_set_classic(struct ifaltq *ifq)
3090 {
3091         ifq_set_methods(ifq, ifq->altq_ifp->if_mapsubq,
3092             ifsq_classic_enqueue, ifsq_classic_dequeue, ifsq_classic_request);
3093 }
3094
3095 void
3096 ifq_set_methods(struct ifaltq *ifq, altq_mapsubq_t mapsubq,
3097     ifsq_enqueue_t enqueue, ifsq_dequeue_t dequeue, ifsq_request_t request)
3098 {
3099         int q;
3100
3101         KASSERT(mapsubq != NULL, ("mapsubq is not specified"));
3102         KASSERT(enqueue != NULL, ("enqueue is not specified"));
3103         KASSERT(dequeue != NULL, ("dequeue is not specified"));
3104         KASSERT(request != NULL, ("request is not specified"));
3105
3106         ifq->altq_mapsubq = mapsubq;
3107         for (q = 0; q < ifq->altq_subq_cnt; ++q) {
3108                 struct ifaltq_subque *ifsq = &ifq->altq_subq[q];
3109
3110                 ifsq->ifsq_enqueue = enqueue;
3111                 ifsq->ifsq_dequeue = dequeue;
3112                 ifsq->ifsq_request = request;
3113         }
3114 }
3115
3116 static void
3117 ifsq_norm_enqueue(struct ifaltq_subque *ifsq, struct mbuf *m)
3118 {
3119
3120         classq_add(&ifsq->ifsq_norm, m);
3121         ALTQ_SQ_CNTR_INC(ifsq, m->m_pkthdr.len);
3122 }
3123
3124 static void
3125 ifsq_prio_enqueue(struct ifaltq_subque *ifsq, struct mbuf *m)
3126 {
3127
3128         classq_add(&ifsq->ifsq_prio, m);
3129         ALTQ_SQ_CNTR_INC(ifsq, m->m_pkthdr.len);
3130         ALTQ_SQ_PRIO_CNTR_INC(ifsq, m->m_pkthdr.len);
3131 }
3132
3133 static struct mbuf *
3134 ifsq_norm_dequeue(struct ifaltq_subque *ifsq)
3135 {
3136         struct mbuf *m;
3137
3138         m = classq_get(&ifsq->ifsq_norm);
3139         if (m != NULL)
3140                 ALTQ_SQ_CNTR_DEC(ifsq, m->m_pkthdr.len);
3141         return (m);
3142 }
3143
3144 static struct mbuf *
3145 ifsq_prio_dequeue(struct ifaltq_subque *ifsq)
3146 {
3147         struct mbuf *m;
3148
3149         m = classq_get(&ifsq->ifsq_prio);
3150         if (m != NULL) {
3151                 ALTQ_SQ_CNTR_DEC(ifsq, m->m_pkthdr.len);
3152                 ALTQ_SQ_PRIO_CNTR_DEC(ifsq, m->m_pkthdr.len);
3153         }
3154         return (m);
3155 }
3156
3157 int
3158 ifsq_classic_enqueue(struct ifaltq_subque *ifsq, struct mbuf *m,
3159     struct altq_pktattr *pa __unused)
3160 {
3161
3162         M_ASSERTPKTHDR(m);
3163 again:
3164         if (ifsq->ifsq_len >= ifsq->ifsq_maxlen ||
3165             ifsq->ifsq_bcnt >= ifsq->ifsq_maxbcnt) {
3166                 struct mbuf *m_drop;
3167
3168                 if (m->m_flags & M_PRIO) {
3169                         m_drop = NULL;
3170                         if (ifsq->ifsq_prio_len < (ifsq->ifsq_maxlen >> 1) &&
3171                             ifsq->ifsq_prio_bcnt < (ifsq->ifsq_maxbcnt >> 1)) {
3172                                 /* Try dropping some from normal queue. */
3173                                 m_drop = ifsq_norm_dequeue(ifsq);
3174                         }
3175                         if (m_drop == NULL)
3176                                 m_drop = ifsq_prio_dequeue(ifsq);
3177                 } else {
3178                         m_drop = ifsq_norm_dequeue(ifsq);
3179                 }
3180                 if (m_drop != NULL) {
3181                         IFNET_STAT_INC(ifsq->ifsq_ifp, oqdrops, 1);
3182                         m_freem(m_drop);
3183                         goto again;
3184                 }
3185                 /*
3186                  * No old packets could be dropped!
3187                  * NOTE: Caller increases oqdrops.
3188                  */
3189                 m_freem(m);
3190                 return (ENOBUFS);
3191         } else {
3192                 if (m->m_flags & M_PRIO)
3193                         ifsq_prio_enqueue(ifsq, m);
3194                 else
3195                         ifsq_norm_enqueue(ifsq, m);
3196                 return (0);
3197         }
3198 }
3199
3200 struct mbuf *
3201 ifsq_classic_dequeue(struct ifaltq_subque *ifsq, int op)
3202 {
3203         struct mbuf *m;
3204
3205         switch (op) {
3206         case ALTDQ_POLL:
3207                 m = classq_head(&ifsq->ifsq_prio);
3208                 if (m == NULL)
3209                         m = classq_head(&ifsq->ifsq_norm);
3210                 break;
3211
3212         case ALTDQ_REMOVE:
3213                 m = ifsq_prio_dequeue(ifsq);
3214                 if (m == NULL)
3215                         m = ifsq_norm_dequeue(ifsq);
3216                 break;
3217
3218         default:
3219                 panic("unsupported ALTQ dequeue op: %d", op);
3220         }
3221         return m;
3222 }
3223
3224 int
3225 ifsq_classic_request(struct ifaltq_subque *ifsq, int req, void *arg)
3226 {
3227         switch (req) {
3228         case ALTRQ_PURGE:
3229                 for (;;) {
3230                         struct mbuf *m;
3231
3232                         m = ifsq_classic_dequeue(ifsq, ALTDQ_REMOVE);
3233                         if (m == NULL)
3234                                 break;
3235                         m_freem(m);
3236                 }
3237                 break;
3238
3239         default:
3240                 panic("unsupported ALTQ request: %d", req);
3241         }
3242         return 0;
3243 }
3244
3245 static void
3246 ifsq_ifstart_try(struct ifaltq_subque *ifsq, int force_sched)
3247 {
3248         struct ifnet *ifp = ifsq_get_ifp(ifsq);
3249         int running = 0, need_sched;
3250
3251         /*
3252          * Try to do direct ifnet.if_start on the subqueue first, if there is
3253          * contention on the subqueue hardware serializer, ifnet.if_start on
3254          * the subqueue will be scheduled on the subqueue owner CPU.
3255          */
3256         if (!ifsq_tryserialize_hw(ifsq)) {
3257                 /*
3258                  * Subqueue hardware serializer contention happened,
3259                  * ifnet.if_start on the subqueue is scheduled on
3260                  * the subqueue owner CPU, and we keep going.
3261                  */
3262                 ifsq_ifstart_schedule(ifsq, 1);
3263                 return;
3264         }
3265
3266         if ((ifp->if_flags & IFF_RUNNING) && !ifsq_is_oactive(ifsq)) {
3267                 ifp->if_start(ifp, ifsq);
3268                 if ((ifp->if_flags & IFF_RUNNING) && !ifsq_is_oactive(ifsq))
3269                         running = 1;
3270         }
3271         need_sched = ifsq_ifstart_need_schedule(ifsq, running);
3272
3273         ifsq_deserialize_hw(ifsq);
3274
3275         if (need_sched) {
3276                 /*
3277                  * More data need to be transmitted, ifnet.if_start on the
3278                  * subqueue is scheduled on the subqueue owner CPU, and we
3279                  * keep going.
3280                  * NOTE: ifnet.if_start subqueue interlock is not released.
3281                  */
3282                 ifsq_ifstart_schedule(ifsq, force_sched);
3283         }
3284 }
3285
3286 /*
3287  * Subqeue packets staging mechanism:
3288  *
3289  * The packets enqueued into the subqueue are staged to a certain amount
3290  * before the ifnet.if_start on the subqueue is called.  In this way, the
3291  * driver could avoid writing to hardware registers upon every packet,
3292  * instead, hardware registers could be written when certain amount of
3293  * packets are put onto hardware TX ring.  The measurement on several modern
3294  * NICs (emx(4), igb(4), bnx(4), bge(4), jme(4)) shows that the hardware
3295  * registers writing aggregation could save ~20% CPU time when 18bytes UDP
3296  * datagrams are transmitted at 1.48Mpps.  The performance improvement by
3297  * hardware registers writing aggeregation is also mentioned by Luigi Rizzo's
3298  * netmap paper (http://info.iet.unipi.it/~luigi/netmap/).
3299  *
3300  * Subqueue packets staging is performed for two entry points into drivers'
3301  * transmission function:
3302  * - Direct ifnet.if_start calling on the subqueue, i.e. ifsq_ifstart_try()
3303  * - ifnet.if_start scheduling on the subqueue, i.e. ifsq_ifstart_schedule()
3304  *
3305  * Subqueue packets staging will be stopped upon any of the following
3306  * conditions:
3307  * - If the count of packets enqueued on the current CPU is great than or
3308  *   equal to ifsq_stage_cntmax. (XXX this should be per-interface)
3309  * - If the total length of packets enqueued on the current CPU is great
3310  *   than or equal to the hardware's MTU - max_protohdr.  max_protohdr is
3311  *   cut from the hardware's MTU mainly bacause a full TCP segment's size
3312  *   is usually less than hardware's MTU.
3313  * - ifsq_ifstart_schedule() is not pending on the current CPU and
3314  *   ifnet.if_start subqueue interlock (ifaltq_subq.ifsq_started) is not
3315  *   released.
3316  * - The if_start_rollup(), which is registered as low priority netisr
3317  *   rollup function, is called; probably because no more work is pending
3318  *   for netisr.
3319  *
3320  * NOTE:
3321  * Currently subqueue packet staging is only performed in netisr threads.
3322  */
3323 int
3324 ifq_dispatch(struct ifnet *ifp, struct mbuf *m, struct altq_pktattr *pa)
3325 {
3326         struct ifaltq *ifq = &ifp->if_snd;
3327         struct ifaltq_subque *ifsq;
3328         int error, start = 0, len, mcast = 0, avoid_start = 0;
3329         struct ifsubq_stage_head *head = NULL;
3330         struct ifsubq_stage *stage = NULL;
3331         struct globaldata *gd = mycpu;
3332         struct thread *td = gd->gd_curthread;
3333
3334         crit_enter_quick(td);
3335
3336         ifsq = ifq_map_subq(ifq, gd->gd_cpuid);
3337         ASSERT_ALTQ_SQ_NOT_SERIALIZED_HW(ifsq);
3338
3339         len = m->m_pkthdr.len;
3340         if (m->m_flags & M_MCAST)
3341                 mcast = 1;
3342
3343         if (td->td_type == TD_TYPE_NETISR) {
3344                 head = &ifsubq_stage_heads[mycpuid];
3345                 stage = ifsq_get_stage(ifsq, mycpuid);
3346
3347                 stage->stg_cnt++;
3348                 stage->stg_len += len;
3349                 if (stage->stg_cnt < ifsq_stage_cntmax &&
3350                     stage->stg_len < (ifp->if_mtu - max_protohdr))
3351                         avoid_start = 1;
3352         }
3353
3354         ALTQ_SQ_LOCK(ifsq);
3355         error = ifsq_enqueue_locked(ifsq, m, pa);
3356         if (error) {
3357                 IFNET_STAT_INC(ifp, oqdrops, 1);
3358                 if (!ifsq_data_ready(ifsq)) {
3359                         ALTQ_SQ_UNLOCK(ifsq);
3360                         goto done;
3361                 }
3362                 avoid_start = 0;
3363         } else {
3364                 IFNET_STAT_INC(ifp, obytes, len);
3365                 if (mcast)
3366                         IFNET_STAT_INC(ifp, omcasts, 1);
3367         }
3368         if (!ifsq_is_started(ifsq)) {
3369                 if (avoid_start) {
3370                         ALTQ_SQ_UNLOCK(ifsq);
3371
3372                         KKASSERT(!error);
3373                         if ((stage->stg_flags & IFSQ_STAGE_FLAG_QUED) == 0)
3374                                 ifsq_stage_insert(head, stage);
3375
3376                         goto done;
3377                 }
3378
3379                 /*
3380                  * Hold the subqueue interlock of ifnet.if_start
3381                  */
3382                 ifsq_set_started(ifsq);
3383                 start = 1;
3384         }
3385         ALTQ_SQ_UNLOCK(ifsq);
3386
3387         if (stage != NULL) {
3388                 if (!start && (stage->stg_flags & IFSQ_STAGE_FLAG_SCHED)) {
3389                         KKASSERT(stage->stg_flags & IFSQ_STAGE_FLAG_QUED);
3390                         if (!avoid_start) {
3391                                 ifsq_stage_remove(head, stage);
3392                                 ifsq_ifstart_schedule(ifsq, 1);
3393                         }
3394                         goto done;
3395                 }
3396
3397                 if (stage->stg_flags & IFSQ_STAGE_FLAG_QUED) {
3398                         ifsq_stage_remove(head, stage);
3399                 } else {
3400                         stage->stg_cnt = 0;
3401                         stage->stg_len = 0;
3402                 }
3403         }
3404
3405         if (start)
3406                 ifsq_ifstart_try(ifsq, 0);
3407
3408 done:
3409         crit_exit_quick(td);
3410         return error;
3411 }
3412
3413 void *
3414 ifa_create(int size)
3415 {
3416         struct ifaddr *ifa;
3417         int i;
3418
3419         KASSERT(size >= sizeof(*ifa), ("ifaddr size too small"));
3420
3421         ifa = kmalloc(size, M_IFADDR, M_INTWAIT | M_ZERO);
3422
3423         /*
3424          * Make ifa_container availabel on all CPUs, since they
3425          * could be accessed by any threads.
3426          */
3427         ifa->ifa_containers =
3428                 kmalloc(ncpus * sizeof(struct ifaddr_container),
3429                         M_IFADDR,
3430                         M_INTWAIT | M_ZERO | M_CACHEALIGN);
3431
3432         ifa->ifa_ncnt = ncpus;
3433         for (i = 0; i < ncpus; ++i) {
3434                 struct ifaddr_container *ifac = &ifa->ifa_containers[i];
3435
3436                 ifac->ifa_magic = IFA_CONTAINER_MAGIC;
3437                 ifac->ifa = ifa;
3438                 ifac->ifa_refcnt = 1;
3439         }
3440 #ifdef IFADDR_DEBUG
3441         kprintf("alloc ifa %p %d\n", ifa, size);
3442 #endif
3443         return ifa;
3444 }
3445
3446 void
3447 ifac_free(struct ifaddr_container *ifac, int cpu_id)
3448 {
3449         struct ifaddr *ifa = ifac->ifa;
3450
3451         KKASSERT(ifac->ifa_magic == IFA_CONTAINER_MAGIC);
3452         KKASSERT(ifac->ifa_refcnt == 0);
3453         KASSERT(ifac->ifa_listmask == 0,
3454                 ("ifa is still on %#x lists", ifac->ifa_listmask));
3455
3456         ifac->ifa_magic = IFA_CONTAINER_DEAD;
3457
3458 #ifdef IFADDR_DEBUG_VERBOSE
3459         kprintf("try free ifa %p cpu_id %d\n", ifac->ifa, cpu_id);
3460 #endif
3461
3462         KASSERT(ifa->ifa_ncnt > 0 && ifa->ifa_ncnt <= ncpus,
3463                 ("invalid # of ifac, %d", ifa->ifa_ncnt));
3464         if (atomic_fetchadd_int(&ifa->ifa_ncnt, -1) == 1) {
3465 #ifdef IFADDR_DEBUG
3466                 kprintf("free ifa %p\n", ifa);
3467 #endif
3468                 kfree(ifa->ifa_containers, M_IFADDR);
3469                 kfree(ifa, M_IFADDR);
3470         }
3471 }
3472
3473 static void
3474 ifa_iflink_dispatch(netmsg_t nmsg)
3475 {
3476         struct netmsg_ifaddr *msg = (struct netmsg_ifaddr *)nmsg;
3477         struct ifaddr *ifa = msg->ifa;
3478         struct ifnet *ifp = msg->ifp;
3479         int cpu = mycpuid;
3480         struct ifaddr_container *ifac;
3481
3482         crit_enter();
3483
3484         ifac = &ifa->ifa_containers[cpu];
3485         ASSERT_IFAC_VALID(ifac);
3486         KASSERT((ifac->ifa_listmask & IFA_LIST_IFADDRHEAD) == 0,
3487                 ("ifaddr is on if_addrheads"));
3488
3489         ifac->ifa_listmask |= IFA_LIST_IFADDRHEAD;
3490         if (msg->tail)
3491                 TAILQ_INSERT_TAIL(&ifp->if_addrheads[cpu], ifac, ifa_link);
3492         else
3493                 TAILQ_INSERT_HEAD(&ifp->if_addrheads[cpu], ifac, ifa_link);
3494
3495         crit_exit();
3496
3497         netisr_forwardmsg_all(&nmsg->base, cpu + 1);
3498 }
3499
3500 void
3501 ifa_iflink(struct ifaddr *ifa, struct ifnet *ifp, int tail)
3502 {
3503         struct netmsg_ifaddr msg;
3504
3505         netmsg_init(&msg.base, NULL, &curthread->td_msgport,
3506                     0, ifa_iflink_dispatch);
3507         msg.ifa = ifa;
3508         msg.ifp = ifp;
3509         msg.tail = tail;
3510
3511         netisr_domsg(&msg.base, 0);
3512 }
3513
3514 static void
3515 ifa_ifunlink_dispatch(netmsg_t nmsg)
3516 {
3517         struct netmsg_ifaddr *msg = (struct netmsg_ifaddr *)nmsg;
3518         struct ifaddr *ifa = msg->ifa;
3519         struct ifnet *ifp = msg->ifp;
3520         int cpu = mycpuid;
3521         struct ifaddr_container *ifac;
3522
3523         crit_enter();
3524
3525         ifac = &ifa->ifa_containers[cpu];
3526         ASSERT_IFAC_VALID(ifac);
3527         KASSERT(ifac->ifa_listmask & IFA_LIST_IFADDRHEAD,
3528                 ("ifaddr is not on if_addrhead"));
3529
3530         TAILQ_REMOVE(&ifp->if_addrheads[cpu], ifac, ifa_link);
3531         ifac->ifa_listmask &= ~IFA_LIST_IFADDRHEAD;
3532
3533         crit_exit();
3534
3535         netisr_forwardmsg_all(&nmsg->base, cpu + 1);
3536 }
3537
3538 void
3539 ifa_ifunlink(struct ifaddr *ifa, struct ifnet *ifp)
3540 {
3541         struct netmsg_ifaddr msg;
3542
3543         netmsg_init(&msg.base, NULL, &curthread->td_msgport,
3544                     0, ifa_ifunlink_dispatch);
3545         msg.ifa = ifa;
3546         msg.ifp = ifp;
3547
3548         netisr_domsg(&msg.base, 0);
3549 }
3550
3551 static void
3552 ifa_destroy_dispatch(netmsg_t nmsg)
3553 {
3554         struct netmsg_ifaddr *msg = (struct netmsg_ifaddr *)nmsg;
3555
3556         IFAFREE(msg->ifa);
3557         netisr_forwardmsg_all(&nmsg->base, mycpuid + 1);
3558 }
3559
3560 void
3561 ifa_destroy(struct ifaddr *ifa)
3562 {
3563         struct netmsg_ifaddr msg;
3564
3565         netmsg_init(&msg.base, NULL, &curthread->td_msgport,
3566                     0, ifa_destroy_dispatch);
3567         msg.ifa = ifa;
3568
3569         netisr_domsg(&msg.base, 0);
3570 }
3571
3572 static void
3573 if_start_rollup(void)
3574 {
3575         struct ifsubq_stage_head *head = &ifsubq_stage_heads[mycpuid];
3576         struct ifsubq_stage *stage;
3577
3578         crit_enter();
3579
3580         while ((stage = TAILQ_FIRST(&head->stg_head)) != NULL) {
3581                 struct ifaltq_subque *ifsq = stage->stg_subq;
3582                 int is_sched = 0;
3583
3584                 if (stage->stg_flags & IFSQ_STAGE_FLAG_SCHED)
3585                         is_sched = 1;
3586                 ifsq_stage_remove(head, stage);
3587
3588                 if (is_sched) {
3589                         ifsq_ifstart_schedule(ifsq, 1);
3590                 } else {
3591                         int start = 0;
3592
3593                         ALTQ_SQ_LOCK(ifsq);
3594                         if (!ifsq_is_started(ifsq)) {
3595                                 /*
3596                                  * Hold the subqueue interlock of
3597                                  * ifnet.if_start
3598                                  */
3599                                 ifsq_set_started(ifsq);
3600                                 start = 1;
3601                         }
3602                         ALTQ_SQ_UNLOCK(ifsq);
3603
3604                         if (start)
3605                                 ifsq_ifstart_try(ifsq, 1);
3606                 }
3607                 KKASSERT((stage->stg_flags &
3608                     (IFSQ_STAGE_FLAG_QUED | IFSQ_STAGE_FLAG_SCHED)) == 0);
3609         }
3610
3611         crit_exit();
3612 }
3613
3614 static void
3615 ifnetinit(void *dummy __unused)
3616 {
3617         int i;
3618
3619         /* XXX netisr_ncpus */
3620         for (i = 0; i < ncpus; ++i)
3621                 TAILQ_INIT(&ifsubq_stage_heads[i].stg_head);
3622         netisr_register_rollup(if_start_rollup, NETISR_ROLLUP_PRIO_IFSTART);
3623 }
3624
3625 void
3626 if_register_com_alloc(u_char type,
3627     if_com_alloc_t *a, if_com_free_t *f)
3628 {
3629
3630         KASSERT(if_com_alloc[type] == NULL,
3631             ("if_register_com_alloc: %d already registered", type));
3632         KASSERT(if_com_free[type] == NULL,
3633             ("if_register_com_alloc: %d free already registered", type));
3634
3635         if_com_alloc[type] = a;
3636         if_com_free[type] = f;
3637 }
3638
3639 void
3640 if_deregister_com_alloc(u_char type)
3641 {
3642
3643         KASSERT(if_com_alloc[type] != NULL,
3644             ("if_deregister_com_alloc: %d not registered", type));
3645         KASSERT(if_com_free[type] != NULL,
3646             ("if_deregister_com_alloc: %d free not registered", type));
3647         if_com_alloc[type] = NULL;
3648         if_com_free[type] = NULL;
3649 }
3650
3651 void
3652 ifq_set_maxlen(struct ifaltq *ifq, int len)
3653 {
3654         ifq->altq_maxlen = len + (ncpus * ifsq_stage_cntmax);
3655 }
3656
3657 int
3658 ifq_mapsubq_default(struct ifaltq *ifq __unused, int cpuid __unused)
3659 {
3660         return ALTQ_SUBQ_INDEX_DEFAULT;
3661 }
3662
3663 int
3664 ifq_mapsubq_modulo(struct ifaltq *ifq, int cpuid)
3665 {
3666
3667         return (cpuid % ifq->altq_subq_mappriv);
3668 }
3669
3670 /*
3671  * Watchdog timeout.  Process callback as appropriate.  If we cannot
3672  * serialize the ifnet just try again on the next timeout.
3673  *
3674  * NOTE: The ifnet can adjust wd_timer while holding the serializer.  We
3675  *       can only safely adjust it under the same circumstances.
3676  */
3677 static void
3678 ifsq_watchdog(void *arg)
3679 {
3680         struct ifsubq_watchdog *wd = arg;
3681         struct ifnet *ifp;
3682         int count;
3683
3684         /*
3685          * Fast track.  Try to avoid acquiring the serializer when not
3686          * near the terminal count, unless asked to.  If the atomic op
3687          * to decrement the count fails just retry on the next callout.
3688          */
3689         count = wd->wd_timer;
3690         cpu_ccfence();
3691         if (count == 0)
3692                 goto done;
3693         if (count > 2 && (wd->wd_flags & IF_WDOG_ALLTICKS) == 0) {
3694                 (void)atomic_cmpset_int(&wd->wd_timer, count, count - 1);
3695                 goto done;
3696         }
3697
3698         /*
3699          * Obtain the serializer and then re-test all wd_timer conditions
3700          * as it may have changed.  NICs do not mess with wd_timer without
3701          * holding the serializer.
3702          *
3703          * If we are unable to obtain the serializer just retry the same
3704          * count on the next callout.
3705          *
3706          * - call watchdog in terminal count (0)
3707          * - call watchdog on last tick (1) if requested
3708          * - call watchdog on all ticks if requested
3709          */
3710         ifp = ifsq_get_ifp(wd->wd_subq);
3711         if (ifnet_tryserialize_all(ifp) == 0)
3712                 goto done;
3713         if (atomic_cmpset_int(&wd->wd_timer, count, count - 1)) {
3714                 --count;
3715                 if (count == 0 ||
3716                     (wd->wd_flags & IF_WDOG_ALLTICKS) ||
3717                     ((wd->wd_flags & IF_WDOG_LASTTICK) && count == 1)) {
3718                         wd->wd_watchdog(wd->wd_subq);
3719                 }
3720         }
3721         ifnet_deserialize_all(ifp);
3722 done:
3723         ifsq_watchdog_reset(wd);
3724 }
3725
3726 static void
3727 ifsq_watchdog_reset(struct ifsubq_watchdog *wd)
3728 {
3729         callout_reset_bycpu(&wd->wd_callout, hz, ifsq_watchdog, wd,
3730             ifsq_get_cpuid(wd->wd_subq));
3731 }
3732
3733 void
3734 ifsq_watchdog_init(struct ifsubq_watchdog *wd, struct ifaltq_subque *ifsq,
3735                    ifsq_watchdog_t watchdog, int flags)
3736 {
3737         callout_init_mp(&wd->wd_callout);
3738         wd->wd_timer = 0;
3739         wd->wd_flags = flags;
3740         wd->wd_subq = ifsq;
3741         wd->wd_watchdog = watchdog;
3742 }
3743
3744 void
3745 ifsq_watchdog_start(struct ifsubq_watchdog *wd)
3746 {
3747         atomic_swap_int(&wd->wd_timer, 0);
3748         ifsq_watchdog_reset(wd);
3749 }
3750
3751 void
3752 ifsq_watchdog_stop(struct ifsubq_watchdog *wd)
3753 {
3754         atomic_swap_int(&wd->wd_timer, 0);
3755         callout_stop(&wd->wd_callout);
3756 }
3757
3758 void
3759 ifsq_watchdog_set_count(struct ifsubq_watchdog *wd, int count)
3760 {
3761         atomic_swap_int(&wd->wd_timer, count);
3762 }
3763
3764 void
3765 ifnet_lock(void)
3766 {
3767         KASSERT(curthread->td_type != TD_TYPE_NETISR,
3768             ("try holding ifnet lock in netisr"));
3769         mtx_lock(&ifnet_mtx);
3770 }
3771
3772 void
3773 ifnet_unlock(void)
3774 {
3775         KASSERT(curthread->td_type != TD_TYPE_NETISR,
3776             ("try holding ifnet lock in netisr"));
3777         mtx_unlock(&ifnet_mtx);
3778 }
3779
3780 static struct ifnet_array *
3781 ifnet_array_alloc(int count)
3782 {
3783         struct ifnet_array *arr;
3784
3785         arr = kmalloc(__offsetof(struct ifnet_array, ifnet_arr[count]),
3786             M_IFNET, M_WAITOK);
3787         arr->ifnet_count = count;
3788
3789         return arr;
3790 }
3791
3792 static void
3793 ifnet_array_free(struct ifnet_array *arr)
3794 {
3795         if (arr == &ifnet_array0)
3796                 return;
3797         kfree(arr, M_IFNET);
3798 }
3799
3800 static struct ifnet_array *
3801 ifnet_array_add(struct ifnet *ifp, const struct ifnet_array *old_arr)
3802 {
3803         struct ifnet_array *arr;
3804         int count, i;
3805
3806         KASSERT(old_arr->ifnet_count >= 0,
3807             ("invalid ifnet array count %d", old_arr->ifnet_count));
3808         count = old_arr->ifnet_count + 1;
3809         arr = ifnet_array_alloc(count);
3810
3811         /*
3812          * Save the old ifnet array and append this ifp to the end of
3813          * the new ifnet array.
3814          */
3815         for (i = 0; i < old_arr->ifnet_count; ++i) {
3816                 KASSERT(old_arr->ifnet_arr[i] != ifp,
3817                     ("%s is already in ifnet array", ifp->if_xname));
3818                 arr->ifnet_arr[i] = old_arr->ifnet_arr[i];
3819         }
3820         KASSERT(i == count - 1,
3821             ("add %s, ifnet array index mismatch, should be %d, but got %d",
3822              ifp->if_xname, count - 1, i));
3823         arr->ifnet_arr[i] = ifp;
3824
3825         return arr;
3826 }
3827
3828 static struct ifnet_array *
3829 ifnet_array_del(struct ifnet *ifp, const struct ifnet_array *old_arr)
3830 {
3831         struct ifnet_array *arr;
3832         int count, i, idx, found = 0;
3833
3834         KASSERT(old_arr->ifnet_count > 0,
3835             ("invalid ifnet array count %d", old_arr->ifnet_count));
3836         count = old_arr->ifnet_count - 1;
3837         arr = ifnet_array_alloc(count);
3838
3839         /*
3840          * Save the old ifnet array, but skip this ifp.
3841          */
3842         idx = 0;
3843         for (i = 0; i < old_arr->ifnet_count; ++i) {
3844                 if (old_arr->ifnet_arr[i] == ifp) {
3845                         KASSERT(!found,
3846                             ("dup %s is in ifnet array", ifp->if_xname));
3847                         found = 1;
3848                         continue;
3849                 }
3850                 KASSERT(idx < count,
3851                     ("invalid ifnet array index %d, count %d", idx, count));
3852                 arr->ifnet_arr[idx] = old_arr->ifnet_arr[i];
3853                 ++idx;
3854         }
3855         KASSERT(found, ("%s is not in ifnet array", ifp->if_xname));
3856         KASSERT(idx == count,
3857             ("del %s, ifnet array count mismatch, should be %d, but got %d ",
3858              ifp->if_xname, count, idx));
3859
3860         return arr;
3861 }
3862
3863 const struct ifnet_array *
3864 ifnet_array_get(void)
3865 {
3866         const struct ifnet_array *ret;
3867
3868         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
3869         ret = ifnet_array;
3870         /* Make sure 'ret' is really used. */
3871         cpu_ccfence();
3872         return (ret);
3873 }
3874
3875 int
3876 ifnet_array_isempty(void)
3877 {
3878         KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
3879         if (ifnet_array->ifnet_count == 0)
3880                 return 1;
3881         else
3882                 return 0;
3883 }
3884
3885 void
3886 ifa_marker_init(struct ifaddr_marker *mark, struct ifnet *ifp)
3887 {
3888         struct ifaddr *ifa;
3889
3890         memset(mark, 0, sizeof(*mark));
3891         ifa = &mark->ifa;
3892
3893         mark->ifac.ifa = ifa;
3894
3895         ifa->ifa_addr = &mark->addr;
3896         ifa->ifa_dstaddr = &mark->dstaddr;
3897         ifa->ifa_netmask = &mark->netmask;
3898         ifa->ifa_ifp = ifp;
3899 }
3900
3901 static int
3902 if_ringcnt_fixup(int ring_cnt, int ring_cntmax)
3903 {
3904
3905         KASSERT(ring_cntmax > 0, ("invalid ring count max %d", ring_cntmax));
3906
3907         if (ring_cnt <= 0 || ring_cnt > ring_cntmax)
3908                 ring_cnt = ring_cntmax;
3909         if (ring_cnt > netisr_ncpus)
3910                 ring_cnt = netisr_ncpus;
3911         return (ring_cnt);
3912 }
3913
3914 static void
3915 if_ringmap_set_grid(device_t dev, struct if_ringmap *rm, int grid)
3916 {
3917         int i, offset;
3918
3919         KASSERT(grid > 0, ("invalid if_ringmap grid %d", grid));
3920         KASSERT(grid >= rm->rm_cnt, ("invalid if_ringmap grid %d, count %d",
3921             grid, rm->rm_cnt));
3922         rm->rm_grid = grid;
3923
3924         offset = (rm->rm_grid * device_get_unit(dev)) % netisr_ncpus;
3925         for (i = 0; i < rm->rm_cnt; ++i) {
3926                 rm->rm_cpumap[i] = offset + i;
3927                 KASSERT(rm->rm_cpumap[i] < netisr_ncpus,
3928                     ("invalid cpumap[%d] = %d, offset %d", i,
3929                      rm->rm_cpumap[i], offset));
3930         }
3931 }
3932
3933 static struct if_ringmap *
3934 if_ringmap_alloc_flags(device_t dev, int ring_cnt, int ring_cntmax,
3935     uint32_t flags)
3936 {
3937         struct if_ringmap *rm;
3938         int i, grid = 0, prev_grid;
3939
3940         ring_cnt = if_ringcnt_fixup(ring_cnt, ring_cntmax);
3941         rm = kmalloc(__offsetof(struct if_ringmap, rm_cpumap[ring_cnt]),
3942             M_DEVBUF, M_WAITOK | M_ZERO);
3943
3944         rm->rm_cnt = ring_cnt;
3945         if (flags & RINGMAP_FLAG_POWEROF2)
3946                 rm->rm_cnt = 1 << (fls(rm->rm_cnt) - 1);
3947
3948         prev_grid = netisr_ncpus;
3949         for (i = 0; i < netisr_ncpus; ++i) {
3950                 if (netisr_ncpus % (i + 1) != 0)
3951                         continue;
3952
3953                 grid = netisr_ncpus / (i + 1);
3954                 if (rm->rm_cnt > grid) {
3955                         grid = prev_grid;
3956                         break;
3957                 }
3958
3959                 if (rm->rm_cnt > netisr_ncpus / (i + 2))
3960                         break;
3961                 prev_grid = grid;
3962         }
3963         if_ringmap_set_grid(dev, rm, grid);
3964
3965         return (rm);
3966 }
3967
3968 struct if_ringmap *
3969 if_ringmap_alloc(device_t dev, int ring_cnt, int ring_cntmax)
3970 {
3971
3972         return (if_ringmap_alloc_flags(dev, ring_cnt, ring_cntmax,
3973             RINGMAP_FLAG_NONE));
3974 }
3975
3976 struct if_ringmap *
3977 if_ringmap_alloc2(device_t dev, int ring_cnt, int ring_cntmax)
3978 {
3979
3980         return (if_ringmap_alloc_flags(dev, ring_cnt, ring_cntmax,
3981             RINGMAP_FLAG_POWEROF2));
3982 }
3983
3984 void
3985 if_ringmap_free(struct if_ringmap *rm)
3986 {
3987
3988         kfree(rm, M_DEVBUF);
3989 }
3990
3991 /*
3992  * Align the two ringmaps.
3993  *
3994  * e.g. 8 netisrs, rm0 contains 4 rings, rm1 contains 2 rings.
3995  *
3996  * Before:
3997  *
3998  * CPU      0  1  2  3   4  5  6  7
3999  * NIC_RX               n0 n1 n2 n3
4000  * NIC_TX        N0 N1
4001  *
4002  * After:
4003  *
4004  * CPU      0  1  2  3   4  5  6  7
4005  * NIC_RX               n0 n1 n2 n3
4006  * NIC_TX               N0 N1
4007  */
4008 void
4009 if_ringmap_align(device_t dev, struct if_ringmap *rm0, struct if_ringmap *rm1)
4010 {
4011
4012         if (rm0->rm_grid > rm1->rm_grid)
4013                 if_ringmap_set_grid(dev, rm1, rm0->rm_grid);
4014         else if (rm0->rm_grid < rm1->rm_grid)
4015                 if_ringmap_set_grid(dev, rm0, rm1->rm_grid);
4016 }
4017
4018 void
4019 if_ringmap_match(device_t dev, struct if_ringmap *rm0, struct if_ringmap *rm1)
4020 {
4021         int subset_grid, cnt, divisor, mod, offset, i;
4022         struct if_ringmap *subset_rm, *rm;
4023         int old_rm0_grid, old_rm1_grid;
4024
4025         if (rm0->rm_grid == rm1->rm_grid)
4026                 return;
4027
4028         /* Save grid for later use */
4029         old_rm0_grid = rm0->rm_grid;
4030         old_rm1_grid = rm1->rm_grid;
4031
4032         if_ringmap_align(dev, rm0, rm1);
4033
4034         /*
4035          * Re-shuffle rings to get more even distribution.
4036          *
4037          * e.g. 12 netisrs, rm0 contains 4 rings, rm1 contains 2 rings.
4038          *
4039          * CPU       0  1  2  3   4  5  6  7   8  9 10 11
4040          *
4041          * NIC_RX   a0 a1 a2 a3  b0 b1 b2 b3  c0 c1 c2 c3
4042          * NIC_TX   A0 A1        B0 B1        C0 C1
4043          *
4044          * NIC_RX   d0 d1 d2 d3  e0 e1 e2 e3  f0 f1 f2 f3
4045          * NIC_TX         D0 D1        E0 E1        F0 F1
4046          */
4047
4048         if (rm0->rm_cnt >= (2 * old_rm1_grid)) {
4049                 cnt = rm0->rm_cnt;
4050                 subset_grid = old_rm1_grid;
4051                 subset_rm = rm1;
4052                 rm = rm0;
4053         } else if (rm1->rm_cnt > (2 * old_rm0_grid)) {
4054                 cnt = rm1->rm_cnt;
4055                 subset_grid = old_rm0_grid;
4056                 subset_rm = rm0;
4057                 rm = rm1;
4058         } else {
4059                 /* No space to shuffle. */
4060                 return;
4061         }
4062
4063         mod = cnt / subset_grid;
4064         KKASSERT(mod >= 2);
4065         divisor = netisr_ncpus / rm->rm_grid;
4066         offset = ((device_get_unit(dev) / divisor) % mod) * subset_grid;
4067
4068         for (i = 0; i < subset_rm->rm_cnt; ++i) {
4069                 subset_rm->rm_cpumap[i] += offset;
4070                 KASSERT(subset_rm->rm_cpumap[i] < netisr_ncpus,
4071                     ("match: invalid cpumap[%d] = %d, offset %d",
4072                      i, subset_rm->rm_cpumap[i], offset));
4073         }
4074 #ifdef INVARIANTS
4075         for (i = 0; i < subset_rm->rm_cnt; ++i) {
4076                 int j;
4077
4078                 for (j = 0; j < rm->rm_cnt; ++j) {
4079                         if (rm->rm_cpumap[j] == subset_rm->rm_cpumap[i])
4080                                 break;
4081                 }
4082                 KASSERT(j < rm->rm_cnt,
4083                     ("subset cpumap[%d] = %d not found in superset",
4084                      i, subset_rm->rm_cpumap[i]));
4085         }
4086 #endif
4087 }
4088
4089 int
4090 if_ringmap_count(const struct if_ringmap *rm)
4091 {
4092
4093         return (rm->rm_cnt);
4094 }
4095
4096 int
4097 if_ringmap_cpumap(const struct if_ringmap *rm, int ring)
4098 {
4099
4100         KASSERT(ring >= 0 && ring < rm->rm_cnt, ("invalid ring %d", ring));
4101         return (rm->rm_cpumap[ring]);
4102 }
4103
4104 void
4105 if_ringmap_rdrtable(const struct if_ringmap *rm, int table[], int table_nent)
4106 {
4107         int i, grid_idx, grid_cnt, patch_off, patch_cnt, ncopy;
4108
4109         KASSERT(table_nent > 0 && (table_nent & NETISR_CPUMASK) == 0,
4110             ("invalid redirect table entries %d", table_nent));
4111
4112         grid_idx = 0;
4113         for (i = 0; i < NETISR_CPUMAX; ++i) {
4114                 table[i] = grid_idx++ % rm->rm_cnt;
4115
4116                 if (grid_idx == rm->rm_grid)
4117                         grid_idx = 0;
4118         }
4119
4120         /*
4121          * Make the ring distributed more evenly for the remainder
4122          * of each grid.
4123          *
4124          * e.g. 12 netisrs, rm contains 8 rings.
4125          *
4126          * Redirect table before:
4127          *
4128          *  0  1  2  3  4  5  6  7  0  1  2  3  0  1  2  3
4129          *  4  5  6  7  0  1  2  3  0  1  2  3  4  5  6  7
4130          *  0  1  2  3  0  1  2  3  4  5  6  7  0  1  2  3
4131          *  ....
4132          *
4133          * Redirect table after being patched (pX, patched entries):
4134          *
4135          *  0  1  2  3  4  5  6  7 p0 p1 p2 p3  0  1  2  3
4136          *  4  5  6  7 p4 p5 p6 p7  0  1  2  3  4  5  6  7
4137          * p0 p1 p2 p3  0  1  2  3  4  5  6  7 p4 p5 p6 p7
4138          *  ....
4139          */
4140         patch_cnt = rm->rm_grid % rm->rm_cnt;
4141         if (patch_cnt == 0)
4142                 goto done;
4143         patch_off = rm->rm_grid - (rm->rm_grid % rm->rm_cnt);
4144
4145         grid_cnt = roundup(NETISR_CPUMAX, rm->rm_grid) / rm->rm_grid;
4146         grid_idx = 0;
4147         for (i = 0; i < grid_cnt; ++i) {
4148                 int j;
4149
4150                 for (j = 0; j < patch_cnt; ++j) {
4151                         int fix_idx;
4152
4153                         fix_idx = (i * rm->rm_grid) + patch_off + j;
4154                         if (fix_idx >= NETISR_CPUMAX)
4155                                 goto done;
4156                         table[fix_idx] = grid_idx++ % rm->rm_cnt;
4157                 }
4158         }
4159 done:
4160         /*
4161          * If the device supports larger redirect table, duplicate
4162          * the first NETISR_CPUMAX entries to the rest of the table,
4163          * so that it matches upper layer's expectation:
4164          * (hash & NETISR_CPUMASK) % netisr_ncpus
4165          */
4166         ncopy = table_nent / NETISR_CPUMAX;
4167         for (i = 1; i < ncopy; ++i) {
4168                 memcpy(&table[i * NETISR_CPUMAX], table,
4169                     NETISR_CPUMAX * sizeof(table[0]));
4170         }
4171         if (if_ringmap_dumprdr) {
4172                 for (i = 0; i < table_nent; ++i) {
4173                         if (i != 0 && i % 16 == 0)
4174                                 kprintf("\n");
4175                         kprintf("%03d ", table[i]);
4176                 }
4177                 kprintf("\n");
4178         }
4179 }
4180
4181 int
4182 if_ringmap_cpumap_sysctl(SYSCTL_HANDLER_ARGS)
4183 {
4184         struct if_ringmap *rm = arg1;
4185         int i, error = 0;
4186
4187         for (i = 0; i < rm->rm_cnt; ++i) {
4188                 int cpu = rm->rm_cpumap[i];
4189
4190                 error = SYSCTL_OUT(req, &cpu, sizeof(cpu));
4191                 if (error)
4192                         break;
4193         }
4194         return (error);
4195 }