sys/dev/virtual/virtio/net/if_vtnet.c

   1 /*-
   2  * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice unmodified, this list of conditions, and the following
  10  *    disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 /* Driver for VirtIO network devices. */
  28
  29 #include <sys/cdefs.h>
  30
  31 #include <sys/param.h>
  32 #include <sys/systm.h>
  33 #include <sys/kernel.h>
  34 #include <sys/sockio.h>
  35 #include <sys/mbuf.h>
  36 #include <sys/malloc.h>
  37 #include <sys/module.h>
  38 #include <sys/socket.h>
  39 #include <sys/sysctl.h>
  40 #include <sys/taskqueue.h>
  41 #include <sys/random.h>
  42 #include <sys/sglist.h>
  43 #include <sys/serialize.h>
  44 #include <sys/bus.h>
  45 #include <sys/rman.h>
  46
  47 #include <net/ethernet.h>
  48 #include <net/if.h>
  49 #include <net/if_arp.h>
  50 #include <net/if_dl.h>
  51 #include <net/if_types.h>
  52 #include <net/if_media.h>
  53 #include <net/vlan/if_vlan_var.h>
  54 #include <net/vlan/if_vlan_ether.h>
  55 #include <net/ifq_var.h>
  56
  57 #include <net/bpf.h>
  58
  59 #include <netinet/in_systm.h>
  60 #include <netinet/in.h>
  61 #include <netinet/ip.h>
  62 #include <netinet/ip6.h>
  63 #include <netinet/udp.h>
  64 #include <netinet/tcp.h>
  65 #include <netinet/sctp.h>
  66
  67 #include <dev/virtual/virtio/virtio/virtio.h>
  68 #include <dev/virtual/virtio/virtio/virtqueue.h>
  69
  70 #include "virtio_net.h"
  71 #include "virtio_if.h"
  72
  73 struct vtnet_statistics {
  74         unsigned long           mbuf_alloc_failed;
  75
  76         unsigned long           rx_frame_too_large;
  77         unsigned long           rx_enq_replacement_failed;
  78         unsigned long           rx_mergeable_failed;
  79         unsigned long           rx_csum_bad_ethtype;
  80         unsigned long           rx_csum_bad_start;
  81         unsigned long           rx_csum_bad_ipproto;
  82         unsigned long           rx_csum_bad_offset;
  83         unsigned long           rx_csum_failed;
  84         unsigned long           rx_csum_offloaded;
  85         unsigned long           rx_task_rescheduled;
  86
  87         unsigned long           tx_csum_offloaded;
  88         unsigned long           tx_tso_offloaded;
  89         unsigned long           tx_csum_bad_ethtype;
  90         unsigned long           tx_tso_bad_ethtype;
  91         unsigned long           tx_task_rescheduled;
  92 };
  93
  94 struct vtnet_softc {
  95         device_t                vtnet_dev;
  96         struct ifnet            *vtnet_ifp;
  97         struct lwkt_serialize   vtnet_slz;
  98
  99         uint32_t                vtnet_flags;
 100 #define VTNET_FLAG_LINK         0x0001
 101 #define VTNET_FLAG_SUSPENDED    0x0002
 102 #define VTNET_FLAG_CTRL_VQ      0x0004
 103 #define VTNET_FLAG_CTRL_RX      0x0008
 104 #define VTNET_FLAG_VLAN_FILTER  0x0010
 105 #define VTNET_FLAG_TSO_ECN      0x0020
 106 #define VTNET_FLAG_MRG_RXBUFS   0x0040
 107 #define VTNET_FLAG_LRO_NOMRG    0x0080
 108
 109         struct virtqueue        *vtnet_rx_vq;
 110         struct virtqueue        *vtnet_tx_vq;
 111         struct virtqueue        *vtnet_ctrl_vq;
 112
 113         struct vtnet_tx_header  *vtnet_txhdrarea;
 114         uint32_t                vtnet_txhdridx;
 115         struct vtnet_mac_filter *vtnet_macfilter;
 116
 117         int                     vtnet_hdr_size;
 118         int                     vtnet_tx_size;
 119         int                     vtnet_rx_size;
 120         int                     vtnet_rx_process_limit;
 121         int                     vtnet_rx_mbuf_size;
 122         int                     vtnet_rx_mbuf_count;
 123         int                     vtnet_if_flags;
 124         int                     vtnet_watchdog_timer;
 125         uint64_t                vtnet_features;
 126
 127         struct task             vtnet_cfgchg_task;
 128
 129         struct vtnet_statistics vtnet_stats;
 130
 131         struct sysctl_ctx_list  vtnet_sysctl_ctx;
 132         struct sysctl_oid       *vtnet_sysctl_tree;
 133
 134         struct callout          vtnet_tick_ch;
 135
 136         eventhandler_tag        vtnet_vlan_attach;
 137         eventhandler_tag        vtnet_vlan_detach;
 138
 139         struct ifmedia          vtnet_media;
 140         /*
 141          * Fake media type; the host does not provide us with
 142          * any real media information.
 143          */
 144 #define VTNET_MEDIATYPE         (IFM_ETHER | IFM_1000_T | IFM_FDX)
 145         char                    vtnet_hwaddr[ETHER_ADDR_LEN];
 146
 147         /*
 148          * During reset, the host's VLAN filtering table is lost. The
 149          * array below is used to restore all the VLANs configured on
 150          * this interface after a reset.
 151          */
 152 #define VTNET_VLAN_SHADOW_SIZE  (4096 / 32)
 153         int                     vtnet_nvlans;
 154         uint32_t                vtnet_vlan_shadow[VTNET_VLAN_SHADOW_SIZE];
 155
 156         char                    vtnet_mtx_name[16];
 157 };
 158
 159 /*
 160  * When mergeable buffers are not negotiated, the vtnet_rx_header structure
 161  * below is placed at the beginning of the mbuf data. Use 4 bytes of pad to
 162  * both keep the VirtIO header and the data non-contiguous and to keep the
 163  * frame's payload 4 byte aligned.
 164  *
 165  * When mergeable buffers are negotiated, the host puts the VirtIO header in
 166  * the beginning of the first mbuf's data.
 167  */
 168 #define VTNET_RX_HEADER_PAD     4
 169 struct vtnet_rx_header {
 170         struct virtio_net_hdr   vrh_hdr;
 171         char                    vrh_pad[VTNET_RX_HEADER_PAD];
 172 } __packed;
 173
 174 /*
 175  * For each outgoing frame, the vtnet_tx_header below is allocated from
 176  * the vtnet_tx_header_zone.
 177  */
 178 struct vtnet_tx_header {
 179         union {
 180                 struct virtio_net_hdr           hdr;
 181                 struct virtio_net_hdr_mrg_rxbuf mhdr;
 182         } vth_uhdr;
 183
 184         struct mbuf             *vth_mbuf;
 185 };
 186
 187 MALLOC_DEFINE(M_VTNET, "VTNET_TX", "Outgoing VTNET TX frame header");
 188
 189 /*
 190  * The VirtIO specification does not place a limit on the number of MAC
 191  * addresses the guest driver may request to be filtered. In practice,
 192  * the host is constrained by available resources. To simplify this driver,
 193  * impose a reasonably high limit of MAC addresses we will filter before
 194  * falling back to promiscuous or all-multicast modes.
 195  */
 196 #define VTNET_MAX_MAC_ENTRIES   128
 197
 198 struct vtnet_mac_table {
 199         uint32_t                nentries;
 200         uint8_t                 macs[VTNET_MAX_MAC_ENTRIES][ETHER_ADDR_LEN];
 201 } __packed;
 202
 203 struct vtnet_mac_filter {
 204         struct vtnet_mac_table  vmf_unicast;
 205         uint32_t                vmf_pad; /* Make tables non-contiguous. */
 206         struct vtnet_mac_table  vmf_multicast;
 207 };
 208
 209 #define VTNET_WATCHDOG_TIMEOUT  5
 210 #define VTNET_CSUM_OFFLOAD      (CSUM_TCP | CSUM_UDP)// | CSUM_SCTP)
 211
 212 /* Features desired/implemented by this driver. */
 213 #define VTNET_FEATURES          \
 214     (VIRTIO_NET_F_MAC           | \
 215      VIRTIO_NET_F_STATUS        | \
 216      VIRTIO_NET_F_CTRL_VQ       | \
 217      VIRTIO_NET_F_CTRL_RX       | \
 218      VIRTIO_NET_F_CTRL_VLAN     | \
 219      VIRTIO_NET_F_CSUM          | \
 220      VIRTIO_NET_F_HOST_TSO4     | \
 221      VIRTIO_NET_F_HOST_TSO6     | \
 222      VIRTIO_NET_F_HOST_ECN      | \
 223      VIRTIO_NET_F_GUEST_CSUM    | \
 224      VIRTIO_NET_F_GUEST_TSO4    | \
 225      VIRTIO_NET_F_GUEST_TSO6    | \
 226      VIRTIO_NET_F_GUEST_ECN     | \
 227      VIRTIO_NET_F_MRG_RXBUF)
 228
 229 /*
 230  * The VIRTIO_NET_F_GUEST_TSO[46] features permit the host to send us
 231  * frames larger than 1514 bytes. We do not yet support software LRO
 232  * via tcp_lro_rx().
 233  */
 234 #define VTNET_LRO_FEATURES (VIRTIO_NET_F_GUEST_TSO4 | \
 235                             VIRTIO_NET_F_GUEST_TSO6 | VIRTIO_NET_F_GUEST_ECN)
 236
 237 #define VTNET_MAX_MTU           65536
 238 #define VTNET_MAX_RX_SIZE       65550
 239
 240 /*
 241  * Used to preallocate the Vq indirect descriptors. The first segment
 242  * is reserved for the header.
 243  */
 244 #define VTNET_MIN_RX_SEGS       2
 245 #define VTNET_MAX_RX_SEGS       34
 246 #define VTNET_MAX_TX_SEGS       34
 247
 248 #ifndef CSUM_TSO
 249 #define CSUM_TSO                0
 250 #endif
 251
 252 #define IFCAP_TSO4              0x00100 /* can do TCP Segmentation Offload */
 253 #define IFCAP_TSO6              0x00200 /* can do TCP6 Segmentation Offload */
 254 #define IFCAP_LRO               0x00400 /* can do Large Receive Offload */
 255 #define IFCAP_VLAN_HWFILTER     0x10000 /* interface hw can filter vlan tag */
 256 #define IFCAP_VLAN_HWTSO        0x40000 /* can do IFCAP_TSO on VLANs */
 257
 258
 259 /*
 260  * Assert we can receive and transmit the maximum with regular
 261  * size clusters.
 262  */
 263 CTASSERT(((VTNET_MAX_RX_SEGS - 1) * MCLBYTES) >= VTNET_MAX_RX_SIZE);
 264 CTASSERT(((VTNET_MAX_TX_SEGS - 1) * MCLBYTES) >= VTNET_MAX_MTU);
 265
 266 /*
 267  * Determine how many mbufs are in each receive buffer. For LRO without
 268  * mergeable descriptors, we must allocate an mbuf chain large enough to
 269  * hold both the vtnet_rx_header and the maximum receivable data.
 270  */
 271 #define VTNET_NEEDED_RX_MBUFS(_sc)                                      \
 272         ((_sc)->vtnet_flags & VTNET_FLAG_LRO_NOMRG) == 0 ? 1 :          \
 273         howmany(sizeof(struct vtnet_rx_header) + VTNET_MAX_RX_SIZE,     \
 274         (_sc)->vtnet_rx_mbuf_size)
 275
 276 static int      vtnet_modevent(module_t, int, void *);
 277
 278 static int      vtnet_probe(device_t);
 279 static int      vtnet_attach(device_t);
 280 static int      vtnet_detach(device_t);
 281 static int      vtnet_suspend(device_t);
 282 static int      vtnet_resume(device_t);
 283 static int      vtnet_shutdown(device_t);
 284 static int      vtnet_config_change(device_t);
 285
 286 static void     vtnet_negotiate_features(struct vtnet_softc *);
 287 static int      vtnet_alloc_virtqueues(struct vtnet_softc *);
 288 static void     vtnet_get_hwaddr(struct vtnet_softc *);
 289 static void     vtnet_set_hwaddr(struct vtnet_softc *);
 290 static int      vtnet_is_link_up(struct vtnet_softc *);
 291 static void     vtnet_update_link_status(struct vtnet_softc *);
 292 #if 0
 293 static void     vtnet_watchdog(struct vtnet_softc *);
 294 #endif
 295 static void     vtnet_config_change_task(void *, int);
 296 static int      vtnet_change_mtu(struct vtnet_softc *, int);
 297 static int      vtnet_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 298
 299 static int      vtnet_init_rx_vq(struct vtnet_softc *);
 300 static void     vtnet_free_rx_mbufs(struct vtnet_softc *);
 301 static void     vtnet_free_tx_mbufs(struct vtnet_softc *);
 302 static void     vtnet_free_ctrl_vq(struct vtnet_softc *);
 303
 304 static struct mbuf * vtnet_alloc_rxbuf(struct vtnet_softc *, int,
 305                     struct mbuf **);
 306 static int      vtnet_replace_rxbuf(struct vtnet_softc *,
 307                     struct mbuf *, int);
 308 static int      vtnet_newbuf(struct vtnet_softc *);
 309 static void     vtnet_discard_merged_rxbuf(struct vtnet_softc *, int);
 310 static void     vtnet_discard_rxbuf(struct vtnet_softc *, struct mbuf *);
 311 static int      vtnet_enqueue_rxbuf(struct vtnet_softc *, struct mbuf *);
 312 static void     vtnet_vlan_tag_remove(struct mbuf *);
 313 static int      vtnet_rx_csum(struct vtnet_softc *, struct mbuf *,
 314                     struct virtio_net_hdr *);
 315 static int      vtnet_rxeof_merged(struct vtnet_softc *, struct mbuf *, int);
 316 static int      vtnet_rxeof(struct vtnet_softc *, int, int *);
 317 static void     vtnet_rx_intr_task(void *);
 318 static int      vtnet_rx_vq_intr(void *);
 319
 320 static void     vtnet_txeof(struct vtnet_softc *);
 321 static struct mbuf * vtnet_tx_offload(struct vtnet_softc *, struct mbuf *,
 322                     struct virtio_net_hdr *);
 323 static int      vtnet_enqueue_txbuf(struct vtnet_softc *, struct mbuf **,
 324                     struct vtnet_tx_header *);
 325 static int      vtnet_encap(struct vtnet_softc *, struct mbuf **);
 326 static void     vtnet_start_locked(struct ifnet *, struct ifaltq_subque *);
 327 static void     vtnet_start(struct ifnet *, struct ifaltq_subque *);
 328 static void     vtnet_tick(void *);
 329 static void     vtnet_tx_intr_task(void *);
 330 static int      vtnet_tx_vq_intr(void *);
 331
 332 static void     vtnet_stop(struct vtnet_softc *);
 333 static int      vtnet_reinit(struct vtnet_softc *);
 334 static void     vtnet_init_locked(struct vtnet_softc *);
 335 static void     vtnet_init(void *);
 336
 337 static void     vtnet_exec_ctrl_cmd(struct vtnet_softc *, void *,
 338                     struct sglist *, int, int);
 339
 340 static void     vtnet_rx_filter(struct vtnet_softc *sc);
 341 static int      vtnet_ctrl_rx_cmd(struct vtnet_softc *, int, int);
 342 static int      vtnet_set_promisc(struct vtnet_softc *, int);
 343 static int      vtnet_set_allmulti(struct vtnet_softc *, int);
 344 static void     vtnet_rx_filter_mac(struct vtnet_softc *);
 345
 346 static int      vtnet_exec_vlan_filter(struct vtnet_softc *, int, uint16_t);
 347 static void     vtnet_rx_filter_vlan(struct vtnet_softc *);
 348 static void     vtnet_set_vlan_filter(struct vtnet_softc *, int, uint16_t);
 349 static void     vtnet_register_vlan(void *, struct ifnet *, uint16_t);
 350 static void     vtnet_unregister_vlan(void *, struct ifnet *, uint16_t);
 351
 352 static int      vtnet_ifmedia_upd(struct ifnet *);
 353 static void     vtnet_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 354
 355 static void     vtnet_add_statistics(struct vtnet_softc *);
 356
 357 static int      vtnet_enable_rx_intr(struct vtnet_softc *);
 358 static int      vtnet_enable_tx_intr(struct vtnet_softc *);
 359 static void     vtnet_disable_rx_intr(struct vtnet_softc *);
 360 static void     vtnet_disable_tx_intr(struct vtnet_softc *);
 361
 362 /* Tunables. */
 363 static int vtnet_csum_disable = 0;
 364 TUNABLE_INT("hw.vtnet.csum_disable", &vtnet_csum_disable);
 365 static int vtnet_tso_disable = 1;
 366 TUNABLE_INT("hw.vtnet.tso_disable", &vtnet_tso_disable);
 367 static int vtnet_lro_disable = 1;
 368 TUNABLE_INT("hw.vtnet.lro_disable", &vtnet_lro_disable);
 369
 370 /*
 371  * Reducing the number of transmit completed interrupts can
 372  * improve performance. To do so, the define below keeps the
 373  * Tx vq interrupt disabled and adds calls to vtnet_txeof()
 374  * in the start and watchdog paths. The price to pay for this
 375  * is the m_free'ing of transmitted mbufs may be delayed until
 376  * the watchdog fires.
 377  */
 378 #define VTNET_TX_INTR_MODERATION
 379
 380 static struct virtio_feature_desc vtnet_feature_desc[] = {
 381         { VIRTIO_NET_F_CSUM,            "TxChecksum"    },
 382         { VIRTIO_NET_F_GUEST_CSUM,      "RxChecksum"    },
 383         { VIRTIO_NET_F_MAC,             "MacAddress"    },
 384         { VIRTIO_NET_F_GSO,             "TxAllGSO"      },
 385         { VIRTIO_NET_F_GUEST_TSO4,      "RxTSOv4"       },
 386         { VIRTIO_NET_F_GUEST_TSO6,      "RxTSOv6"       },
 387         { VIRTIO_NET_F_GUEST_ECN,       "RxECN"         },
 388         { VIRTIO_NET_F_GUEST_UFO,       "RxUFO"         },
 389         { VIRTIO_NET_F_HOST_TSO4,       "TxTSOv4"       },
 390         { VIRTIO_NET_F_HOST_TSO6,       "TxTSOv6"       },
 391         { VIRTIO_NET_F_HOST_ECN,        "TxTSOECN"      },
 392         { VIRTIO_NET_F_HOST_UFO,        "TxUFO"         },
 393         { VIRTIO_NET_F_MRG_RXBUF,       "MrgRxBuf"      },
 394         { VIRTIO_NET_F_STATUS,          "Status"        },
 395         { VIRTIO_NET_F_CTRL_VQ,         "ControlVq"     },
 396         { VIRTIO_NET_F_CTRL_RX,         "RxMode"        },
 397         { VIRTIO_NET_F_CTRL_VLAN,       "VLanFilter"    },
 398         { VIRTIO_NET_F_CTRL_RX_EXTRA,   "RxModeExtra"   },
 399         { VIRTIO_NET_F_MQ,              "RFS"           },
 400         { 0, NULL }
 401 };
 402
 403 static device_method_t vtnet_methods[] = {
 404         /* Device methods. */
 405         DEVMETHOD(device_probe,         vtnet_probe),
 406         DEVMETHOD(device_attach,        vtnet_attach),
 407         DEVMETHOD(device_detach,        vtnet_detach),
 408         DEVMETHOD(device_suspend,       vtnet_suspend),
 409         DEVMETHOD(device_resume,        vtnet_resume),
 410         DEVMETHOD(device_shutdown,      vtnet_shutdown),
 411
 412         /* VirtIO methods. */
 413         DEVMETHOD(virtio_config_change, vtnet_config_change),
 414
 415         { 0, 0 }
 416 };
 417
 418 static driver_t vtnet_driver = {
 419         "vtnet",
 420         vtnet_methods,
 421         sizeof(struct vtnet_softc)
 422 };
 423
 424 static devclass_t vtnet_devclass;
 425
 426 DRIVER_MODULE(vtnet, virtio_pci, vtnet_driver, vtnet_devclass,
 427               vtnet_modevent, 0);
 428 MODULE_VERSION(vtnet, 1);
 429 MODULE_DEPEND(vtnet, virtio, 1, 1, 1);
 430
 431 static int
 432 vtnet_modevent(module_t mod, int type, void *unused)
 433 {
 434         int error;
 435
 436         error = 0;
 437
 438         switch (type) {
 439         case MOD_LOAD:
 440                 break;
 441         case MOD_UNLOAD:
 442                 break;
 443         case MOD_SHUTDOWN:
 444                 break;
 445         default:
 446                 error = EOPNOTSUPP;
 447                 break;
 448         }
 449
 450         return (error);
 451 }
 452
 453 static int
 454 vtnet_probe(device_t dev)
 455 {
 456         if (virtio_get_device_type(dev) != VIRTIO_ID_NETWORK)
 457                 return (ENXIO);
 458
 459         device_set_desc(dev, "VirtIO Networking Adapter");
 460
 461         return (BUS_PROBE_DEFAULT);
 462 }
 463
 464 static int
 465 vtnet_attach(device_t dev)
 466 {
 467         struct vtnet_softc *sc;
 468         struct ifnet *ifp;
 469         int tx_size, error;
 470
 471         sc = device_get_softc(dev);
 472         sc->vtnet_dev = dev;
 473
 474         lwkt_serialize_init(&sc->vtnet_slz);
 475         callout_init(&sc->vtnet_tick_ch);
 476
 477         ifmedia_init(&sc->vtnet_media, IFM_IMASK, vtnet_ifmedia_upd,
 478                      vtnet_ifmedia_sts);
 479         ifmedia_add(&sc->vtnet_media, VTNET_MEDIATYPE, 0, NULL);
 480         ifmedia_set(&sc->vtnet_media, VTNET_MEDIATYPE);
 481
 482         vtnet_add_statistics(sc);
 483
 484         virtio_set_feature_desc(dev, vtnet_feature_desc);
 485         vtnet_negotiate_features(sc);
 486
 487         if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) {
 488                 sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS;
 489                 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 490         } else {
 491                 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
 492         }
 493
 494         sc->vtnet_rx_mbuf_size = MCLBYTES;
 495         sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc);
 496
 497         if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VQ)) {
 498                 sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ;
 499
 500                 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX))
 501                         sc->vtnet_flags |= VTNET_FLAG_CTRL_RX;
 502                 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VLAN))
 503                         sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER;
 504         }
 505
 506         vtnet_get_hwaddr(sc);
 507
 508         error = vtnet_alloc_virtqueues(sc);
 509         if (error) {
 510                 device_printf(dev, "cannot allocate virtqueues\n");
 511                 goto fail;
 512         }
 513
 514         ifp = sc->vtnet_ifp = if_alloc(IFT_ETHER);
 515         if (ifp == NULL) {
 516                 device_printf(dev, "cannot allocate ifnet structure\n");
 517                 error = ENOSPC;
 518                 goto fail;
 519         }
 520
 521         ifp->if_softc = sc;
 522         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 523         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 524         ifp->if_init = vtnet_init;
 525         ifp->if_start = vtnet_start;
 526         ifp->if_ioctl = vtnet_ioctl;
 527
 528         sc->vtnet_rx_size = virtqueue_size(sc->vtnet_rx_vq);
 529         sc->vtnet_rx_process_limit = sc->vtnet_rx_size;
 530
 531         tx_size = virtqueue_size(sc->vtnet_tx_vq);
 532         sc->vtnet_tx_size = tx_size;
 533         sc->vtnet_txhdridx = 0;
 534         sc->vtnet_txhdrarea = contigmalloc(
 535             ((sc->vtnet_tx_size / 2) + 1) * sizeof(struct vtnet_tx_header),
 536             M_VTNET, M_WAITOK, 0, BUS_SPACE_MAXADDR, 4, 0);
 537         if (sc->vtnet_txhdrarea == NULL) {
 538                 device_printf(dev, "cannot contigmalloc the tx headers\n");
 539                 goto fail;
 540         }
 541         sc->vtnet_macfilter = contigmalloc(
 542             sizeof(struct vtnet_mac_filter),
 543             M_DEVBUF, M_WAITOK, 0, BUS_SPACE_MAXADDR, 4, 0);
 544         if (sc->vtnet_macfilter == NULL) {
 545                 device_printf(dev,
 546                     "cannot contigmalloc the mac filter table\n");
 547                 goto fail;
 548         }
 549         ifq_set_maxlen(&ifp->if_snd, tx_size - 1);
 550         ifq_set_ready(&ifp->if_snd);
 551
 552         ether_ifattach(ifp, sc->vtnet_hwaddr, NULL);
 553
 554         if (virtio_with_feature(dev, VIRTIO_NET_F_STATUS)){
 555                 //ifp->if_capabilities |= IFCAP_LINKSTATE;
 556                  kprintf("add dynamic link state\n");
 557         }
 558
 559         /* Tell the upper layer(s) we support long frames. */
 560         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
 561         ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU;
 562
 563         if (virtio_with_feature(dev, VIRTIO_NET_F_CSUM)) {
 564                 ifp->if_capabilities |= IFCAP_TXCSUM;
 565
 566                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4))
 567                         ifp->if_capabilities |= IFCAP_TSO4;
 568                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
 569                         ifp->if_capabilities |= IFCAP_TSO6;
 570                 if (ifp->if_capabilities & IFCAP_TSO)
 571                         ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
 572
 573                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN))
 574                         sc->vtnet_flags |= VTNET_FLAG_TSO_ECN;
 575         }
 576
 577         if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_CSUM)) {
 578                 ifp->if_capabilities |= IFCAP_RXCSUM;
 579
 580                 if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) ||
 581                     virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6))
 582                         ifp->if_capabilities |= IFCAP_LRO;
 583         }
 584
 585         if (ifp->if_capabilities & IFCAP_HWCSUM) {
 586                 /*
 587                  * VirtIO does not support VLAN tagging, but we can fake
 588                  * it by inserting and removing the 802.1Q header during
 589                  * transmit and receive. We are then able to do checksum
 590                  * offloading of VLAN frames.
 591                  */
 592                 ifp->if_capabilities |=
 593                         IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM;
 594         }
 595
 596         ifp->if_capenable = ifp->if_capabilities;
 597
 598         /*
 599          * Capabilities after here are not enabled by default.
 600          */
 601
 602         if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) {
 603                 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
 604
 605                 sc->vtnet_vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 606                     vtnet_register_vlan, sc, EVENTHANDLER_PRI_FIRST);
 607                 sc->vtnet_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 608                     vtnet_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST);
 609         }
 610
 611         TASK_INIT(&sc->vtnet_cfgchg_task, 0, vtnet_config_change_task, sc);
 612
 613         error = virtio_setup_intr(dev, &sc->vtnet_slz);
 614         if (error) {
 615                 device_printf(dev, "cannot setup virtqueue interrupts\n");
 616                 ether_ifdetach(ifp);
 617                 goto fail;
 618         }
 619
 620         /*
 621          * Device defaults to promiscuous mode for backwards
 622          * compatibility. Turn it off if possible.
 623          */
 624         if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
 625                 lwkt_serialize_enter(&sc->vtnet_slz);
 626                 if (vtnet_set_promisc(sc, 0) != 0) {
 627                         ifp->if_flags |= IFF_PROMISC;
 628                         device_printf(dev,
 629                             "cannot disable promiscuous mode\n");
 630                 }
 631                 lwkt_serialize_exit(&sc->vtnet_slz);
 632         } else
 633                 ifp->if_flags |= IFF_PROMISC;
 634
 635 fail:
 636         if (error)
 637                 vtnet_detach(dev);
 638
 639         return (error);
 640 }
 641
 642 static int
 643 vtnet_detach(device_t dev)
 644 {
 645         struct vtnet_softc *sc;
 646         struct ifnet *ifp;
 647
 648         sc = device_get_softc(dev);
 649         ifp = sc->vtnet_ifp;
 650
 651         if (device_is_attached(dev)) {
 652                 lwkt_serialize_enter(&sc->vtnet_slz);
 653                 vtnet_stop(sc);
 654                 lwkt_serialize_exit(&sc->vtnet_slz);
 655
 656                 callout_stop(&sc->vtnet_tick_ch);
 657                 taskqueue_drain(taskqueue_swi, &sc->vtnet_cfgchg_task);
 658
 659                 ether_ifdetach(ifp);
 660         }
 661
 662         if (sc->vtnet_vlan_attach != NULL) {
 663                 EVENTHANDLER_DEREGISTER(vlan_config, sc->vtnet_vlan_attach);
 664                 sc->vtnet_vlan_attach = NULL;
 665         }
 666         if (sc->vtnet_vlan_detach != NULL) {
 667                 EVENTHANDLER_DEREGISTER(vlan_unconfg, sc->vtnet_vlan_detach);
 668                 sc->vtnet_vlan_detach = NULL;
 669         }
 670
 671         if (ifp) {
 672                 if_free(ifp);
 673                 sc->vtnet_ifp = NULL;
 674         }
 675
 676         if (sc->vtnet_rx_vq != NULL)
 677                 vtnet_free_rx_mbufs(sc);
 678         if (sc->vtnet_tx_vq != NULL)
 679                 vtnet_free_tx_mbufs(sc);
 680         if (sc->vtnet_ctrl_vq != NULL)
 681                 vtnet_free_ctrl_vq(sc);
 682
 683         if (sc->vtnet_txhdrarea != NULL) {
 684                 contigfree(sc->vtnet_txhdrarea,
 685                     ((sc->vtnet_tx_size / 2) + 1) *
 686                     sizeof(struct vtnet_tx_header), M_VTNET);
 687                 sc->vtnet_txhdrarea = NULL;
 688         }
 689         if (sc->vtnet_macfilter != NULL) {
 690                 contigfree(sc->vtnet_macfilter,
 691                     sizeof(struct vtnet_mac_filter), M_DEVBUF);
 692                 sc->vtnet_macfilter = NULL;
 693         }
 694
 695         ifmedia_removeall(&sc->vtnet_media);
 696
 697         return (0);
 698 }
 699
 700 static int
 701 vtnet_suspend(device_t dev)
 702 {
 703         struct vtnet_softc *sc;
 704
 705         sc = device_get_softc(dev);
 706
 707         lwkt_serialize_enter(&sc->vtnet_slz);
 708         vtnet_stop(sc);
 709         sc->vtnet_flags |= VTNET_FLAG_SUSPENDED;
 710         lwkt_serialize_exit(&sc->vtnet_slz);
 711
 712         return (0);
 713 }
 714
 715 static int
 716 vtnet_resume(device_t dev)
 717 {
 718         struct vtnet_softc *sc;
 719         struct ifnet *ifp;
 720
 721         sc = device_get_softc(dev);
 722         ifp = sc->vtnet_ifp;
 723
 724         lwkt_serialize_enter(&sc->vtnet_slz);
 725         if (ifp->if_flags & IFF_UP)
 726                 vtnet_init_locked(sc);
 727         sc->vtnet_flags &= ~VTNET_FLAG_SUSPENDED;
 728         lwkt_serialize_exit(&sc->vtnet_slz);
 729
 730         return (0);
 731 }
 732
 733 static int
 734 vtnet_shutdown(device_t dev)
 735 {
 736
 737         /*
 738          * Suspend already does all of what we need to
 739          * do here; we just never expect to be resumed.
 740          */
 741         return (vtnet_suspend(dev));
 742 }
 743
 744 static int
 745 vtnet_config_change(device_t dev)
 746 {
 747         struct vtnet_softc *sc;
 748
 749         sc = device_get_softc(dev);
 750
 751         taskqueue_enqueue(taskqueue_thread[mycpuid], &sc->vtnet_cfgchg_task);
 752
 753         return (1);
 754 }
 755
 756 static void
 757 vtnet_negotiate_features(struct vtnet_softc *sc)
 758 {
 759         device_t dev;
 760         uint64_t mask, features;
 761
 762         dev = sc->vtnet_dev;
 763         mask = 0;
 764
 765         if (vtnet_csum_disable)
 766                 mask |= VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM;
 767
 768         /*
 769          * TSO and LRO are only available when their corresponding
 770          * checksum offload feature is also negotiated.
 771          */
 772
 773         if (vtnet_csum_disable || vtnet_tso_disable)
 774                 mask |= VIRTIO_NET_F_HOST_TSO4 | VIRTIO_NET_F_HOST_TSO6 |
 775                     VIRTIO_NET_F_HOST_ECN;
 776
 777         if (vtnet_csum_disable || vtnet_lro_disable)
 778                 mask |= VTNET_LRO_FEATURES;
 779
 780         features = VTNET_FEATURES & ~mask;
 781         features |= VIRTIO_F_NOTIFY_ON_EMPTY;
 782         sc->vtnet_features = virtio_negotiate_features(dev, features);
 783 }
 784
 785 static int
 786 vtnet_alloc_virtqueues(struct vtnet_softc *sc)
 787 {
 788         device_t dev;
 789         struct vq_alloc_info vq_info[3];
 790         int nvqs, rxsegs;
 791
 792         dev = sc->vtnet_dev;
 793         nvqs = 2;
 794
 795         /*
 796          * Indirect descriptors are not needed for the Rx
 797          * virtqueue when mergeable buffers are negotiated.
 798          * The header is placed inline with the data, not
 799          * in a separate descriptor, and mbuf clusters are
 800          * always physically contiguous.
 801          */
 802         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
 803                 rxsegs = sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG ?
 804                     VTNET_MAX_RX_SEGS : VTNET_MIN_RX_SEGS;
 805         } else
 806                 rxsegs = 0;
 807
 808         VQ_ALLOC_INFO_INIT(&vq_info[0], rxsegs,
 809             vtnet_rx_vq_intr, sc, &sc->vtnet_rx_vq,
 810             "%s receive", device_get_nameunit(dev));
 811
 812         VQ_ALLOC_INFO_INIT(&vq_info[1], VTNET_MAX_TX_SEGS,
 813             vtnet_tx_vq_intr, sc, &sc->vtnet_tx_vq,
 814             "%s transmit", device_get_nameunit(dev));
 815
 816         if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
 817                 nvqs++;
 818
 819                 VQ_ALLOC_INFO_INIT(&vq_info[2], 0, NULL, NULL,
 820                     &sc->vtnet_ctrl_vq, "%s control",
 821                     device_get_nameunit(dev));
 822         }
 823
 824         return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
 825 }
 826
 827 static void
 828 vtnet_get_hwaddr(struct vtnet_softc *sc)
 829 {
 830         device_t dev;
 831
 832         dev = sc->vtnet_dev;
 833
 834         if (virtio_with_feature(dev, VIRTIO_NET_F_MAC)) {
 835                 virtio_read_device_config(dev,
 836                     offsetof(struct virtio_net_config, mac),
 837                     sc->vtnet_hwaddr, ETHER_ADDR_LEN);
 838         } else {
 839                 /* Generate random locally administered unicast address. */
 840                 sc->vtnet_hwaddr[0] = 0xB2;
 841                 karc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1);
 842
 843                 vtnet_set_hwaddr(sc);
 844         }
 845 }
 846
 847 static void
 848 vtnet_set_hwaddr(struct vtnet_softc *sc)
 849 {
 850         device_t dev;
 851
 852         dev = sc->vtnet_dev;
 853
 854         virtio_write_device_config(dev,
 855             offsetof(struct virtio_net_config, mac),
 856             sc->vtnet_hwaddr, ETHER_ADDR_LEN);
 857 }
 858
 859 static int
 860 vtnet_is_link_up(struct vtnet_softc *sc)
 861 {
 862         device_t dev;
 863         struct ifnet *ifp;
 864         uint16_t status;
 865
 866         dev = sc->vtnet_dev;
 867         ifp = sc->vtnet_ifp;
 868
 869         ASSERT_SERIALIZED(&sc->vtnet_slz);
 870
 871         status = virtio_read_dev_config_2(dev,
 872                         offsetof(struct virtio_net_config, status));
 873
 874         return ((status & VIRTIO_NET_S_LINK_UP) != 0);
 875 }
 876
 877 static void
 878 vtnet_update_link_status(struct vtnet_softc *sc)
 879 {
 880         device_t dev;
 881         struct ifnet *ifp;
 882         struct ifaltq_subque *ifsq;
 883         int link;
 884
 885         dev = sc->vtnet_dev;
 886         ifp = sc->vtnet_ifp;
 887         ifsq = ifq_get_subq_default(&ifp->if_snd);
 888
 889         link = vtnet_is_link_up(sc);
 890
 891         if (link && ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0)) {
 892                 sc->vtnet_flags |= VTNET_FLAG_LINK;
 893                 if (bootverbose)
 894                         device_printf(dev, "Link is up\n");
 895                 ifp->if_link_state = LINK_STATE_UP;
 896                 if_link_state_change(ifp);
 897                 if (!ifsq_is_empty(ifsq))
 898                         vtnet_start_locked(ifp, ifsq);
 899         } else if (!link && (sc->vtnet_flags & VTNET_FLAG_LINK)) {
 900                 sc->vtnet_flags &= ~VTNET_FLAG_LINK;
 901                 if (bootverbose)
 902                         device_printf(dev, "Link is down\n");
 903
 904                 ifp->if_link_state = LINK_STATE_DOWN;
 905                 if_link_state_change(ifp);
 906         }
 907 }
 908
 909 #if 0
 910 static void
 911 vtnet_watchdog(struct vtnet_softc *sc)
 912 {
 913         struct ifnet *ifp;
 914
 915         ifp = sc->vtnet_ifp;
 916
 917 #ifdef VTNET_TX_INTR_MODERATION
 918         vtnet_txeof(sc);
 919 #endif
 920
 921         if (sc->vtnet_watchdog_timer == 0 || --sc->vtnet_watchdog_timer)
 922                 return;
 923
 924         if_printf(ifp, "watchdog timeout -- resetting\n");
 925 #ifdef VTNET_DEBUG
 926         virtqueue_dump(sc->vtnet_tx_vq);
 927 #endif
 928         ifp->if_oerrors++;
 929         ifp->if_flags &= ~IFF_RUNNING;
 930         vtnet_init_locked(sc);
 931 }
 932 #endif
 933
 934 static void
 935 vtnet_config_change_task(void *arg, int pending)
 936 {
 937         struct vtnet_softc *sc;
 938
 939         sc = arg;
 940
 941         lwkt_serialize_enter(&sc->vtnet_slz);
 942         vtnet_update_link_status(sc);
 943         lwkt_serialize_exit(&sc->vtnet_slz);
 944 }
 945
 946 static int
 947 vtnet_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data,struct ucred *cr)
 948 {
 949         struct vtnet_softc *sc;
 950         struct ifreq *ifr;
 951         int reinit, mask, error;
 952
 953         sc = ifp->if_softc;
 954         ifr = (struct ifreq *) data;
 955         reinit = 0;
 956         error = 0;
 957
 958         switch (cmd) {
 959         case SIOCSIFMTU:
 960                 if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > VTNET_MAX_MTU)
 961                         error = EINVAL;
 962                 else if (ifp->if_mtu != ifr->ifr_mtu) {
 963                         lwkt_serialize_enter(&sc->vtnet_slz);
 964                         error = vtnet_change_mtu(sc, ifr->ifr_mtu);
 965                         lwkt_serialize_exit(&sc->vtnet_slz);
 966                 }
 967                 break;
 968
 969         case SIOCSIFFLAGS:
 970                 lwkt_serialize_enter(&sc->vtnet_slz);
 971                 if ((ifp->if_flags & IFF_UP) == 0) {
 972                         if (ifp->if_flags & IFF_RUNNING)
 973                                 vtnet_stop(sc);
 974                 } else if (ifp->if_flags & IFF_RUNNING) {
 975                         if ((ifp->if_flags ^ sc->vtnet_if_flags) &
 976                             (IFF_PROMISC | IFF_ALLMULTI)) {
 977                                 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX)
 978                                         vtnet_rx_filter(sc);
 979                                 else
 980                                         error = ENOTSUP;
 981                         }
 982                 } else
 983                         vtnet_init_locked(sc);
 984
 985                 if (error == 0)
 986                         sc->vtnet_if_flags = ifp->if_flags;
 987                 lwkt_serialize_exit(&sc->vtnet_slz);
 988                 break;
 989
 990         case SIOCADDMULTI:
 991         case SIOCDELMULTI:
 992                 lwkt_serialize_enter(&sc->vtnet_slz);
 993                 if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) &&
 994                     (ifp->if_flags & IFF_RUNNING))
 995                         vtnet_rx_filter_mac(sc);
 996                 lwkt_serialize_exit(&sc->vtnet_slz);
 997                 break;
 998
 999         case SIOCSIFMEDIA:
1000         case SIOCGIFMEDIA:
1001                 error = ifmedia_ioctl(ifp, ifr, &sc->vtnet_media, cmd);
1002                 break;
1003
1004         case SIOCSIFCAP:
1005                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1006
1007                 lwkt_serialize_enter(&sc->vtnet_slz);
1008
1009                 if (mask & IFCAP_TXCSUM) {
1010                         ifp->if_capenable ^= IFCAP_TXCSUM;
1011                         if (ifp->if_capenable & IFCAP_TXCSUM)
1012                                 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;
1013                         else
1014                                 ifp->if_hwassist &= ~VTNET_CSUM_OFFLOAD;
1015                 }
1016
1017                 if (mask & IFCAP_TSO4) {
1018                         ifp->if_capenable ^= IFCAP_TSO4;
1019                         if (ifp->if_capenable & IFCAP_TSO4)
1020                                 ifp->if_hwassist |= CSUM_TSO;
1021                         else
1022                                 ifp->if_hwassist &= ~CSUM_TSO;
1023                 }
1024
1025                 if (mask & IFCAP_RXCSUM) {
1026                         ifp->if_capenable ^= IFCAP_RXCSUM;
1027                         reinit = 1;
1028                 }
1029
1030                 if (mask & IFCAP_LRO) {
1031                         ifp->if_capenable ^= IFCAP_LRO;
1032                         reinit = 1;
1033                 }
1034
1035                 if (mask & IFCAP_VLAN_HWFILTER) {
1036                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1037                         reinit = 1;
1038                 }
1039
1040                 if (mask & IFCAP_VLAN_HWTSO)
1041                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1042
1043                 if (mask & IFCAP_VLAN_HWTAGGING)
1044                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1045
1046                 if (reinit && (ifp->if_flags & IFF_RUNNING)) {
1047                         ifp->if_flags &= ~IFF_RUNNING;
1048                         vtnet_init_locked(sc);
1049                 }
1050                 //VLAN_CAPABILITIES(ifp);
1051
1052                 lwkt_serialize_exit(&sc->vtnet_slz);
1053                 break;
1054
1055         default:
1056                 error = ether_ioctl(ifp, cmd, data);
1057                 break;
1058         }
1059
1060         return (error);
1061 }
1062
1063 static int
1064 vtnet_change_mtu(struct vtnet_softc *sc, int new_mtu)
1065 {
1066         struct ifnet *ifp;
1067         int new_frame_size, clsize;
1068
1069         ifp = sc->vtnet_ifp;
1070
1071         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1072                 new_frame_size = sizeof(struct vtnet_rx_header) +
1073                     sizeof(struct ether_vlan_header) + new_mtu;
1074
1075                 if (new_frame_size > MJUM9BYTES)
1076                         return (EINVAL);
1077
1078                 if (new_frame_size <= MCLBYTES)
1079                         clsize = MCLBYTES;
1080                 else
1081                         clsize = MJUM9BYTES;
1082         } else {
1083                 new_frame_size = sizeof(struct virtio_net_hdr_mrg_rxbuf) +
1084                     sizeof(struct ether_vlan_header) + new_mtu;
1085
1086                 if (new_frame_size <= MCLBYTES)
1087                         clsize = MCLBYTES;
1088                 else
1089                         clsize = MJUMPAGESIZE;
1090         }
1091
1092         sc->vtnet_rx_mbuf_size = clsize;
1093         sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc);
1094         KASSERT(sc->vtnet_rx_mbuf_count < VTNET_MAX_RX_SEGS,
1095             ("too many rx mbufs: %d", sc->vtnet_rx_mbuf_count));
1096
1097         ifp->if_mtu = new_mtu;
1098
1099         if (ifp->if_flags & IFF_RUNNING) {
1100                 ifp->if_flags &= ~IFF_RUNNING;
1101                 vtnet_init_locked(sc);
1102         }
1103
1104         return (0);
1105 }
1106
1107 static int
1108 vtnet_init_rx_vq(struct vtnet_softc *sc)
1109 {
1110         struct virtqueue *vq;
1111         int nbufs, error;
1112
1113         vq = sc->vtnet_rx_vq;
1114         nbufs = 0;
1115         error = ENOSPC;
1116
1117         while (!virtqueue_full(vq)) {
1118                 if ((error = vtnet_newbuf(sc)) != 0)
1119                         break;
1120                 nbufs++;
1121         }
1122
1123         if (nbufs > 0) {
1124                 virtqueue_notify(vq, &sc->vtnet_slz);
1125
1126                 /*
1127                  * EMSGSIZE signifies the virtqueue did not have enough
1128                  * entries available to hold the last mbuf. This is not
1129                  * an error. We should not get ENOSPC since we check if
1130                  * the virtqueue is full before attempting to add a
1131                  * buffer.
1132                  */
1133                 if (error == EMSGSIZE)
1134                         error = 0;
1135         }
1136
1137         return (error);
1138 }
1139
1140 static void
1141 vtnet_free_rx_mbufs(struct vtnet_softc *sc)
1142 {
1143         struct virtqueue *vq;
1144         struct mbuf *m;
1145         int last;
1146
1147         vq = sc->vtnet_rx_vq;
1148         last = 0;
1149
1150         while ((m = virtqueue_drain(vq, &last)) != NULL)
1151                 m_freem(m);
1152
1153         KASSERT(virtqueue_empty(vq), ("mbufs remaining in Rx Vq"));
1154 }
1155
1156 static void
1157 vtnet_free_tx_mbufs(struct vtnet_softc *sc)
1158 {
1159         struct virtqueue *vq;
1160         struct vtnet_tx_header *txhdr;
1161         int last;
1162
1163         vq = sc->vtnet_tx_vq;
1164         last = 0;
1165
1166         while ((txhdr = virtqueue_drain(vq, &last)) != NULL) {
1167                 m_freem(txhdr->vth_mbuf);
1168         }
1169
1170         KASSERT(virtqueue_empty(vq), ("mbufs remaining in Tx Vq"));
1171 }
1172
1173 static void
1174 vtnet_free_ctrl_vq(struct vtnet_softc *sc)
1175 {
1176         /*
1177          * The control virtqueue is only polled, therefore
1178          * it should already be empty.
1179          */
1180         KASSERT(virtqueue_empty(sc->vtnet_ctrl_vq),
1181                 ("Ctrl Vq not empty"));
1182 }
1183
1184 static struct mbuf *
1185 vtnet_alloc_rxbuf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp)
1186 {
1187         struct mbuf *m_head, *m_tail, *m;
1188         int i, clsize;
1189
1190         clsize = sc->vtnet_rx_mbuf_size;
1191
1192         /*use getcl instead of getjcl. see  if_mxge.c comment line 2398*/
1193         //m_head = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, clsize);
1194         m_head = m_getcl(MB_DONTWAIT, MT_DATA, M_PKTHDR );
1195         if (m_head == NULL)
1196                 goto fail;
1197
1198         m_head->m_len = clsize;
1199         m_tail = m_head;
1200
1201         if (nbufs > 1) {
1202                 KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
1203                         ("chained Rx mbuf requested without LRO_NOMRG"));
1204
1205                 for (i = 0; i < nbufs - 1; i++) {
1206                         //m = m_getjcl(M_DONTWAIT, MT_DATA, 0, clsize);
1207                         m = m_getcl(MB_DONTWAIT, MT_DATA, 0);
1208                         if (m == NULL)
1209                                 goto fail;
1210
1211                         m->m_len = clsize;
1212                         m_tail->m_next = m;
1213                         m_tail = m;
1214                 }
1215         }
1216
1217         if (m_tailp != NULL)
1218                 *m_tailp = m_tail;
1219
1220         return (m_head);
1221
1222 fail:
1223         sc->vtnet_stats.mbuf_alloc_failed++;
1224         m_freem(m_head);
1225
1226         return (NULL);
1227 }
1228
1229 static int
1230 vtnet_replace_rxbuf(struct vtnet_softc *sc, struct mbuf *m0, int len0)
1231 {
1232         struct mbuf *m, *m_prev;
1233         struct mbuf *m_new, *m_tail;
1234         int len, clsize, nreplace, error;
1235
1236         m = m0;
1237         m_prev = NULL;
1238         len = len0;
1239
1240         m_tail = NULL;
1241         clsize = sc->vtnet_rx_mbuf_size;
1242         nreplace = 0;
1243
1244         if (m->m_next != NULL)
1245                 KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
1246                     ("chained Rx mbuf without LRO_NOMRG"));
1247
1248         /*
1249          * Since LRO_NOMRG mbuf chains are so large, we want to avoid
1250          * allocating an entire chain for each received frame. When
1251          * the received frame's length is less than that of the chain,
1252          * the unused mbufs are reassigned to the new chain.
1253          */
1254         while (len > 0) {
1255                 /*
1256                  * Something is seriously wrong if we received
1257                  * a frame larger than the mbuf chain. Drop it.
1258                  */
1259                 if (m == NULL) {
1260                         sc->vtnet_stats.rx_frame_too_large++;
1261                         return (EMSGSIZE);
1262                 }
1263
1264                 KASSERT(m->m_len == clsize,
1265                     ("mbuf length not expected cluster size: %d",
1266                     m->m_len));
1267
1268                 m->m_len = MIN(m->m_len, len);
1269                 len -= m->m_len;
1270
1271                 m_prev = m;
1272                 m = m->m_next;
1273                 nreplace++;
1274         }
1275
1276         KASSERT(m_prev != NULL, ("m_prev == NULL"));
1277         KASSERT(nreplace <= sc->vtnet_rx_mbuf_count,
1278                 ("too many replacement mbufs: %d/%d", nreplace,
1279                 sc->vtnet_rx_mbuf_count));
1280
1281         m_new = vtnet_alloc_rxbuf(sc, nreplace, &m_tail);
1282         if (m_new == NULL) {
1283                 m_prev->m_len = clsize;
1284                 return (ENOBUFS);
1285         }
1286
1287         /*
1288          * Move unused mbufs, if any, from the original chain
1289          * onto the end of the new chain.
1290          */
1291         if (m_prev->m_next != NULL) {
1292                 m_tail->m_next = m_prev->m_next;
1293                 m_prev->m_next = NULL;
1294         }
1295
1296         error = vtnet_enqueue_rxbuf(sc, m_new);
1297         if (error) {
1298                 /*
1299                  * BAD! We could not enqueue the replacement mbuf chain. We
1300                  * must restore the m0 chain to the original state if it was
1301                  * modified so we can subsequently discard it.
1302                  *
1303                  * NOTE: The replacement is suppose to be an identical copy
1304                  * to the one just dequeued so this is an unexpected error.
1305                  */
1306                 sc->vtnet_stats.rx_enq_replacement_failed++;
1307
1308                 if (m_tail->m_next != NULL) {
1309                         m_prev->m_next = m_tail->m_next;
1310                         m_tail->m_next = NULL;
1311                 }
1312
1313                 m_prev->m_len = clsize;
1314                 m_freem(m_new);
1315         }
1316
1317         return (error);
1318 }
1319
1320 static int
1321 vtnet_newbuf(struct vtnet_softc *sc)
1322 {
1323         struct mbuf *m;
1324         int error;
1325
1326         m = vtnet_alloc_rxbuf(sc, sc->vtnet_rx_mbuf_count, NULL);
1327         if (m == NULL)
1328                 return (ENOBUFS);
1329
1330         error = vtnet_enqueue_rxbuf(sc, m);
1331         if (error)
1332                 m_freem(m);
1333
1334         return (error);
1335 }
1336
1337 static void
1338 vtnet_discard_merged_rxbuf(struct vtnet_softc *sc, int nbufs)
1339 {
1340         struct virtqueue *vq;
1341         struct mbuf *m;
1342
1343         vq = sc->vtnet_rx_vq;
1344
1345         while (--nbufs > 0) {
1346                 if ((m = virtqueue_dequeue(vq, NULL)) == NULL)
1347                         break;
1348                 vtnet_discard_rxbuf(sc, m);
1349         }
1350 }
1351
1352 static void
1353 vtnet_discard_rxbuf(struct vtnet_softc *sc, struct mbuf *m)
1354 {
1355         int error;
1356
1357         /*
1358          * Requeue the discarded mbuf. This should always be
1359          * successful since it was just dequeued.
1360          */
1361         error = vtnet_enqueue_rxbuf(sc, m);
1362         KASSERT(error == 0, ("cannot requeue discarded mbuf"));
1363 }
1364
1365 static int
1366 vtnet_enqueue_rxbuf(struct vtnet_softc *sc, struct mbuf *m)
1367 {
1368         struct sglist sg;
1369         struct sglist_seg segs[VTNET_MAX_RX_SEGS];
1370         struct vtnet_rx_header *rxhdr;
1371         struct virtio_net_hdr *hdr;
1372         uint8_t *mdata;
1373         int offset, error;
1374
1375         ASSERT_SERIALIZED(&sc->vtnet_slz);
1376         if ((sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) == 0)
1377                 KASSERT(m->m_next == NULL, ("chained Rx mbuf"));
1378
1379         sglist_init(&sg, VTNET_MAX_RX_SEGS, segs);
1380
1381         mdata = mtod(m, uint8_t *);
1382         offset = 0;
1383
1384         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1385                 rxhdr = (struct vtnet_rx_header *) mdata;
1386                 hdr = &rxhdr->vrh_hdr;
1387                 offset += sizeof(struct vtnet_rx_header);
1388
1389                 error = sglist_append(&sg, hdr, sc->vtnet_hdr_size);
1390                 KASSERT(error == 0, ("cannot add header to sglist"));
1391         }
1392
1393         error = sglist_append(&sg, mdata + offset, m->m_len - offset);
1394         if (error)
1395                 return (error);
1396
1397         if (m->m_next != NULL) {
1398                 error = sglist_append_mbuf(&sg, m->m_next);
1399                 if (error)
1400                         return (error);
1401         }
1402
1403         return (virtqueue_enqueue(sc->vtnet_rx_vq, m, &sg, 0, sg.sg_nseg));
1404 }
1405
1406 static void
1407 vtnet_vlan_tag_remove(struct mbuf *m)
1408 {
1409         struct ether_vlan_header *evl;
1410
1411         evl = mtod(m, struct ether_vlan_header *);
1412
1413         m->m_pkthdr.ether_vlantag = ntohs(evl->evl_tag);
1414         m->m_flags |= M_VLANTAG;
1415
1416         /* Strip the 802.1Q header. */
1417         bcopy((char *) evl, (char *) evl + ETHER_VLAN_ENCAP_LEN,
1418             ETHER_HDR_LEN - ETHER_TYPE_LEN);
1419         m_adj(m, ETHER_VLAN_ENCAP_LEN);
1420 }
1421
1422 /*
1423  * Alternative method of doing receive checksum offloading. Rather
1424  * than parsing the received frame down to the IP header, use the
1425  * csum_offset to determine which CSUM_* flags are appropriate. We
1426  * can get by with doing this only because the checksum offsets are
1427  * unique for the things we care about.
1428  */
1429 static int
1430 vtnet_rx_csum(struct vtnet_softc *sc, struct mbuf *m,
1431     struct virtio_net_hdr *hdr)
1432 {
1433         struct ether_header *eh;
1434         struct ether_vlan_header *evh;
1435         struct udphdr *udp;
1436         int csum_len;
1437         uint16_t eth_type;
1438
1439         csum_len = hdr->csum_start + hdr->csum_offset;
1440
1441         if (csum_len < sizeof(struct ether_header) + sizeof(struct ip))
1442                 return (1);
1443         if (m->m_len < csum_len)
1444                 return (1);
1445
1446         eh = mtod(m, struct ether_header *);
1447         eth_type = ntohs(eh->ether_type);
1448         if (eth_type == ETHERTYPE_VLAN) {
1449                 evh = mtod(m, struct ether_vlan_header *);
1450                 eth_type = ntohs(evh->evl_proto);
1451         }
1452
1453         if (eth_type != ETHERTYPE_IP && eth_type != ETHERTYPE_IPV6) {
1454                 sc->vtnet_stats.rx_csum_bad_ethtype++;
1455                 return (1);
1456         }
1457
1458         /* Use the offset to determine the appropriate CSUM_* flags. */
1459         switch (hdr->csum_offset) {
1460         case offsetof(struct udphdr, uh_sum):
1461                 if (m->m_len < hdr->csum_start + sizeof(struct udphdr))
1462                         return (1);
1463                 udp = (struct udphdr *)(mtod(m, uint8_t *) + hdr->csum_start);
1464                 if (udp->uh_sum == 0)
1465                         return (0);
1466
1467                 /* FALLTHROUGH */
1468
1469         case offsetof(struct tcphdr, th_sum):
1470                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1471                 m->m_pkthdr.csum_data = 0xFFFF;
1472                 break;
1473
1474         case offsetof(struct sctphdr, checksum):
1475                 //m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID;
1476                 break;
1477
1478         default:
1479                 sc->vtnet_stats.rx_csum_bad_offset++;
1480                 return (1);
1481         }
1482
1483         sc->vtnet_stats.rx_csum_offloaded++;
1484
1485         return (0);
1486 }
1487
1488 static int
1489 vtnet_rxeof_merged(struct vtnet_softc *sc, struct mbuf *m_head, int nbufs)
1490 {
1491         struct ifnet *ifp;
1492         struct virtqueue *vq;
1493         struct mbuf *m, *m_tail;
1494         int len;
1495
1496         ifp = sc->vtnet_ifp;
1497         vq = sc->vtnet_rx_vq;
1498         m_tail = m_head;
1499
1500         while (--nbufs > 0) {
1501                 m = virtqueue_dequeue(vq, &len);
1502                 if (m == NULL) {
1503                         ifp->if_ierrors++;
1504                         goto fail;
1505                 }
1506
1507                 if (vtnet_newbuf(sc) != 0) {
1508                         ifp->if_iqdrops++;
1509                         vtnet_discard_rxbuf(sc, m);
1510                         if (nbufs > 1)
1511                                 vtnet_discard_merged_rxbuf(sc, nbufs);
1512                         goto fail;
1513                 }
1514
1515                 if (m->m_len < len)
1516                         len = m->m_len;
1517
1518                 m->m_len = len;
1519                 m->m_flags &= ~M_PKTHDR;
1520
1521                 m_head->m_pkthdr.len += len;
1522                 m_tail->m_next = m;
1523                 m_tail = m;
1524         }
1525
1526         return (0);
1527
1528 fail:
1529         sc->vtnet_stats.rx_mergeable_failed++;
1530         m_freem(m_head);
1531
1532         return (1);
1533 }
1534
1535 static int
1536 vtnet_rxeof(struct vtnet_softc *sc, int count, int *rx_npktsp)
1537 {
1538         struct virtio_net_hdr lhdr;
1539         struct ifnet *ifp;
1540         struct virtqueue *vq;
1541         struct mbuf *m;
1542         struct ether_header *eh;
1543         struct virtio_net_hdr *hdr;
1544         struct virtio_net_hdr_mrg_rxbuf *mhdr;
1545         int len, deq, nbufs, adjsz, rx_npkts;
1546
1547         ifp = sc->vtnet_ifp;
1548         vq = sc->vtnet_rx_vq;
1549         hdr = &lhdr;
1550         deq = 0;
1551         rx_npkts = 0;
1552
1553         ASSERT_SERIALIZED(&sc->vtnet_slz);
1554
1555         while (--count >= 0) {
1556                 m = virtqueue_dequeue(vq, &len);
1557                 if (m == NULL)
1558                         break;
1559                 deq++;
1560
1561                 if (len < sc->vtnet_hdr_size + ETHER_HDR_LEN) {
1562                         ifp->if_ierrors++;
1563                         vtnet_discard_rxbuf(sc, m);
1564                         continue;
1565                 }
1566
1567                 if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1568                         nbufs = 1;
1569                         adjsz = sizeof(struct vtnet_rx_header);
1570                         /*
1571                          * Account for our pad between the header and
1572                          * the actual start of the frame.
1573                          */
1574                         len += VTNET_RX_HEADER_PAD;
1575                 } else {
1576                         mhdr = mtod(m, struct virtio_net_hdr_mrg_rxbuf *);
1577                         nbufs = mhdr->num_buffers;
1578                         adjsz = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1579                 }
1580
1581                 if (vtnet_replace_rxbuf(sc, m, len) != 0) {
1582                         ifp->if_iqdrops++;
1583                         vtnet_discard_rxbuf(sc, m);
1584                         if (nbufs > 1)
1585                                 vtnet_discard_merged_rxbuf(sc, nbufs);
1586                         continue;
1587                 }
1588
1589                 m->m_pkthdr.len = len;
1590                 m->m_pkthdr.rcvif = ifp;
1591                 m->m_pkthdr.csum_flags = 0;
1592
1593                 if (nbufs > 1) {
1594                         if (vtnet_rxeof_merged(sc, m, nbufs) != 0)
1595                                 continue;
1596                 }
1597
1598                 ifp->if_ipackets++;
1599
1600                 /*
1601                  * Save copy of header before we strip it. For both mergeable
1602                  * and non-mergeable, the VirtIO header is placed first in the
1603                  * mbuf's data. We no longer need num_buffers, so always use a
1604                  * virtio_net_hdr.
1605                  */
1606                 memcpy(hdr, mtod(m, void *), sizeof(struct virtio_net_hdr));
1607                 m_adj(m, adjsz);
1608
1609                 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1610                         eh = mtod(m, struct ether_header *);
1611                         if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
1612                                 vtnet_vlan_tag_remove(m);
1613
1614                                 /*
1615                                  * With the 802.1Q header removed, update the
1616                                  * checksum starting location accordingly.
1617                                  */
1618                                 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
1619                                         hdr->csum_start -=
1620                                             ETHER_VLAN_ENCAP_LEN;
1621                         }
1622                 }
1623
1624                 if (ifp->if_capenable & IFCAP_RXCSUM &&
1625                     hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
1626                         if (vtnet_rx_csum(sc, m, hdr) != 0)
1627                                 sc->vtnet_stats.rx_csum_failed++;
1628                 }
1629
1630                 lwkt_serialize_exit(&sc->vtnet_slz);
1631                 rx_npkts++;
1632                 (*ifp->if_input)(ifp, m);
1633                 lwkt_serialize_enter(&sc->vtnet_slz);
1634
1635                 /*
1636                  * The interface may have been stopped while we were
1637                  * passing the packet up the network stack.
1638                  */
1639                 if ((ifp->if_flags & IFF_RUNNING) == 0)
1640                         break;
1641         }
1642
1643         virtqueue_notify(vq, &sc->vtnet_slz);
1644
1645         if (rx_npktsp != NULL)
1646                 *rx_npktsp = rx_npkts;
1647
1648         return (count > 0 ? 0 : EAGAIN);
1649 }
1650
1651 static void
1652 vtnet_rx_intr_task(void *arg)
1653 {
1654         struct vtnet_softc *sc;
1655         struct ifnet *ifp;
1656         int more;
1657
1658         sc = arg;
1659         ifp = sc->vtnet_ifp;
1660
1661 //      lwkt_serialize_enter(&sc->vtnet_slz);
1662
1663         if ((ifp->if_flags & IFF_RUNNING) == 0) {
1664                 vtnet_enable_rx_intr(sc);
1665 //              lwkt_serialize_exit(&sc->vtnet_slz);
1666                 return;
1667         }
1668
1669         more = vtnet_rxeof(sc, sc->vtnet_rx_process_limit, NULL);
1670         if (!more && vtnet_enable_rx_intr(sc) != 0) {
1671                 vtnet_disable_rx_intr(sc);
1672                 more = 1;
1673         }
1674
1675 //      lwkt_serialize_exit(&sc->vtnet_slz);
1676
1677         if (more) {
1678                 sc->vtnet_stats.rx_task_rescheduled++;
1679                 /* XXX: Loop?! */
1680                 vtnet_rx_intr_task(sc);
1681         }
1682 }
1683
1684 static int
1685 vtnet_rx_vq_intr(void *xsc)
1686 {
1687         struct vtnet_softc *sc;
1688
1689         sc = xsc;
1690
1691         vtnet_disable_rx_intr(sc);
1692         vtnet_rx_intr_task(sc);
1693
1694         return (1);
1695 }
1696
1697 static void
1698 vtnet_txeof(struct vtnet_softc *sc)
1699 {
1700         struct virtqueue *vq;
1701         struct ifnet *ifp;
1702         struct vtnet_tx_header *txhdr;
1703         int deq;
1704
1705         vq = sc->vtnet_tx_vq;
1706         ifp = sc->vtnet_ifp;
1707         deq = 0;
1708
1709         ASSERT_SERIALIZED(&sc->vtnet_slz);
1710
1711         while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) {
1712                 deq++;
1713                 ifp->if_opackets++;
1714                 m_freem(txhdr->vth_mbuf);
1715         }
1716
1717         if (deq > 0) {
1718                 ifq_clr_oactive(&ifp->if_snd);
1719                 if (virtqueue_empty(vq))
1720                         sc->vtnet_watchdog_timer = 0;
1721         }
1722 }
1723
1724 static struct mbuf *
1725 vtnet_tx_offload(struct vtnet_softc *sc, struct mbuf *m,
1726     struct virtio_net_hdr *hdr)
1727 {
1728         struct ifnet *ifp;
1729         struct ether_header *eh;
1730         struct ether_vlan_header *evh;
1731         struct ip *ip;
1732         struct ip6_hdr *ip6;
1733         struct tcphdr *tcp;
1734         int ip_offset;
1735         uint16_t eth_type, csum_start;
1736         uint8_t ip_proto, gso_type;
1737
1738         ifp = sc->vtnet_ifp;
1739         M_ASSERTPKTHDR(m);
1740
1741         ip_offset = sizeof(struct ether_header);
1742         if (m->m_len < ip_offset) {
1743                 if ((m = m_pullup(m, ip_offset)) == NULL)
1744                         return (NULL);
1745         }
1746
1747         eh = mtod(m, struct ether_header *);
1748         eth_type = ntohs(eh->ether_type);
1749         if (eth_type == ETHERTYPE_VLAN) {
1750                 ip_offset = sizeof(struct ether_vlan_header);
1751                 if (m->m_len < ip_offset) {
1752                         if ((m = m_pullup(m, ip_offset)) == NULL)
1753                                 return (NULL);
1754                 }
1755                 evh = mtod(m, struct ether_vlan_header *);
1756                 eth_type = ntohs(evh->evl_proto);
1757         }
1758
1759         switch (eth_type) {
1760         case ETHERTYPE_IP:
1761                 if (m->m_len < ip_offset + sizeof(struct ip)) {
1762                         m = m_pullup(m, ip_offset + sizeof(struct ip));
1763                         if (m == NULL)
1764                                 return (NULL);
1765                 }
1766
1767                 ip = (struct ip *)(mtod(m, uint8_t *) + ip_offset);
1768                 ip_proto = ip->ip_p;
1769                 csum_start = ip_offset + (ip->ip_hl << 2);
1770                 gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
1771                 break;
1772
1773         case ETHERTYPE_IPV6:
1774                 if (m->m_len < ip_offset + sizeof(struct ip6_hdr)) {
1775                         m = m_pullup(m, ip_offset + sizeof(struct ip6_hdr));
1776                         if (m == NULL)
1777                                 return (NULL);
1778                 }
1779
1780                 ip6 = (struct ip6_hdr *)(mtod(m, uint8_t *) + ip_offset);
1781                 /*
1782                  * XXX Assume no extension headers are present. Presently,
1783                  * this will always be true in the case of TSO, and FreeBSD
1784                  * does not perform checksum offloading of IPv6 yet.
1785                  */
1786                 ip_proto = ip6->ip6_nxt;
1787                 csum_start = ip_offset + sizeof(struct ip6_hdr);
1788                 gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
1789                 break;
1790
1791         default:
1792                 return (m);
1793         }
1794
1795         if (m->m_pkthdr.csum_flags & VTNET_CSUM_OFFLOAD) {
1796                 hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM;
1797                 hdr->csum_start = csum_start;
1798                 hdr->csum_offset = m->m_pkthdr.csum_data;
1799
1800                 sc->vtnet_stats.tx_csum_offloaded++;
1801         }
1802
1803         if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1804                 if (ip_proto != IPPROTO_TCP)
1805                         return (m);
1806
1807                 if (m->m_len < csum_start + sizeof(struct tcphdr)) {
1808                         m = m_pullup(m, csum_start + sizeof(struct tcphdr));
1809                         if (m == NULL)
1810                                 return (NULL);
1811                 }
1812
1813                 tcp = (struct tcphdr *)(mtod(m, uint8_t *) + csum_start);
1814                 hdr->gso_type = gso_type;
1815                 hdr->hdr_len = csum_start + (tcp->th_off << 2);
1816                 hdr->gso_size = m->m_pkthdr.tso_segsz;
1817
1818                 if (tcp->th_flags & TH_CWR) {
1819                         /*
1820                          * Drop if we did not negotiate VIRTIO_NET_F_HOST_ECN.
1821                          * ECN support is only configurable globally with the
1822                          * net.inet.tcp.ecn.enable sysctl knob.
1823                          */
1824                         if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) {
1825                                 if_printf(ifp, "TSO with ECN not supported "
1826                                     "by host\n");
1827                                 m_freem(m);
1828                                 return (NULL);
1829                         }
1830
1831                         hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN;
1832                 }
1833
1834                 sc->vtnet_stats.tx_tso_offloaded++;
1835         }
1836
1837         return (m);
1838 }
1839
1840 static int
1841 vtnet_enqueue_txbuf(struct vtnet_softc *sc, struct mbuf **m_head,
1842     struct vtnet_tx_header *txhdr)
1843 {
1844         struct sglist sg;
1845         struct sglist_seg segs[VTNET_MAX_TX_SEGS];
1846         struct virtqueue *vq;
1847         struct mbuf *m;
1848         int collapsed, error;
1849
1850         vq = sc->vtnet_tx_vq;
1851         m = *m_head;
1852         collapsed = 0;
1853
1854         sglist_init(&sg, VTNET_MAX_TX_SEGS, segs);
1855         error = sglist_append(&sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size);
1856         KASSERT(error == 0 && sg.sg_nseg == 1,
1857             ("cannot add header to sglist"));
1858
1859 again:
1860         error = sglist_append_mbuf(&sg, m);
1861         if (error) {
1862                 if (collapsed)
1863                         goto fail;
1864
1865                 //m = m_collapse(m, MB_DONTWAIT, VTNET_MAX_TX_SEGS - 1);
1866                 m = m_defrag(m, MB_DONTWAIT);
1867                 if (m == NULL)
1868                         goto fail;
1869
1870                 *m_head = m;
1871                 collapsed = 1;
1872                 goto again;
1873         }
1874
1875         txhdr->vth_mbuf = m;
1876
1877         return (virtqueue_enqueue(vq, txhdr, &sg, sg.sg_nseg, 0));
1878
1879 fail:
1880         m_freem(*m_head);
1881         *m_head = NULL;
1882
1883         return (ENOBUFS);
1884 }
1885
1886 static struct mbuf *
1887 vtnet_vlan_tag_insert(struct mbuf *m)
1888 {
1889         struct mbuf *n;
1890         struct ether_vlan_header *evl;
1891
1892         if (M_WRITABLE(m) == 0) {
1893                 n = m_dup(m, MB_DONTWAIT);
1894                 m_freem(m);
1895                 if ((m = n) == NULL)
1896                         return (NULL);
1897         }
1898
1899         M_PREPEND(m, ETHER_VLAN_ENCAP_LEN, MB_DONTWAIT);
1900         if (m == NULL)
1901                 return (NULL);
1902         if (m->m_len < sizeof(struct ether_vlan_header)) {
1903                 m = m_pullup(m, sizeof(struct ether_vlan_header));
1904                 if (m == NULL)
1905                         return (NULL);
1906         }
1907
1908         /* Insert 802.1Q header into the existing Ethernet header. */
1909         evl = mtod(m, struct ether_vlan_header *);
1910         bcopy((char *) evl + ETHER_VLAN_ENCAP_LEN,
1911               (char *) evl, ETHER_HDR_LEN - ETHER_TYPE_LEN);
1912         evl->evl_encap_proto = htons(ETHERTYPE_VLAN);
1913         evl->evl_tag = htons(m->m_pkthdr.ether_vlantag);
1914         m->m_flags &= ~M_VLANTAG;
1915
1916         return (m);
1917 }
1918
1919 static int
1920 vtnet_encap(struct vtnet_softc *sc, struct mbuf **m_head)
1921 {
1922         struct vtnet_tx_header *txhdr;
1923         struct virtio_net_hdr *hdr;
1924         struct mbuf *m;
1925         int error;
1926
1927         txhdr = &sc->vtnet_txhdrarea[sc->vtnet_txhdridx];
1928         memset(txhdr, 0, sizeof(struct vtnet_tx_header));
1929
1930         /*
1931          * Always use the non-mergeable header to simplify things. When
1932          * the mergeable feature is negotiated, the num_buffers field
1933          * must be set to zero. We use vtnet_hdr_size later to enqueue
1934          * the correct header size to the host.
1935          */
1936         hdr = &txhdr->vth_uhdr.hdr;
1937         m = *m_head;
1938
1939         error = ENOBUFS;
1940
1941         if (m->m_flags & M_VLANTAG) {
1942                 //m = ether_vlanencap(m, m->m_pkthdr.ether_vtag);
1943                 m = vtnet_vlan_tag_insert(m);
1944                 if ((*m_head = m) == NULL)
1945                         goto fail;
1946                 m->m_flags &= ~M_VLANTAG;
1947         }
1948
1949         if (m->m_pkthdr.csum_flags != 0) {
1950                 m = vtnet_tx_offload(sc, m, hdr);
1951                 if ((*m_head = m) == NULL)
1952                         goto fail;
1953         }
1954
1955         error = vtnet_enqueue_txbuf(sc, m_head, txhdr);
1956         if (error == 0)
1957                 sc->vtnet_txhdridx =
1958                     (sc->vtnet_txhdridx + 1) % ((sc->vtnet_tx_size / 2) + 1);
1959 fail:
1960         return (error);
1961 }
1962
1963 static void
1964 vtnet_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
1965 {
1966         struct vtnet_softc *sc;
1967
1968         sc = ifp->if_softc;
1969
1970         ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq);
1971         lwkt_serialize_enter(&sc->vtnet_slz);
1972         vtnet_start_locked(ifp, ifsq);
1973         lwkt_serialize_exit(&sc->vtnet_slz);
1974 }
1975
1976 static void
1977 vtnet_start_locked(struct ifnet *ifp, struct ifaltq_subque *ifsq)
1978 {
1979         struct vtnet_softc *sc;
1980         struct virtqueue *vq;
1981         struct mbuf *m0;
1982         int enq;
1983
1984         sc = ifp->if_softc;
1985         vq = sc->vtnet_tx_vq;
1986         enq = 0;
1987
1988         ASSERT_SERIALIZED(&sc->vtnet_slz);
1989
1990         if ((ifp->if_flags & (IFF_RUNNING)) !=
1991             IFF_RUNNING || ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0))
1992                 return;
1993
1994 #ifdef VTNET_TX_INTR_MODERATION
1995         if (virtqueue_nused(vq) >= sc->vtnet_tx_size / 2)
1996                 vtnet_txeof(sc);
1997 #endif
1998
1999         while (!ifsq_is_empty(ifsq)) {
2000                 if (virtqueue_full(vq)) {
2001                         ifq_set_oactive(&ifp->if_snd);
2002                         break;
2003                 }
2004
2005                 m0 = ifq_dequeue(&ifp->if_snd);
2006                 if (m0 == NULL)
2007                         break;
2008
2009                 if (vtnet_encap(sc, &m0) != 0) {
2010                         if (m0 == NULL)
2011                                 break;
2012                         ifq_prepend(&ifp->if_snd, m0);
2013                         ifq_set_oactive(&ifp->if_snd);
2014                         break;
2015                 }
2016
2017                 enq++;
2018                 ETHER_BPF_MTAP(ifp, m0);
2019         }
2020
2021         if (enq > 0) {
2022                 virtqueue_notify(vq, &sc->vtnet_slz);
2023                 sc->vtnet_watchdog_timer = VTNET_WATCHDOG_TIMEOUT;
2024         }
2025 }
2026
2027 static void
2028 vtnet_tick(void *xsc)
2029 {
2030         struct vtnet_softc *sc;
2031
2032         sc = xsc;
2033
2034 #if 0
2035         ASSERT_SERIALIZED(&sc->vtnet_slz);
2036 #ifdef VTNET_DEBUG
2037         virtqueue_dump(sc->vtnet_rx_vq);
2038         virtqueue_dump(sc->vtnet_tx_vq);
2039 #endif
2040
2041         vtnet_watchdog(sc);
2042         callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc);
2043 #endif
2044 }
2045
2046 static void
2047 vtnet_tx_intr_task(void *arg)
2048 {
2049         struct vtnet_softc *sc;
2050         struct ifnet *ifp;
2051         struct ifaltq_subque *ifsq;
2052
2053         sc = arg;
2054         ifp = sc->vtnet_ifp;
2055         ifsq = ifq_get_subq_default(&ifp->if_snd);
2056
2057 //      lwkt_serialize_enter(&sc->vtnet_slz);
2058
2059         if ((ifp->if_flags & IFF_RUNNING) == 0) {
2060                 vtnet_enable_tx_intr(sc);
2061 //              lwkt_serialize_exit(&sc->vtnet_slz);
2062                 return;
2063         }
2064
2065         vtnet_txeof(sc);
2066
2067         if (!ifsq_is_empty(ifsq))
2068                 vtnet_start_locked(ifp, ifsq);
2069
2070         if (vtnet_enable_tx_intr(sc) != 0) {
2071                 vtnet_disable_tx_intr(sc);
2072                 sc->vtnet_stats.tx_task_rescheduled++;
2073 //              lwkt_serialize_exit(&sc->vtnet_slz);
2074                 vtnet_tx_intr_task(sc);
2075                 /* XXX: loop?! */
2076                 return;
2077         }
2078
2079 //      lwkt_serialize_exit(&sc->vtnet_slz);
2080 }
2081
2082 static int
2083 vtnet_tx_vq_intr(void *xsc)
2084 {
2085         struct vtnet_softc *sc;
2086
2087         sc = xsc;
2088
2089         vtnet_disable_tx_intr(sc);
2090         vtnet_tx_intr_task(sc);
2091
2092         return (1);
2093 }
2094
2095 static void
2096 vtnet_stop(struct vtnet_softc *sc)
2097 {
2098         device_t dev;
2099         struct ifnet *ifp;
2100
2101         dev = sc->vtnet_dev;
2102         ifp = sc->vtnet_ifp;
2103
2104         ASSERT_SERIALIZED(&sc->vtnet_slz);
2105
2106         sc->vtnet_watchdog_timer = 0;
2107         callout_stop(&sc->vtnet_tick_ch);
2108         ifq_clr_oactive(&ifp->if_snd);
2109         ifp->if_flags &= ~(IFF_RUNNING);
2110
2111         vtnet_disable_rx_intr(sc);
2112         vtnet_disable_tx_intr(sc);
2113
2114         /*
2115          * Stop the host VirtIO adapter. Note this will reset the host
2116          * adapter's state back to the pre-initialized state, so in
2117          * order to make the device usable again, we must drive it
2118          * through virtio_reinit() and virtio_reinit_complete().
2119          */
2120         virtio_stop(dev);
2121
2122         sc->vtnet_flags &= ~VTNET_FLAG_LINK;
2123
2124         vtnet_free_rx_mbufs(sc);
2125         vtnet_free_tx_mbufs(sc);
2126 }
2127
2128 static int
2129 vtnet_reinit(struct vtnet_softc *sc)
2130 {
2131         struct ifnet *ifp;
2132         uint64_t features;
2133
2134         ifp = sc->vtnet_ifp;
2135         features = sc->vtnet_features;
2136
2137         /*
2138          * Re-negotiate with the host, removing any disabled receive
2139          * features. Transmit features are disabled only on our side
2140          * via if_capenable and if_hwassist.
2141          */
2142
2143         if (ifp->if_capabilities & IFCAP_RXCSUM) {
2144                 if ((ifp->if_capenable & IFCAP_RXCSUM) == 0)
2145                         features &= ~VIRTIO_NET_F_GUEST_CSUM;
2146         }
2147
2148         if (ifp->if_capabilities & IFCAP_LRO) {
2149                 if ((ifp->if_capenable & IFCAP_LRO) == 0)
2150                         features &= ~VTNET_LRO_FEATURES;
2151         }
2152
2153         if (ifp->if_capabilities & IFCAP_VLAN_HWFILTER) {
2154                 if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
2155                         features &= ~VIRTIO_NET_F_CTRL_VLAN;
2156         }
2157
2158         return (virtio_reinit(sc->vtnet_dev, features));
2159 }
2160
2161 static void
2162 vtnet_init_locked(struct vtnet_softc *sc)
2163 {
2164         device_t dev;
2165         struct ifnet *ifp;
2166         int error;
2167
2168         dev = sc->vtnet_dev;
2169         ifp = sc->vtnet_ifp;
2170
2171         ASSERT_SERIALIZED(&sc->vtnet_slz);
2172
2173         if (ifp->if_flags & IFF_RUNNING)
2174                 return;
2175
2176         /* Stop host's adapter, cancel any pending I/O. */
2177         vtnet_stop(sc);
2178
2179         /* Reinitialize the host device. */
2180         error = vtnet_reinit(sc);
2181         if (error) {
2182                 device_printf(dev,
2183                     "reinitialization failed, stopping device...\n");
2184                 vtnet_stop(sc);
2185                 return;
2186         }
2187
2188         /* Update host with assigned MAC address. */
2189         bcopy(IF_LLADDR(ifp), sc->vtnet_hwaddr, ETHER_ADDR_LEN);
2190         vtnet_set_hwaddr(sc);
2191
2192         ifp->if_hwassist = 0;
2193         if (ifp->if_capenable & IFCAP_TXCSUM)
2194                 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;
2195         if (ifp->if_capenable & IFCAP_TSO4)
2196                 ifp->if_hwassist |= CSUM_TSO;
2197
2198         error = vtnet_init_rx_vq(sc);
2199         if (error) {
2200                 device_printf(dev,
2201                     "cannot allocate mbufs for Rx virtqueue\n");
2202                 vtnet_stop(sc);
2203                 return;
2204         }
2205
2206         if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
2207                 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
2208                         /* Restore promiscuous and all-multicast modes. */
2209                         vtnet_rx_filter(sc);
2210
2211                         /* Restore filtered MAC addresses. */
2212                         vtnet_rx_filter_mac(sc);
2213                 }
2214
2215                 /* Restore VLAN filters. */
2216                 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
2217                         vtnet_rx_filter_vlan(sc);
2218         }
2219
2220         {
2221                 vtnet_enable_rx_intr(sc);
2222                 vtnet_enable_tx_intr(sc);
2223         }
2224
2225         ifp->if_flags |= IFF_RUNNING;
2226         ifq_clr_oactive(&ifp->if_snd);
2227
2228         virtio_reinit_complete(dev);
2229
2230         vtnet_update_link_status(sc);
2231         callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc);
2232 }
2233
2234 static void
2235 vtnet_init(void *xsc)
2236 {
2237         struct vtnet_softc *sc;
2238
2239         sc = xsc;
2240
2241         lwkt_serialize_enter(&sc->vtnet_slz);
2242         vtnet_init_locked(sc);
2243         lwkt_serialize_exit(&sc->vtnet_slz);
2244 }
2245
2246 static void
2247 vtnet_exec_ctrl_cmd(struct vtnet_softc *sc, void *cookie,
2248     struct sglist *sg, int readable, int writable)
2249 {
2250         struct virtqueue *vq;
2251         void *c;
2252
2253         vq = sc->vtnet_ctrl_vq;
2254
2255         ASSERT_SERIALIZED(&sc->vtnet_slz);
2256         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ,
2257             ("no control virtqueue"));
2258         KASSERT(virtqueue_empty(vq),
2259             ("control command already enqueued"));
2260
2261         if (virtqueue_enqueue(vq, cookie, sg, readable, writable) != 0)
2262                 return;
2263
2264         virtqueue_notify(vq, &sc->vtnet_slz);
2265
2266         /*
2267          * Poll until the command is complete. Previously, we would
2268          * sleep until the control virtqueue interrupt handler woke
2269          * us up, but dropping the VTNET_MTX leads to serialization
2270          * difficulties.
2271          *
2272          * Furthermore, it appears QEMU/KVM only allocates three MSIX
2273          * vectors. Two of those vectors are needed for the Rx and Tx
2274          * virtqueues. We do not support sharing both a Vq and config
2275          * changed notification on the same MSIX vector.
2276          */
2277         c = virtqueue_poll(vq, NULL);
2278         KASSERT(c == cookie, ("unexpected control command response"));
2279 }
2280
2281 static void
2282 vtnet_rx_filter(struct vtnet_softc *sc)
2283 {
2284         device_t dev;
2285         struct ifnet *ifp;
2286
2287         dev = sc->vtnet_dev;
2288         ifp = sc->vtnet_ifp;
2289
2290         ASSERT_SERIALIZED(&sc->vtnet_slz);
2291         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
2292             ("CTRL_RX feature not negotiated"));
2293
2294         if (vtnet_set_promisc(sc, ifp->if_flags & IFF_PROMISC) != 0)
2295                 device_printf(dev, "cannot %s promiscuous mode\n",
2296                     ifp->if_flags & IFF_PROMISC ? "enable" : "disable");
2297
2298         if (vtnet_set_allmulti(sc, ifp->if_flags & IFF_ALLMULTI) != 0)
2299                 device_printf(dev, "cannot %s all-multicast mode\n",
2300                     ifp->if_flags & IFF_ALLMULTI ? "enable" : "disable");
2301 }
2302
2303 static int
2304 vtnet_ctrl_rx_cmd(struct vtnet_softc *sc, int cmd, int on)
2305 {
2306         struct virtio_net_ctrl_hdr hdr __aligned(2);
2307         struct sglist_seg segs[3];
2308         struct sglist sg;
2309         uint8_t onoff, ack;
2310         int error;
2311
2312         if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0)
2313                 return (ENOTSUP);
2314
2315         error = 0;
2316
2317         hdr.class = VIRTIO_NET_CTRL_RX;
2318         hdr.cmd = cmd;
2319         onoff = !!on;
2320         ack = VIRTIO_NET_ERR;
2321
2322         sglist_init(&sg, 3, segs);
2323         error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
2324         error |= sglist_append(&sg, &onoff, sizeof(uint8_t));
2325         error |= sglist_append(&sg, &ack, sizeof(uint8_t));
2326         KASSERT(error == 0 && sg.sg_nseg == 3,
2327             ("error adding Rx filter message to sglist"));
2328
2329         vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
2330
2331         return (ack == VIRTIO_NET_OK ? 0 : EIO);
2332 }
2333
2334 static int
2335 vtnet_set_promisc(struct vtnet_softc *sc, int on)
2336 {
2337
2338         return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_PROMISC, on));
2339 }
2340
2341 static int
2342 vtnet_set_allmulti(struct vtnet_softc *sc, int on)
2343 {
2344
2345         return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, on));
2346 }
2347
2348 static void
2349 vtnet_rx_filter_mac(struct vtnet_softc *sc)
2350 {
2351         struct virtio_net_ctrl_hdr hdr __aligned(2);
2352         struct vtnet_mac_filter *filter;
2353         struct sglist_seg segs[4];
2354         struct sglist sg;
2355         struct ifnet *ifp;
2356         struct ifaddr *ifa;
2357         struct ifaddr_container *ifac;
2358         struct ifmultiaddr *ifma;
2359         int ucnt, mcnt, promisc, allmulti, error;
2360         uint8_t ack;
2361
2362         ifp = sc->vtnet_ifp;
2363         ucnt = 0;
2364         mcnt = 0;
2365         promisc = 0;
2366         allmulti = 0;
2367         error = 0;
2368
2369         ASSERT_SERIALIZED(&sc->vtnet_slz);
2370         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
2371             ("CTRL_RX feature not negotiated"));
2372
2373         /* Use the MAC filtering table allocated in vtnet_attach. */
2374         filter = sc->vtnet_macfilter;
2375         memset(filter, 0, sizeof(struct vtnet_mac_filter));
2376
2377         /* Unicast MAC addresses: */
2378         //if_addr_rlock(ifp);
2379         TAILQ_FOREACH(ifac, &ifp->if_addrheads[mycpuid], ifa_link) {
2380                 ifa = ifac->ifa;
2381                 if (ifa->ifa_addr->sa_family != AF_LINK)
2382                         continue;
2383                 else if (ucnt == VTNET_MAX_MAC_ENTRIES)
2384                         break;
2385
2386                 bcopy(LLADDR((struct sockaddr_dl *)ifa->ifa_addr),
2387                     &filter->vmf_unicast.macs[ucnt], ETHER_ADDR_LEN);
2388                 ucnt++;
2389         }
2390         //if_addr_runlock(ifp);
2391
2392         if (ucnt >= VTNET_MAX_MAC_ENTRIES) {
2393                 promisc = 1;
2394                 filter->vmf_unicast.nentries = 0;
2395
2396                 if_printf(ifp, "more than %d MAC addresses assigned, "
2397                     "falling back to promiscuous mode\n",
2398                     VTNET_MAX_MAC_ENTRIES);
2399         } else
2400                 filter->vmf_unicast.nentries = ucnt;
2401
2402         /* Multicast MAC addresses: */
2403         //if_maddr_rlock(ifp);
2404         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2405                 if (ifma->ifma_addr->sa_family != AF_LINK)
2406                         continue;
2407                 else if (mcnt == VTNET_MAX_MAC_ENTRIES)
2408                         break;
2409
2410                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2411                     &filter->vmf_multicast.macs[mcnt], ETHER_ADDR_LEN);
2412                 mcnt++;
2413         }
2414         //if_maddr_runlock(ifp);
2415
2416         if (mcnt >= VTNET_MAX_MAC_ENTRIES) {
2417                 allmulti = 1;
2418                 filter->vmf_multicast.nentries = 0;
2419
2420                 if_printf(ifp, "more than %d multicast MAC addresses "
2421                     "assigned, falling back to all-multicast mode\n",
2422                     VTNET_MAX_MAC_ENTRIES);
2423         } else
2424                 filter->vmf_multicast.nentries = mcnt;
2425
2426         if (promisc && allmulti)
2427                 goto out;
2428
2429         hdr.class = VIRTIO_NET_CTRL_MAC;
2430         hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
2431         ack = VIRTIO_NET_ERR;
2432
2433         sglist_init(&sg, 4, segs);
2434         error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
2435         error |= sglist_append(&sg, &filter->vmf_unicast,
2436             sizeof(struct vtnet_mac_table));
2437         error |= sglist_append(&sg, &filter->vmf_multicast,
2438             sizeof(struct vtnet_mac_table));
2439         error |= sglist_append(&sg, &ack, sizeof(uint8_t));
2440         KASSERT(error == 0 && sg.sg_nseg == 4,
2441             ("error adding MAC filtering message to sglist"));
2442
2443         vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
2444
2445         if (ack != VIRTIO_NET_OK)
2446                 if_printf(ifp, "error setting host MAC filter table\n");
2447
2448 out:
2449         if (promisc)
2450                 if (vtnet_set_promisc(sc, 1) != 0)
2451                         if_printf(ifp, "cannot enable promiscuous mode\n");
2452         if (allmulti)
2453                 if (vtnet_set_allmulti(sc, 1) != 0)
2454                         if_printf(ifp, "cannot enable all-multicast mode\n");
2455 }
2456
2457 static int
2458 vtnet_exec_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
2459 {
2460         struct virtio_net_ctrl_hdr hdr __aligned(2);
2461         struct sglist_seg segs[3];
2462         struct sglist sg;
2463         uint8_t ack;
2464         int error;
2465
2466         hdr.class = VIRTIO_NET_CTRL_VLAN;
2467         hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
2468         ack = VIRTIO_NET_ERR;
2469         error = 0;
2470
2471         sglist_init(&sg, 3, segs);
2472         error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
2473         error |= sglist_append(&sg, &tag, sizeof(uint16_t));
2474         error |= sglist_append(&sg, &ack, sizeof(uint8_t));
2475         KASSERT(error == 0 && sg.sg_nseg == 3,
2476             ("error adding VLAN control message to sglist"));
2477
2478         vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
2479
2480         return (ack == VIRTIO_NET_OK ? 0 : EIO);
2481 }
2482
2483 static void
2484 vtnet_rx_filter_vlan(struct vtnet_softc *sc)
2485 {
2486         device_t dev;
2487         uint32_t w, mask;
2488         uint16_t tag;
2489         int i, nvlans, error;
2490
2491         ASSERT_SERIALIZED(&sc->vtnet_slz);
2492         KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER,
2493             ("VLAN_FILTER feature not negotiated"));
2494
2495         dev = sc->vtnet_dev;
2496         nvlans = sc->vtnet_nvlans;
2497         error = 0;
2498
2499         /* Enable filtering for each configured VLAN. */
2500         for (i = 0; i < VTNET_VLAN_SHADOW_SIZE && nvlans > 0; i++) {
2501                 w = sc->vtnet_vlan_shadow[i];
2502                 for (mask = 1, tag = i * 32; w != 0; mask <<= 1, tag++) {
2503                         if ((w & mask) != 0) {
2504                                 w &= ~mask;
2505                                 nvlans--;
2506                                 if (vtnet_exec_vlan_filter(sc, 1, tag) != 0)
2507                                         error++;
2508                         }
2509                 }
2510         }
2511
2512         KASSERT(nvlans == 0, ("VLAN count incorrect"));
2513         if (error)
2514                 device_printf(dev, "cannot restore VLAN filter table\n");
2515 }
2516
2517 static void
2518 vtnet_set_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
2519 {
2520         struct ifnet *ifp;
2521         int idx, bit;
2522
2523         KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER,
2524             ("VLAN_FILTER feature not negotiated"));
2525
2526         if ((tag == 0) || (tag > 4095))
2527                 return;
2528
2529         ifp = sc->vtnet_ifp;
2530         idx = (tag >> 5) & 0x7F;
2531         bit = tag & 0x1F;
2532
2533         lwkt_serialize_enter(&sc->vtnet_slz);
2534
2535         /* Update shadow VLAN table. */
2536         if (add) {
2537                 sc->vtnet_nvlans++;
2538                 sc->vtnet_vlan_shadow[idx] |= (1 << bit);
2539         } else {
2540                 sc->vtnet_nvlans--;
2541                 sc->vtnet_vlan_shadow[idx] &= ~(1 << bit);
2542         }
2543
2544         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
2545                 if (vtnet_exec_vlan_filter(sc, add, tag) != 0) {
2546                         device_printf(sc->vtnet_dev,
2547                             "cannot %s VLAN %d %s the host filter table\n",
2548                             add ? "add" : "remove", tag,
2549                             add ? "to" : "from");
2550                 }
2551         }
2552
2553         lwkt_serialize_exit(&sc->vtnet_slz);
2554 }
2555
2556 static void
2557 vtnet_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
2558 {
2559
2560         if (ifp->if_softc != arg)
2561                 return;
2562
2563         vtnet_set_vlan_filter(arg, 1, tag);
2564 }
2565
2566 static void
2567 vtnet_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
2568 {
2569
2570         if (ifp->if_softc != arg)
2571                 return;
2572
2573         vtnet_set_vlan_filter(arg, 0, tag);
2574 }
2575
2576 static int
2577 vtnet_ifmedia_upd(struct ifnet *ifp)
2578 {
2579         struct vtnet_softc *sc;
2580         struct ifmedia *ifm;
2581
2582         sc = ifp->if_softc;
2583         ifm = &sc->vtnet_media;
2584
2585         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
2586                 return (EINVAL);
2587
2588         return (0);
2589 }
2590
2591 static void
2592 vtnet_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
2593 {
2594         struct vtnet_softc *sc;
2595
2596         sc = ifp->if_softc;
2597
2598         ifmr->ifm_status = IFM_AVALID;
2599         ifmr->ifm_active = IFM_ETHER;
2600
2601         lwkt_serialize_enter(&sc->vtnet_slz);
2602         if (vtnet_is_link_up(sc) != 0) {
2603                 ifmr->ifm_status |= IFM_ACTIVE;
2604                 ifmr->ifm_active |= VTNET_MEDIATYPE;
2605         } else
2606                 ifmr->ifm_active |= IFM_NONE;
2607         lwkt_serialize_exit(&sc->vtnet_slz);
2608 }
2609
2610 static void
2611 vtnet_add_statistics(struct vtnet_softc *sc)
2612 {
2613         device_t dev;
2614         struct vtnet_statistics *stats;
2615         //struct sysctl_ctx_list *ctx;
2616         //struct sysctl_oid *tree;
2617         //struct sysctl_oid_list *child;
2618         int error = 0;
2619
2620         dev = sc->vtnet_dev;
2621         stats = &sc->vtnet_stats;
2622         sysctl_ctx_init(&sc->vtnet_sysctl_ctx);
2623         sc->vtnet_sysctl_tree = SYSCTL_ADD_NODE(&sc->vtnet_sysctl_ctx,
2624                                                 SYSCTL_STATIC_CHILDREN(_hw),
2625                                                 OID_AUTO,
2626                                                 device_get_nameunit(dev),
2627                                                 CTLFLAG_RD, 0, "");
2628
2629         if (sc->vtnet_sysctl_tree == NULL) {
2630                 device_printf(dev, "can't add sysctl node\n");
2631                 error = ENXIO;
2632         }
2633
2634
2635         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx,
2636                          SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO,
2637                          "mbuf_alloc_failed", CTLFLAG_RD, &stats->mbuf_alloc_failed,
2638                          "Mbuf cluster allocation failures");
2639         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx,
2640                          SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO,
2641                          "rx_frame_too_large", CTLFLAG_RD, &stats->rx_frame_too_large,
2642                          "Received frame larger than the mbuf chain");
2643         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx,SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_enq_replacement_failed",
2644             CTLFLAG_RD, &stats->rx_enq_replacement_failed,
2645             "Enqueuing the replacement receive mbuf failed");
2646         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_mergeable_failed",
2647             CTLFLAG_RD, &stats->rx_mergeable_failed,
2648             "Mergeable buffers receive failures");
2649         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_bad_ethtype",
2650             CTLFLAG_RD, &stats->rx_csum_bad_ethtype,
2651             "Received checksum offloaded buffer with unsupported "
2652             "Ethernet type");
2653         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_bad_start",
2654             CTLFLAG_RD, &stats->rx_csum_bad_start,
2655             "Received checksum offloaded buffer with incorrect start offset");
2656         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_bad_ipproto",
2657             CTLFLAG_RD, &stats->rx_csum_bad_ipproto,
2658             "Received checksum offloaded buffer with incorrect IP protocol");
2659         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_bad_offset",
2660             CTLFLAG_RD, &stats->rx_csum_bad_offset,
2661             "Received checksum offloaded buffer with incorrect offset");
2662         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_failed",
2663             CTLFLAG_RD, &stats->rx_csum_failed,
2664             "Received buffer checksum offload failed");
2665         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_offloaded",
2666             CTLFLAG_RD, &stats->rx_csum_offloaded,
2667             "Received buffer checksum offload succeeded");
2668         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_task_rescheduled",
2669             CTLFLAG_RD, &stats->rx_task_rescheduled,
2670             "Times the receive interrupt task rescheduled itself");
2671
2672         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_csum_offloaded",
2673             CTLFLAG_RD, &stats->tx_csum_offloaded,
2674             "Offloaded checksum of transmitted buffer");
2675         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_tso_offloaded",
2676             CTLFLAG_RD, &stats->tx_tso_offloaded,
2677             "Segmentation offload of transmitted buffer");
2678         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_csum_bad_ethtype",
2679             CTLFLAG_RD, &stats->tx_csum_bad_ethtype,
2680             "Aborted transmit of checksum offloaded buffer with unknown "
2681             "Ethernet type");
2682         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_tso_bad_ethtype",
2683             CTLFLAG_RD, &stats->tx_tso_bad_ethtype,
2684             "Aborted transmit of TSO buffer with unknown Ethernet type");
2685         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_task_rescheduled",
2686             CTLFLAG_RD, &stats->tx_task_rescheduled,
2687             "Times the transmit interrupt task rescheduled itself");
2688 }
2689
2690 static int
2691 vtnet_enable_rx_intr(struct vtnet_softc *sc)
2692 {
2693
2694         return (virtqueue_enable_intr(sc->vtnet_rx_vq));
2695 }
2696
2697 static void
2698 vtnet_disable_rx_intr(struct vtnet_softc *sc)
2699 {
2700
2701         virtqueue_disable_intr(sc->vtnet_rx_vq);
2702 }
2703
2704 static int
2705 vtnet_enable_tx_intr(struct vtnet_softc *sc)
2706 {
2707
2708 #ifdef VTNET_TX_INTR_MODERATION
2709         return (0);
2710 #else
2711         return (virtqueue_enable_intr(sc->vtnet_tx_vq));
2712 #endif
2713 }
2714
2715 static void
2716 vtnet_disable_tx_intr(struct vtnet_softc *sc)
2717 {
2718
2719         virtqueue_disable_intr(sc->vtnet_tx_vq);
2720 }