sys/dev/virtual/virtio/net/if_vtnet.c

   1 /*-
   2  * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice unmodified, this list of conditions, and the following
  10  *    disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 /* Driver for VirtIO network devices. */
  28
  29 #include <sys/cdefs.h>
  30
  31 #include <sys/param.h>
  32 #include <sys/systm.h>
  33 #include <sys/kernel.h>
  34 #include <sys/sockio.h>
  35 #include <sys/mbuf.h>
  36 #include <sys/malloc.h>
  37 #include <sys/module.h>
  38 #include <sys/socket.h>
  39 #include <sys/sysctl.h>
  40 #include <sys/taskqueue.h>
  41 #include <sys/random.h>
  42 #include <sys/sglist.h>
  43 #include <sys/serialize.h>
  44 #include <sys/bus.h>
  45 #include <sys/rman.h>
  46
  47 #include <machine/limits.h>
  48
  49 #include <net/ethernet.h>
  50 #include <net/if.h>
  51 #include <net/if_arp.h>
  52 #include <net/if_dl.h>
  53 #include <net/if_types.h>
  54 #include <net/if_media.h>
  55 #include <net/vlan/if_vlan_var.h>
  56 #include <net/vlan/if_vlan_ether.h>
  57 #include <net/ifq_var.h>
  58
  59 #include <net/bpf.h>
  60
  61 #include <netinet/in_systm.h>
  62 #include <netinet/in.h>
  63 #include <netinet/ip.h>
  64 #include <netinet/ip6.h>
  65 #include <netinet/udp.h>
  66 #include <netinet/tcp.h>
  67
  68 #include <dev/virtual/virtio/virtio/virtio.h>
  69 #include <dev/virtual/virtio/virtio/virtqueue.h>
  70
  71 #include "virtio_net.h"
  72 #include "virtio_if.h"
  73
  74 struct vtnet_statistics {
  75         uint64_t        mbuf_alloc_failed;
  76
  77         uint64_t        rx_frame_too_large;
  78         uint64_t        rx_enq_replacement_failed;
  79         uint64_t        rx_mergeable_failed;
  80         uint64_t        rx_csum_bad_ethtype;
  81         uint64_t        rx_csum_bad_ipproto;
  82         uint64_t        rx_csum_bad_offset;
  83         uint64_t        rx_csum_failed;
  84         uint64_t        rx_csum_offloaded;
  85         uint64_t        rx_task_rescheduled;
  86
  87         uint64_t        tx_csum_offloaded;
  88         uint64_t        tx_tso_offloaded;
  89         uint64_t        tx_csum_bad_ethtype;
  90         uint64_t        tx_tso_bad_ethtype;
  91         uint64_t        tx_task_rescheduled;
  92 };
  93
  94 struct vtnet_softc {
  95         device_t                vtnet_dev;
  96         struct ifnet            *vtnet_ifp;
  97         struct lwkt_serialize   vtnet_slz;
  98
  99         uint32_t                vtnet_flags;
 100 #define VTNET_FLAG_LINK         0x0001
 101 #define VTNET_FLAG_SUSPENDED    0x0002
 102 #define VTNET_FLAG_MAC          0x0004
 103 #define VTNET_FLAG_CTRL_VQ      0x0008
 104 #define VTNET_FLAG_CTRL_RX      0x0010
 105 #define VTNET_FLAG_CTRL_MAC     0x0020
 106 #define VTNET_FLAG_VLAN_FILTER  0x0040
 107 #define VTNET_FLAG_TSO_ECN      0x0080
 108 #define VTNET_FLAG_MRG_RXBUFS   0x0100
 109 #define VTNET_FLAG_LRO_NOMRG    0x0200
 110
 111         struct virtqueue        *vtnet_rx_vq;
 112         struct virtqueue        *vtnet_tx_vq;
 113         struct virtqueue        *vtnet_ctrl_vq;
 114
 115         struct vtnet_tx_header  *vtnet_txhdrarea;
 116         uint32_t                vtnet_txhdridx;
 117         struct vtnet_mac_filter *vtnet_macfilter;
 118
 119         int                     vtnet_hdr_size;
 120         int                     vtnet_tx_size;
 121         int                     vtnet_rx_size;
 122         int                     vtnet_rx_process_limit;
 123         int                     vtnet_rx_mbuf_size;
 124         int                     vtnet_rx_mbuf_count;
 125         int                     vtnet_if_flags;
 126         int                     vtnet_watchdog_timer;
 127         uint64_t                vtnet_features;
 128
 129         struct task             vtnet_cfgchg_task;
 130
 131         struct vtnet_statistics vtnet_stats;
 132
 133         struct callout          vtnet_tick_ch;
 134
 135         eventhandler_tag        vtnet_vlan_attach;
 136         eventhandler_tag        vtnet_vlan_detach;
 137
 138         struct ifmedia          vtnet_media;
 139         /*
 140          * Fake media type; the host does not provide us with
 141          * any real media information.
 142          */
 143 #define VTNET_MEDIATYPE         (IFM_ETHER | IFM_1000_T | IFM_FDX)
 144         char                    vtnet_hwaddr[ETHER_ADDR_LEN];
 145
 146         /*
 147          * During reset, the host's VLAN filtering table is lost. The
 148          * array below is used to restore all the VLANs configured on
 149          * this interface after a reset.
 150          */
 151 #define VTNET_VLAN_SHADOW_SIZE  (4096 / 32)
 152         int                     vtnet_nvlans;
 153         uint32_t                vtnet_vlan_shadow[VTNET_VLAN_SHADOW_SIZE];
 154
 155         char                    vtnet_mtx_name[16];
 156 };
 157
 158 /*
 159  * When mergeable buffers are not negotiated, the vtnet_rx_header structure
 160  * below is placed at the beginning of the mbuf data. Use 4 bytes of pad to
 161  * both keep the VirtIO header and the data non-contiguous and to keep the
 162  * frame's payload 4 byte aligned.
 163  *
 164  * When mergeable buffers are negotiated, the host puts the VirtIO header in
 165  * the beginning of the first mbuf's data.
 166  */
 167 #define VTNET_RX_HEADER_PAD     4
 168 struct vtnet_rx_header {
 169         struct virtio_net_hdr   vrh_hdr;
 170         char                    vrh_pad[VTNET_RX_HEADER_PAD];
 171 } __packed;
 172
 173 /*
 174  * For each outgoing frame, the vtnet_tx_header below is allocated from
 175  * the vtnet_tx_header_zone.
 176  */
 177 struct vtnet_tx_header {
 178         union {
 179                 struct virtio_net_hdr           hdr;
 180                 struct virtio_net_hdr_mrg_rxbuf mhdr;
 181         } vth_uhdr;
 182
 183         struct mbuf             *vth_mbuf;
 184 };
 185
 186 MALLOC_DEFINE(M_VTNET, "VTNET_TX", "Outgoing VTNET TX frame header");
 187
 188 /*
 189  * The VirtIO specification does not place a limit on the number of MAC
 190  * addresses the guest driver may request to be filtered. In practice,
 191  * the host is constrained by available resources. To simplify this driver,
 192  * impose a reasonably high limit of MAC addresses we will filter before
 193  * falling back to promiscuous or all-multicast modes.
 194  */
 195 #define VTNET_MAX_MAC_ENTRIES   128
 196
 197 struct vtnet_mac_table {
 198         uint32_t                nentries;
 199         uint8_t                 macs[VTNET_MAX_MAC_ENTRIES][ETHER_ADDR_LEN];
 200 } __packed;
 201
 202 struct vtnet_mac_filter {
 203         struct vtnet_mac_table  vmf_unicast;
 204         uint32_t                vmf_pad; /* Make tables non-contiguous. */
 205         struct vtnet_mac_table  vmf_multicast;
 206 };
 207
 208 #define VTNET_WATCHDOG_TIMEOUT  5
 209 #define VTNET_CSUM_OFFLOAD      (CSUM_TCP | CSUM_UDP)
 210
 211 /* Features desired/implemented by this driver. */
 212 #define VTNET_FEATURES          \
 213     (VIRTIO_NET_F_MAC           | \
 214      VIRTIO_NET_F_STATUS        | \
 215      VIRTIO_NET_F_CTRL_VQ       | \
 216      VIRTIO_NET_F_CTRL_RX       | \
 217      VIRTIO_NET_F_CTRL_MAC_ADDR | \
 218      VIRTIO_NET_F_CTRL_VLAN     | \
 219      VIRTIO_NET_F_CSUM          | \
 220      VIRTIO_NET_F_HOST_TSO4     | \
 221      VIRTIO_NET_F_HOST_TSO6     | \
 222      VIRTIO_NET_F_HOST_ECN      | \
 223      VIRTIO_NET_F_GUEST_CSUM    | \
 224      VIRTIO_NET_F_GUEST_TSO4    | \
 225      VIRTIO_NET_F_GUEST_TSO6    | \
 226      VIRTIO_NET_F_GUEST_ECN     | \
 227      VIRTIO_NET_F_MRG_RXBUF)
 228
 229 /*
 230  * The VIRTIO_NET_F_GUEST_TSO[46] features permit the host to send us
 231  * frames larger than 1514 bytes. We do not yet support software LRO
 232  * via tcp_lro_rx().
 233  */
 234 #define VTNET_LRO_FEATURES (VIRTIO_NET_F_GUEST_TSO4 | \
 235                             VIRTIO_NET_F_GUEST_TSO6 | VIRTIO_NET_F_GUEST_ECN)
 236
 237 #define VTNET_MAX_MTU           65536
 238 #define VTNET_MAX_RX_SIZE       65550
 239
 240 /*
 241  * Used to preallocate the Vq indirect descriptors. The first segment
 242  * is reserved for the header.
 243  */
 244 #define VTNET_MIN_RX_SEGS       2
 245 #define VTNET_MAX_RX_SEGS       34
 246 #define VTNET_MAX_TX_SEGS       34
 247
 248 #define IFCAP_LRO               0x00400 /* can do Large Receive Offload */
 249 #define IFCAP_VLAN_HWFILTER     0x10000 /* interface hw can filter vlan tag */
 250 #define IFCAP_VLAN_HWTSO        0x40000 /* can do IFCAP_TSO on VLANs */
 251
 252
 253 /*
 254  * Assert we can receive and transmit the maximum with regular
 255  * size clusters.
 256  */
 257 CTASSERT(((VTNET_MAX_RX_SEGS - 1) * MCLBYTES) >= VTNET_MAX_RX_SIZE);
 258 CTASSERT(((VTNET_MAX_TX_SEGS - 1) * MCLBYTES) >= VTNET_MAX_MTU);
 259
 260 /*
 261  * Determine how many mbufs are in each receive buffer. For LRO without
 262  * mergeable descriptors, we must allocate an mbuf chain large enough to
 263  * hold both the vtnet_rx_header and the maximum receivable data.
 264  */
 265 #define VTNET_NEEDED_RX_MBUFS(_sc)                                      \
 266         ((_sc)->vtnet_flags & VTNET_FLAG_LRO_NOMRG) == 0 ? 1 :          \
 267         howmany(sizeof(struct vtnet_rx_header) + VTNET_MAX_RX_SIZE,     \
 268         (_sc)->vtnet_rx_mbuf_size)
 269
 270 static int      vtnet_modevent(module_t, int, void *);
 271
 272 static int      vtnet_probe(device_t);
 273 static int      vtnet_attach(device_t);
 274 static int      vtnet_detach(device_t);
 275 static int      vtnet_suspend(device_t);
 276 static int      vtnet_resume(device_t);
 277 static int      vtnet_shutdown(device_t);
 278 static int      vtnet_config_change(device_t);
 279
 280 static void     vtnet_negotiate_features(struct vtnet_softc *);
 281 static int      vtnet_alloc_virtqueues(struct vtnet_softc *);
 282 static void     vtnet_get_hwaddr(struct vtnet_softc *);
 283 static void     vtnet_set_hwaddr(struct vtnet_softc *);
 284 static int      vtnet_is_link_up(struct vtnet_softc *);
 285 static void     vtnet_update_link_status(struct vtnet_softc *);
 286 #if 0
 287 static void     vtnet_watchdog(struct vtnet_softc *);
 288 #endif
 289 static void     vtnet_config_change_task(void *, int);
 290 static int      vtnet_setup_interface(struct vtnet_softc *);
 291 static int      vtnet_change_mtu(struct vtnet_softc *, int);
 292 static int      vtnet_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 293
 294 static int      vtnet_init_rx_vq(struct vtnet_softc *);
 295 static void     vtnet_free_rx_mbufs(struct vtnet_softc *);
 296 static void     vtnet_free_tx_mbufs(struct vtnet_softc *);
 297 static void     vtnet_free_ctrl_vq(struct vtnet_softc *);
 298
 299 static struct mbuf * vtnet_alloc_rxbuf(struct vtnet_softc *, int,
 300                     struct mbuf **);
 301 static int      vtnet_replace_rxbuf(struct vtnet_softc *,
 302                     struct mbuf *, int);
 303 static int      vtnet_newbuf(struct vtnet_softc *);
 304 static void     vtnet_discard_merged_rxbuf(struct vtnet_softc *, int);
 305 static void     vtnet_discard_rxbuf(struct vtnet_softc *, struct mbuf *);
 306 static int      vtnet_enqueue_rxbuf(struct vtnet_softc *, struct mbuf *);
 307 static void     vtnet_vlan_tag_remove(struct mbuf *);
 308 static int      vtnet_rx_csum(struct vtnet_softc *, struct mbuf *,
 309                     struct virtio_net_hdr *);
 310 static int      vtnet_rxeof_merged(struct vtnet_softc *, struct mbuf *, int);
 311 static int      vtnet_rxeof(struct vtnet_softc *, int, int *);
 312 static void     vtnet_rx_intr_task(void *);
 313 static int      vtnet_rx_vq_intr(void *);
 314
 315 static void     vtnet_txeof(struct vtnet_softc *);
 316 static struct mbuf * vtnet_tx_offload(struct vtnet_softc *, struct mbuf *,
 317                     struct virtio_net_hdr *);
 318 static int      vtnet_enqueue_txbuf(struct vtnet_softc *, struct mbuf **,
 319                     struct vtnet_tx_header *);
 320 static int      vtnet_encap(struct vtnet_softc *, struct mbuf **);
 321 static void     vtnet_start_locked(struct ifnet *, struct ifaltq_subque *);
 322 static void     vtnet_start(struct ifnet *, struct ifaltq_subque *);
 323 static void     vtnet_tick(void *);
 324 static void     vtnet_tx_intr_task(void *);
 325 static int      vtnet_tx_vq_intr(void *);
 326
 327 static void     vtnet_stop(struct vtnet_softc *);
 328 static int      vtnet_virtio_reinit(struct vtnet_softc *);
 329 static void     vtnet_init_locked(struct vtnet_softc *);
 330 static void     vtnet_init(void *);
 331
 332 static void     vtnet_exec_ctrl_cmd(struct vtnet_softc *, void *,
 333                     struct sglist *, int, int);
 334
 335 static int      vtnet_ctrl_mac_cmd(struct vtnet_softc *, uint8_t *);
 336 static int      vtnet_ctrl_rx_cmd(struct vtnet_softc *, int, int);
 337 static int      vtnet_set_promisc(struct vtnet_softc *, int);
 338 static int      vtnet_set_allmulti(struct vtnet_softc *, int);
 339 static void     vtnet_rx_filter(struct vtnet_softc *sc);
 340 static void     vtnet_rx_filter_mac(struct vtnet_softc *);
 341
 342 static int      vtnet_exec_vlan_filter(struct vtnet_softc *, int, uint16_t);
 343 static void     vtnet_rx_filter_vlan(struct vtnet_softc *);
 344 static void     vtnet_update_vlan_filter(struct vtnet_softc *, int, uint16_t);
 345 static void     vtnet_register_vlan(void *, struct ifnet *, uint16_t);
 346 static void     vtnet_unregister_vlan(void *, struct ifnet *, uint16_t);
 347
 348 static int      vtnet_ifmedia_upd(struct ifnet *);
 349 static void     vtnet_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 350
 351 static void     vtnet_add_statistics(struct vtnet_softc *);
 352
 353 static int      vtnet_enable_rx_intr(struct vtnet_softc *);
 354 static int      vtnet_enable_tx_intr(struct vtnet_softc *);
 355 static void     vtnet_disable_rx_intr(struct vtnet_softc *);
 356 static void     vtnet_disable_tx_intr(struct vtnet_softc *);
 357
 358 /* Tunables. */
 359 static int vtnet_csum_disable = 0;
 360 TUNABLE_INT("hw.vtnet.csum_disable", &vtnet_csum_disable);
 361 static int vtnet_tso_disable = 1;
 362 TUNABLE_INT("hw.vtnet.tso_disable", &vtnet_tso_disable);
 363 static int vtnet_lro_disable = 1;
 364 TUNABLE_INT("hw.vtnet.lro_disable", &vtnet_lro_disable);
 365
 366 /*
 367  * Reducing the number of transmit completed interrupts can
 368  * improve performance. To do so, the define below keeps the
 369  * Tx vq interrupt disabled and adds calls to vtnet_txeof()
 370  * in the start and watchdog paths. The price to pay for this
 371  * is the m_free'ing of transmitted mbufs may be delayed until
 372  * the watchdog fires.
 373  */
 374 #define VTNET_TX_INTR_MODERATION
 375
 376 static struct virtio_feature_desc vtnet_feature_desc[] = {
 377         { VIRTIO_NET_F_CSUM,            "TxChecksum"    },
 378         { VIRTIO_NET_F_GUEST_CSUM,      "RxChecksum"    },
 379         { VIRTIO_NET_F_MAC,             "MacAddress"    },
 380         { VIRTIO_NET_F_GSO,             "TxAllGSO"      },
 381         { VIRTIO_NET_F_GUEST_TSO4,      "RxTSOv4"       },
 382         { VIRTIO_NET_F_GUEST_TSO6,      "RxTSOv6"       },
 383         { VIRTIO_NET_F_GUEST_ECN,       "RxECN"         },
 384         { VIRTIO_NET_F_GUEST_UFO,       "RxUFO"         },
 385         { VIRTIO_NET_F_HOST_TSO4,       "TxTSOv4"       },
 386         { VIRTIO_NET_F_HOST_TSO6,       "TxTSOv6"       },
 387         { VIRTIO_NET_F_HOST_ECN,        "TxTSOECN"      },
 388         { VIRTIO_NET_F_HOST_UFO,        "TxUFO"         },
 389         { VIRTIO_NET_F_MRG_RXBUF,       "MrgRxBuf"      },
 390         { VIRTIO_NET_F_STATUS,          "Status"        },
 391         { VIRTIO_NET_F_CTRL_VQ,         "ControlVq"     },
 392         { VIRTIO_NET_F_CTRL_RX,         "RxMode"        },
 393         { VIRTIO_NET_F_CTRL_VLAN,       "VLanFilter"    },
 394         { VIRTIO_NET_F_CTRL_RX_EXTRA,   "RxModeExtra"   },
 395         { VIRTIO_NET_F_GUEST_ANNOUNCE,  "GuestAnnounce" },
 396         { VIRTIO_NET_F_MQ,              "RFS"           },
 397         { VIRTIO_NET_F_CTRL_MAC_ADDR,   "SetMacAddress" },
 398         { 0, NULL }
 399 };
 400
 401 static device_method_t vtnet_methods[] = {
 402         /* Device methods. */
 403         DEVMETHOD(device_probe,         vtnet_probe),
 404         DEVMETHOD(device_attach,        vtnet_attach),
 405         DEVMETHOD(device_detach,        vtnet_detach),
 406         DEVMETHOD(device_suspend,       vtnet_suspend),
 407         DEVMETHOD(device_resume,        vtnet_resume),
 408         DEVMETHOD(device_shutdown,      vtnet_shutdown),
 409
 410         /* VirtIO methods. */
 411         DEVMETHOD(virtio_config_change, vtnet_config_change),
 412
 413         { 0, 0 }
 414 };
 415
 416 static driver_t vtnet_driver = {
 417         "vtnet",
 418         vtnet_methods,
 419         sizeof(struct vtnet_softc)
 420 };
 421
 422 static devclass_t vtnet_devclass;
 423
 424 DRIVER_MODULE(vtnet, virtio_pci, vtnet_driver, vtnet_devclass,
 425     vtnet_modevent, 0);
 426 MODULE_VERSION(vtnet, 1);
 427 MODULE_DEPEND(vtnet, virtio, 1, 1, 1);
 428
 429 static int
 430 vtnet_modevent(module_t mod, int type, void *unused)
 431 {
 432         int error;
 433
 434         error = 0;
 435
 436         switch (type) {
 437         case MOD_LOAD:
 438                 break;
 439         case MOD_UNLOAD:
 440                 break;
 441         case MOD_SHUTDOWN:
 442                 break;
 443         default:
 444                 error = EOPNOTSUPP;
 445                 break;
 446         }
 447
 448         return (error);
 449 }
 450
 451 static int
 452 vtnet_probe(device_t dev)
 453 {
 454         if (virtio_get_device_type(dev) != VIRTIO_ID_NETWORK)
 455                 return (ENXIO);
 456
 457         device_set_desc(dev, "VirtIO Networking Adapter");
 458
 459         return (BUS_PROBE_DEFAULT);
 460 }
 461
 462 static int
 463 vtnet_attach(device_t dev)
 464 {
 465         struct vtnet_softc *sc;
 466         int error;
 467
 468         sc = device_get_softc(dev);
 469         sc->vtnet_dev = dev;
 470
 471         lwkt_serialize_init(&sc->vtnet_slz);
 472         callout_init(&sc->vtnet_tick_ch);
 473
 474         ifmedia_init(&sc->vtnet_media, IFM_IMASK, vtnet_ifmedia_upd,
 475                      vtnet_ifmedia_sts);
 476         ifmedia_add(&sc->vtnet_media, VTNET_MEDIATYPE, 0, NULL);
 477         ifmedia_set(&sc->vtnet_media, VTNET_MEDIATYPE);
 478
 479         vtnet_add_statistics(sc);
 480
 481         /* Register our feature descriptions. */
 482         virtio_set_feature_desc(dev, vtnet_feature_desc);
 483         vtnet_negotiate_features(sc);
 484
 485         if (virtio_with_feature(dev, VIRTIO_NET_F_MAC)) {
 486                 /* This feature should always be negotiated. */
 487                 sc->vtnet_flags |= VTNET_FLAG_MAC;
 488         }
 489
 490         if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) {
 491                 sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS;
 492                 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 493         } else {
 494                 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
 495         }
 496
 497         sc->vtnet_rx_mbuf_size = MCLBYTES;
 498         sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc);
 499
 500         if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VQ)) {
 501                 sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ;
 502
 503                 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX))
 504                         sc->vtnet_flags |= VTNET_FLAG_CTRL_RX;
 505                 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VLAN))
 506                         sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER;
 507                 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_MAC_ADDR) &&
 508                     virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX))
 509                         sc->vtnet_flags |= VTNET_FLAG_CTRL_MAC;
 510         }
 511
 512         /* Read (or generate) the MAC address for the adapter. */
 513         vtnet_get_hwaddr(sc);
 514
 515         error = vtnet_alloc_virtqueues(sc);
 516         if (error) {
 517                 device_printf(dev, "cannot allocate virtqueues\n");
 518                 goto fail;
 519         }
 520
 521         error = vtnet_setup_interface(sc);
 522         if (error) {
 523                 device_printf(dev, "cannot setup interface\n");
 524                 goto fail;
 525         }
 526
 527         TASK_INIT(&sc->vtnet_cfgchg_task, 0, vtnet_config_change_task, sc);
 528
 529         error = virtio_setup_intr(dev, &sc->vtnet_slz);
 530         if (error) {
 531                 device_printf(dev, "cannot setup virtqueue interrupts\n");
 532                 ether_ifdetach(sc->vtnet_ifp);
 533                 goto fail;
 534         }
 535
 536         /*
 537          * Device defaults to promiscuous mode for backwards
 538          * compatibility. Turn it off if possible.
 539          */
 540         if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
 541                 lwkt_serialize_enter(&sc->vtnet_slz);
 542                 if (vtnet_set_promisc(sc, 0) != 0) {
 543                         sc->vtnet_ifp->if_flags |= IFF_PROMISC;
 544                         device_printf(dev,
 545                             "cannot disable promiscuous mode\n");
 546                 }
 547                 lwkt_serialize_exit(&sc->vtnet_slz);
 548         } else
 549                 sc->vtnet_ifp->if_flags |= IFF_PROMISC;
 550
 551 fail:
 552         if (error)
 553                 vtnet_detach(dev);
 554
 555         return (error);
 556 }
 557
 558 static int
 559 vtnet_detach(device_t dev)
 560 {
 561         struct vtnet_softc *sc;
 562         struct ifnet *ifp;
 563
 564         sc = device_get_softc(dev);
 565         ifp = sc->vtnet_ifp;
 566
 567         if (device_is_attached(dev)) {
 568                 lwkt_serialize_enter(&sc->vtnet_slz);
 569                 vtnet_stop(sc);
 570                 lwkt_serialize_exit(&sc->vtnet_slz);
 571
 572                 callout_stop(&sc->vtnet_tick_ch);
 573                 taskqueue_drain(taskqueue_swi, &sc->vtnet_cfgchg_task);
 574
 575                 ether_ifdetach(ifp);
 576         }
 577
 578         if (sc->vtnet_vlan_attach != NULL) {
 579                 EVENTHANDLER_DEREGISTER(vlan_config, sc->vtnet_vlan_attach);
 580                 sc->vtnet_vlan_attach = NULL;
 581         }
 582         if (sc->vtnet_vlan_detach != NULL) {
 583                 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vtnet_vlan_detach);
 584                 sc->vtnet_vlan_detach = NULL;
 585         }
 586
 587         if (ifp) {
 588                 if_free(ifp);
 589                 sc->vtnet_ifp = NULL;
 590         }
 591
 592         if (sc->vtnet_rx_vq != NULL)
 593                 vtnet_free_rx_mbufs(sc);
 594         if (sc->vtnet_tx_vq != NULL)
 595                 vtnet_free_tx_mbufs(sc);
 596         if (sc->vtnet_ctrl_vq != NULL)
 597                 vtnet_free_ctrl_vq(sc);
 598
 599         if (sc->vtnet_txhdrarea != NULL) {
 600                 contigfree(sc->vtnet_txhdrarea,
 601                     ((sc->vtnet_tx_size / 2) + 1) *
 602                     sizeof(struct vtnet_tx_header), M_VTNET);
 603                 sc->vtnet_txhdrarea = NULL;
 604         }
 605         if (sc->vtnet_macfilter != NULL) {
 606                 contigfree(sc->vtnet_macfilter,
 607                     sizeof(struct vtnet_mac_filter), M_DEVBUF);
 608                 sc->vtnet_macfilter = NULL;
 609         }
 610
 611         ifmedia_removeall(&sc->vtnet_media);
 612
 613         return (0);
 614 }
 615
 616 static int
 617 vtnet_suspend(device_t dev)
 618 {
 619         struct vtnet_softc *sc;
 620
 621         sc = device_get_softc(dev);
 622
 623         lwkt_serialize_enter(&sc->vtnet_slz);
 624         vtnet_stop(sc);
 625         sc->vtnet_flags |= VTNET_FLAG_SUSPENDED;
 626         lwkt_serialize_exit(&sc->vtnet_slz);
 627
 628         return (0);
 629 }
 630
 631 static int
 632 vtnet_resume(device_t dev)
 633 {
 634         struct vtnet_softc *sc;
 635         struct ifnet *ifp;
 636
 637         sc = device_get_softc(dev);
 638         ifp = sc->vtnet_ifp;
 639
 640         lwkt_serialize_enter(&sc->vtnet_slz);
 641         if (ifp->if_flags & IFF_UP)
 642                 vtnet_init_locked(sc);
 643         sc->vtnet_flags &= ~VTNET_FLAG_SUSPENDED;
 644         lwkt_serialize_exit(&sc->vtnet_slz);
 645
 646         return (0);
 647 }
 648
 649 static int
 650 vtnet_shutdown(device_t dev)
 651 {
 652
 653         /*
 654          * Suspend already does all of what we need to
 655          * do here; we just never expect to be resumed.
 656          */
 657         return (vtnet_suspend(dev));
 658 }
 659
 660 static int
 661 vtnet_config_change(device_t dev)
 662 {
 663         struct vtnet_softc *sc;
 664
 665         sc = device_get_softc(dev);
 666
 667         taskqueue_enqueue(taskqueue_thread[mycpuid], &sc->vtnet_cfgchg_task);
 668
 669         return (1);
 670 }
 671
 672 static void
 673 vtnet_negotiate_features(struct vtnet_softc *sc)
 674 {
 675         device_t dev;
 676         uint64_t mask, features;
 677
 678         dev = sc->vtnet_dev;
 679         mask = 0;
 680
 681         if (vtnet_csum_disable)
 682                 mask |= VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM;
 683
 684         /*
 685          * TSO and LRO are only available when their corresponding checksum
 686          * offload feature is also negotiated.
 687          */
 688
 689         if (vtnet_csum_disable || vtnet_tso_disable)
 690                 mask |= VIRTIO_NET_F_HOST_TSO4 | VIRTIO_NET_F_HOST_TSO6 |
 691                     VIRTIO_NET_F_HOST_ECN;
 692
 693         if (vtnet_csum_disable || vtnet_lro_disable)
 694                 mask |= VTNET_LRO_FEATURES;
 695
 696         features = VTNET_FEATURES & ~mask;
 697         features |= VIRTIO_F_NOTIFY_ON_EMPTY;
 698         features |= VIRTIO_F_ANY_LAYOUT;
 699         sc->vtnet_features = virtio_negotiate_features(dev, features);
 700
 701         if (virtio_with_feature(dev, VTNET_LRO_FEATURES) &&
 702             virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF) == 0) {
 703                 /*
 704                  * LRO without mergeable buffers requires special care. This
 705                  * is not ideal because every receive buffer must be large
 706                  * enough to hold the maximum TCP packet, the Ethernet header,
 707                  * and the header. This requires up to 34 descriptors with
 708                  * MCLBYTES clusters. If we do not have indirect descriptors,
 709                  * LRO is disabled since the virtqueue will not contain very
 710                  * many receive buffers.
 711                  */
 712                 if (!virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) {
 713                         device_printf(dev,
 714                             "LRO disabled due to both mergeable buffers and "
 715                             "indirect descriptors not negotiated\n");
 716
 717                         features &= ~VTNET_LRO_FEATURES;
 718                         sc->vtnet_features =
 719                             virtio_negotiate_features(dev, features);
 720                 } else
 721                         sc->vtnet_flags |= VTNET_FLAG_LRO_NOMRG;
 722         }
 723 }
 724
 725 static int
 726 vtnet_alloc_virtqueues(struct vtnet_softc *sc)
 727 {
 728         device_t dev;
 729         struct vq_alloc_info vq_info[3];
 730         int nvqs, rxsegs;
 731
 732         dev = sc->vtnet_dev;
 733         nvqs = 2;
 734
 735         /*
 736          * Indirect descriptors are not needed for the Rx
 737          * virtqueue when mergeable buffers are negotiated.
 738          * The header is placed inline with the data, not
 739          * in a separate descriptor, and mbuf clusters are
 740          * always physically contiguous.
 741          */
 742         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
 743                 rxsegs = sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG ?
 744                     VTNET_MAX_RX_SEGS : VTNET_MIN_RX_SEGS;
 745         } else
 746                 rxsegs = 0;
 747
 748         VQ_ALLOC_INFO_INIT(&vq_info[0], rxsegs,
 749             vtnet_rx_vq_intr, sc, &sc->vtnet_rx_vq,
 750             "%s receive", device_get_nameunit(dev));
 751
 752         VQ_ALLOC_INFO_INIT(&vq_info[1], VTNET_MAX_TX_SEGS,
 753             vtnet_tx_vq_intr, sc, &sc->vtnet_tx_vq,
 754             "%s transmit", device_get_nameunit(dev));
 755
 756         if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
 757                 nvqs++;
 758
 759                 VQ_ALLOC_INFO_INIT(&vq_info[2], 0, NULL, NULL,
 760                     &sc->vtnet_ctrl_vq, "%s control",
 761                     device_get_nameunit(dev));
 762         }
 763
 764         return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
 765 }
 766
 767 static int
 768 vtnet_setup_interface(struct vtnet_softc *sc)
 769 {
 770         device_t dev;
 771         struct ifnet *ifp;
 772         int tx_size;
 773
 774         dev = sc->vtnet_dev;
 775
 776         ifp = sc->vtnet_ifp = if_alloc(IFT_ETHER);
 777         if (ifp == NULL) {
 778                 device_printf(dev, "cannot allocate ifnet structure\n");
 779                 return (ENOSPC);
 780         }
 781
 782         ifp->if_softc = sc;
 783         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 784         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 785         ifp->if_init = vtnet_init;
 786         ifp->if_start = vtnet_start;
 787         ifp->if_ioctl = vtnet_ioctl;
 788
 789         sc->vtnet_rx_size = virtqueue_size(sc->vtnet_rx_vq);
 790         sc->vtnet_rx_process_limit = sc->vtnet_rx_size;
 791
 792         tx_size = virtqueue_size(sc->vtnet_tx_vq);
 793         sc->vtnet_tx_size = tx_size;
 794         sc->vtnet_txhdridx = 0;
 795         sc->vtnet_txhdrarea = contigmalloc(
 796             ((sc->vtnet_tx_size / 2) + 1) * sizeof(struct vtnet_tx_header),
 797             M_VTNET, M_WAITOK, 0, BUS_SPACE_MAXADDR, 4, 0);
 798         if (sc->vtnet_txhdrarea == NULL) {
 799                 device_printf(dev, "cannot contigmalloc the tx headers\n");
 800                 return (ENOMEM);
 801         }
 802         sc->vtnet_macfilter = contigmalloc(
 803             sizeof(struct vtnet_mac_filter),
 804             M_DEVBUF, M_WAITOK, 0, BUS_SPACE_MAXADDR, 4, 0);
 805         if (sc->vtnet_macfilter == NULL) {
 806                 device_printf(dev,
 807                     "cannot contigmalloc the mac filter table\n");
 808                 return (ENOMEM);
 809         }
 810         ifq_set_maxlen(&ifp->if_snd, tx_size - 1);
 811         ifq_set_ready(&ifp->if_snd);
 812
 813         ether_ifattach(ifp, sc->vtnet_hwaddr, NULL);
 814
 815         if (virtio_with_feature(dev, VIRTIO_NET_F_STATUS)){
 816                 //ifp->if_capabilities |= IFCAP_LINKSTATE;
 817                  kprintf("add dynamic link state\n");
 818         }
 819
 820         /* Tell the upper layer(s) we support long frames. */
 821         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
 822         ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU;
 823
 824         if (virtio_with_feature(dev, VIRTIO_NET_F_CSUM)) {
 825                 ifp->if_capabilities |= IFCAP_TXCSUM;
 826
 827                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4))
 828                         ifp->if_capabilities |= IFCAP_TSO4;
 829                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
 830                         ifp->if_capabilities |= IFCAP_TSO6;
 831                 if (ifp->if_capabilities & IFCAP_TSO)
 832                         ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
 833
 834                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN))
 835                         sc->vtnet_flags |= VTNET_FLAG_TSO_ECN;
 836         }
 837
 838         if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_CSUM)) {
 839                 ifp->if_capabilities |= IFCAP_RXCSUM;
 840
 841                 if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) ||
 842                     virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6))
 843                         ifp->if_capabilities |= IFCAP_LRO;
 844         }
 845
 846         if (ifp->if_capabilities & IFCAP_HWCSUM) {
 847                 /*
 848                  * VirtIO does not support VLAN tagging, but we can fake
 849                  * it by inserting and removing the 802.1Q header during
 850                  * transmit and receive. We are then able to do checksum
 851                  * offloading of VLAN frames.
 852                  */
 853                 ifp->if_capabilities |=
 854                         IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM;
 855         }
 856
 857         ifp->if_capenable = ifp->if_capabilities;
 858
 859         /*
 860          * Capabilities after here are not enabled by default.
 861          */
 862
 863         if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) {
 864                 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
 865
 866                 sc->vtnet_vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 867                     vtnet_register_vlan, sc, EVENTHANDLER_PRI_FIRST);
 868                 sc->vtnet_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 869                     vtnet_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST);
 870         }
 871
 872         return (0);
 873 }
 874
 875 static void
 876 vtnet_set_hwaddr(struct vtnet_softc *sc)
 877 {
 878         device_t dev;
 879
 880         dev = sc->vtnet_dev;
 881
 882         if ((sc->vtnet_flags & VTNET_FLAG_CTRL_MAC) &&
 883             (sc->vtnet_flags & VTNET_FLAG_CTRL_RX)) {
 884                 if (vtnet_ctrl_mac_cmd(sc, sc->vtnet_hwaddr) != 0)
 885                         device_printf(dev, "unable to set MAC address\n");
 886         } else if (sc->vtnet_flags & VTNET_FLAG_MAC) {
 887                 virtio_write_device_config(dev,
 888                     offsetof(struct virtio_net_config, mac),
 889                     sc->vtnet_hwaddr, ETHER_ADDR_LEN);
 890         }
 891 }
 892
 893 static void
 894 vtnet_get_hwaddr(struct vtnet_softc *sc)
 895 {
 896         device_t dev;
 897
 898         dev = sc->vtnet_dev;
 899
 900         if ((sc->vtnet_flags & VTNET_FLAG_MAC) == 0) {
 901                 /*
 902                  * Generate a random locally administered unicast address.
 903                  *
 904                  * It would be nice to generate the same MAC address across
 905                  * reboots, but it seems all the hosts currently available
 906                  * support the MAC feature, so this isn't too important.
 907                  */
 908                 sc->vtnet_hwaddr[0] = 0xB2;
 909                 karc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1);
 910                 vtnet_set_hwaddr(sc);
 911                 return;
 912         }
 913
 914         virtio_read_device_config(dev,
 915             offsetof(struct virtio_net_config, mac),
 916             sc->vtnet_hwaddr, ETHER_ADDR_LEN);
 917 }
 918
 919 static int
 920 vtnet_is_link_up(struct vtnet_softc *sc)
 921 {
 922         device_t dev;
 923         struct ifnet *ifp;
 924         uint16_t status;
 925
 926         dev = sc->vtnet_dev;
 927         ifp = sc->vtnet_ifp;
 928
 929         ASSERT_SERIALIZED(&sc->vtnet_slz);
 930
 931         status = virtio_read_dev_config_2(dev,
 932                         offsetof(struct virtio_net_config, status));
 933
 934         return ((status & VIRTIO_NET_S_LINK_UP) != 0);
 935 }
 936
 937 static void
 938 vtnet_update_link_status(struct vtnet_softc *sc)
 939 {
 940         device_t dev;
 941         struct ifnet *ifp;
 942         struct ifaltq_subque *ifsq;
 943         int link;
 944
 945         dev = sc->vtnet_dev;
 946         ifp = sc->vtnet_ifp;
 947         ifsq = ifq_get_subq_default(&ifp->if_snd);
 948
 949         link = vtnet_is_link_up(sc);
 950
 951         if (link && ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0)) {
 952                 sc->vtnet_flags |= VTNET_FLAG_LINK;
 953                 if (bootverbose)
 954                         device_printf(dev, "Link is up\n");
 955                 ifp->if_link_state = LINK_STATE_UP;
 956                 if_link_state_change(ifp);
 957                 if (!ifsq_is_empty(ifsq))
 958                         vtnet_start_locked(ifp, ifsq);
 959         } else if (!link && (sc->vtnet_flags & VTNET_FLAG_LINK)) {
 960                 sc->vtnet_flags &= ~VTNET_FLAG_LINK;
 961                 if (bootverbose)
 962                         device_printf(dev, "Link is down\n");
 963
 964                 ifp->if_link_state = LINK_STATE_DOWN;
 965                 if_link_state_change(ifp);
 966         }
 967 }
 968
 969 #if 0
 970 static void
 971 vtnet_watchdog(struct vtnet_softc *sc)
 972 {
 973         struct ifnet *ifp;
 974
 975         ifp = sc->vtnet_ifp;
 976
 977 #ifdef VTNET_TX_INTR_MODERATION
 978         vtnet_txeof(sc);
 979 #endif
 980
 981         if (sc->vtnet_watchdog_timer == 0 || --sc->vtnet_watchdog_timer)
 982                 return;
 983
 984         if_printf(ifp, "watchdog timeout -- resetting\n");
 985 #ifdef VTNET_DEBUG
 986         virtqueue_dump(sc->vtnet_tx_vq);
 987 #endif
 988         ifp->if_oerrors++;
 989         ifp->if_flags &= ~IFF_RUNNING;
 990         vtnet_init_locked(sc);
 991 }
 992 #endif
 993
 994 static void
 995 vtnet_config_change_task(void *arg, int pending)
 996 {
 997         struct vtnet_softc *sc;
 998
 999         sc = arg;
1000
1001         lwkt_serialize_enter(&sc->vtnet_slz);
1002         vtnet_update_link_status(sc);
1003         lwkt_serialize_exit(&sc->vtnet_slz);
1004 }
1005
1006 static int
1007 vtnet_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data,struct ucred *cr)
1008 {
1009         struct vtnet_softc *sc;
1010         struct ifreq *ifr;
1011         int reinit, mask, error;
1012
1013         sc = ifp->if_softc;
1014         ifr = (struct ifreq *) data;
1015         reinit = 0;
1016         error = 0;
1017
1018         switch (cmd) {
1019         case SIOCSIFMTU:
1020                 if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > VTNET_MAX_MTU)
1021                         error = EINVAL;
1022                 else if (ifp->if_mtu != ifr->ifr_mtu) {
1023                         lwkt_serialize_enter(&sc->vtnet_slz);
1024                         error = vtnet_change_mtu(sc, ifr->ifr_mtu);
1025                         lwkt_serialize_exit(&sc->vtnet_slz);
1026                 }
1027                 break;
1028
1029         case SIOCSIFFLAGS:
1030                 lwkt_serialize_enter(&sc->vtnet_slz);
1031                 if ((ifp->if_flags & IFF_UP) == 0) {
1032                         if (ifp->if_flags & IFF_RUNNING)
1033                                 vtnet_stop(sc);
1034                 } else if (ifp->if_flags & IFF_RUNNING) {
1035                         if ((ifp->if_flags ^ sc->vtnet_if_flags) &
1036                             (IFF_PROMISC | IFF_ALLMULTI)) {
1037                                 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX)
1038                                         vtnet_rx_filter(sc);
1039                                 else
1040                                         error = ENOTSUP;
1041                         }
1042                 } else
1043                         vtnet_init_locked(sc);
1044
1045                 if (error == 0)
1046                         sc->vtnet_if_flags = ifp->if_flags;
1047                 lwkt_serialize_exit(&sc->vtnet_slz);
1048                 break;
1049
1050         case SIOCADDMULTI:
1051         case SIOCDELMULTI:
1052                 lwkt_serialize_enter(&sc->vtnet_slz);
1053                 if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) &&
1054                     (ifp->if_flags & IFF_RUNNING))
1055                         vtnet_rx_filter_mac(sc);
1056                 lwkt_serialize_exit(&sc->vtnet_slz);
1057                 break;
1058
1059         case SIOCSIFMEDIA:
1060         case SIOCGIFMEDIA:
1061                 error = ifmedia_ioctl(ifp, ifr, &sc->vtnet_media, cmd);
1062                 break;
1063
1064         case SIOCSIFCAP:
1065                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1066
1067                 lwkt_serialize_enter(&sc->vtnet_slz);
1068
1069                 if (mask & IFCAP_TXCSUM) {
1070                         ifp->if_capenable ^= IFCAP_TXCSUM;
1071                         if (ifp->if_capenable & IFCAP_TXCSUM)
1072                                 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;
1073                         else
1074                                 ifp->if_hwassist &= ~VTNET_CSUM_OFFLOAD;
1075                 }
1076
1077                 if (mask & IFCAP_TSO4) {
1078                         ifp->if_capenable ^= IFCAP_TSO4;
1079                         if (ifp->if_capenable & IFCAP_TSO4)
1080                                 ifp->if_hwassist |= CSUM_TSO;
1081                         else
1082                                 ifp->if_hwassist &= ~CSUM_TSO;
1083                 }
1084
1085                 if (mask & IFCAP_RXCSUM) {
1086                         ifp->if_capenable ^= IFCAP_RXCSUM;
1087                         reinit = 1;
1088                 }
1089
1090                 if (mask & IFCAP_LRO) {
1091                         ifp->if_capenable ^= IFCAP_LRO;
1092                         reinit = 1;
1093                 }
1094
1095                 if (mask & IFCAP_VLAN_HWFILTER) {
1096                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1097                         reinit = 1;
1098                 }
1099
1100                 if (mask & IFCAP_VLAN_HWTSO)
1101                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1102
1103                 if (mask & IFCAP_VLAN_HWTAGGING)
1104                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1105
1106                 if (reinit && (ifp->if_flags & IFF_RUNNING)) {
1107                         ifp->if_flags &= ~IFF_RUNNING;
1108                         vtnet_init_locked(sc);
1109                 }
1110                 //VLAN_CAPABILITIES(ifp);
1111
1112                 lwkt_serialize_exit(&sc->vtnet_slz);
1113                 break;
1114
1115         default:
1116                 error = ether_ioctl(ifp, cmd, data);
1117                 break;
1118         }
1119
1120         return (error);
1121 }
1122
1123 static int
1124 vtnet_change_mtu(struct vtnet_softc *sc, int new_mtu)
1125 {
1126         struct ifnet *ifp;
1127         int new_frame_size, clsize;
1128
1129         ifp = sc->vtnet_ifp;
1130
1131         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1132                 new_frame_size = sizeof(struct vtnet_rx_header) +
1133                     sizeof(struct ether_vlan_header) + new_mtu;
1134
1135                 if (new_frame_size > MJUM9BYTES)
1136                         return (EINVAL);
1137
1138                 if (new_frame_size <= MCLBYTES)
1139                         clsize = MCLBYTES;
1140                 else
1141                         clsize = MJUM9BYTES;
1142         } else {
1143                 new_frame_size = sizeof(struct virtio_net_hdr_mrg_rxbuf) +
1144                     sizeof(struct ether_vlan_header) + new_mtu;
1145
1146                 if (new_frame_size <= MCLBYTES)
1147                         clsize = MCLBYTES;
1148                 else
1149                         clsize = MJUMPAGESIZE;
1150         }
1151
1152         sc->vtnet_rx_mbuf_size = clsize;
1153         sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc);
1154         KASSERT(sc->vtnet_rx_mbuf_count < VTNET_MAX_RX_SEGS,
1155             ("too many rx mbufs: %d", sc->vtnet_rx_mbuf_count));
1156
1157         ifp->if_mtu = new_mtu;
1158
1159         if (ifp->if_flags & IFF_RUNNING) {
1160                 ifp->if_flags &= ~IFF_RUNNING;
1161                 vtnet_init_locked(sc);
1162         }
1163
1164         return (0);
1165 }
1166
1167 static int
1168 vtnet_init_rx_vq(struct vtnet_softc *sc)
1169 {
1170         struct virtqueue *vq;
1171         int nbufs, error;
1172
1173         vq = sc->vtnet_rx_vq;
1174         nbufs = 0;
1175         error = ENOSPC;
1176
1177         while (!virtqueue_full(vq)) {
1178                 if ((error = vtnet_newbuf(sc)) != 0)
1179                         break;
1180                 nbufs++;
1181         }
1182
1183         if (nbufs > 0) {
1184                 virtqueue_notify(vq, &sc->vtnet_slz);
1185
1186                 /*
1187                  * EMSGSIZE signifies the virtqueue did not have enough
1188                  * entries available to hold the last mbuf. This is not
1189                  * an error. We should not get ENOSPC since we check if
1190                  * the virtqueue is full before attempting to add a
1191                  * buffer.
1192                  */
1193                 if (error == EMSGSIZE)
1194                         error = 0;
1195         }
1196
1197         return (error);
1198 }
1199
1200 static void
1201 vtnet_free_rx_mbufs(struct vtnet_softc *sc)
1202 {
1203         struct virtqueue *vq;
1204         struct mbuf *m;
1205         int last;
1206
1207         vq = sc->vtnet_rx_vq;
1208         last = 0;
1209
1210         while ((m = virtqueue_drain(vq, &last)) != NULL)
1211                 m_freem(m);
1212
1213         KASSERT(virtqueue_empty(vq), ("mbufs remaining in Rx Vq"));
1214 }
1215
1216 static void
1217 vtnet_free_tx_mbufs(struct vtnet_softc *sc)
1218 {
1219         struct virtqueue *vq;
1220         struct vtnet_tx_header *txhdr;
1221         int last;
1222
1223         vq = sc->vtnet_tx_vq;
1224         last = 0;
1225
1226         while ((txhdr = virtqueue_drain(vq, &last)) != NULL) {
1227                 m_freem(txhdr->vth_mbuf);
1228         }
1229
1230         KASSERT(virtqueue_empty(vq), ("mbufs remaining in Tx Vq"));
1231 }
1232
1233 static void
1234 vtnet_free_ctrl_vq(struct vtnet_softc *sc)
1235 {
1236         /*
1237          * The control virtqueue is only polled, therefore
1238          * it should already be empty.
1239          */
1240         KASSERT(virtqueue_empty(sc->vtnet_ctrl_vq),
1241                 ("Ctrl Vq not empty"));
1242 }
1243
1244 static struct mbuf *
1245 vtnet_alloc_rxbuf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp)
1246 {
1247         struct mbuf *m_head, *m_tail, *m;
1248         int i, clsize;
1249
1250         clsize = sc->vtnet_rx_mbuf_size;
1251
1252         /*use getcl instead of getjcl. see  if_mxge.c comment line 2398*/
1253         //m_head = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, clsize);
1254         m_head = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR );
1255         if (m_head == NULL)
1256                 goto fail;
1257
1258         m_head->m_len = clsize;
1259         m_tail = m_head;
1260
1261         if (nbufs > 1) {
1262                 KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
1263                         ("chained Rx mbuf requested without LRO_NOMRG"));
1264
1265                 for (i = 0; i < nbufs - 1; i++) {
1266                         //m = m_getjcl(M_DONTWAIT, MT_DATA, 0, clsize);
1267                         m = m_getcl(M_NOWAIT, MT_DATA, 0);
1268                         if (m == NULL)
1269                                 goto fail;
1270
1271                         m->m_len = clsize;
1272                         m_tail->m_next = m;
1273                         m_tail = m;
1274                 }
1275         }
1276
1277         if (m_tailp != NULL)
1278                 *m_tailp = m_tail;
1279
1280         return (m_head);
1281
1282 fail:
1283         sc->vtnet_stats.mbuf_alloc_failed++;
1284         m_freem(m_head);
1285
1286         return (NULL);
1287 }
1288
1289 static int
1290 vtnet_replace_rxbuf(struct vtnet_softc *sc, struct mbuf *m0, int len0)
1291 {
1292         struct mbuf *m, *m_prev;
1293         struct mbuf *m_new, *m_tail;
1294         int len, clsize, nreplace, error;
1295
1296         m = m0;
1297         m_prev = NULL;
1298         len = len0;
1299
1300         m_tail = NULL;
1301         clsize = sc->vtnet_rx_mbuf_size;
1302         nreplace = 0;
1303
1304         if (m->m_next != NULL)
1305                 KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
1306                     ("chained Rx mbuf without LRO_NOMRG"));
1307
1308         /*
1309          * Since LRO_NOMRG mbuf chains are so large, we want to avoid
1310          * allocating an entire chain for each received frame. When
1311          * the received frame's length is less than that of the chain,
1312          * the unused mbufs are reassigned to the new chain.
1313          */
1314         while (len > 0) {
1315                 /*
1316                  * Something is seriously wrong if we received
1317                  * a frame larger than the mbuf chain. Drop it.
1318                  */
1319                 if (m == NULL) {
1320                         sc->vtnet_stats.rx_frame_too_large++;
1321                         return (EMSGSIZE);
1322                 }
1323
1324                 KASSERT(m->m_len == clsize,
1325                     ("mbuf length not expected cluster size: %d",
1326                     m->m_len));
1327
1328                 m->m_len = MIN(m->m_len, len);
1329                 len -= m->m_len;
1330
1331                 m_prev = m;
1332                 m = m->m_next;
1333                 nreplace++;
1334         }
1335
1336         KASSERT(m_prev != NULL, ("m_prev == NULL"));
1337         KASSERT(nreplace <= sc->vtnet_rx_mbuf_count,
1338                 ("too many replacement mbufs: %d/%d", nreplace,
1339                 sc->vtnet_rx_mbuf_count));
1340
1341         m_new = vtnet_alloc_rxbuf(sc, nreplace, &m_tail);
1342         if (m_new == NULL) {
1343                 m_prev->m_len = clsize;
1344                 return (ENOBUFS);
1345         }
1346
1347         /*
1348          * Move unused mbufs, if any, from the original chain
1349          * onto the end of the new chain.
1350          */
1351         if (m_prev->m_next != NULL) {
1352                 m_tail->m_next = m_prev->m_next;
1353                 m_prev->m_next = NULL;
1354         }
1355
1356         error = vtnet_enqueue_rxbuf(sc, m_new);
1357         if (error) {
1358                 /*
1359                  * BAD! We could not enqueue the replacement mbuf chain. We
1360                  * must restore the m0 chain to the original state if it was
1361                  * modified so we can subsequently discard it.
1362                  *
1363                  * NOTE: The replacement is suppose to be an identical copy
1364                  * to the one just dequeued so this is an unexpected error.
1365                  */
1366                 sc->vtnet_stats.rx_enq_replacement_failed++;
1367
1368                 if (m_tail->m_next != NULL) {
1369                         m_prev->m_next = m_tail->m_next;
1370                         m_tail->m_next = NULL;
1371                 }
1372
1373                 m_prev->m_len = clsize;
1374                 m_freem(m_new);
1375         }
1376
1377         return (error);
1378 }
1379
1380 static int
1381 vtnet_newbuf(struct vtnet_softc *sc)
1382 {
1383         struct mbuf *m;
1384         int error;
1385
1386         m = vtnet_alloc_rxbuf(sc, sc->vtnet_rx_mbuf_count, NULL);
1387         if (m == NULL)
1388                 return (ENOBUFS);
1389
1390         error = vtnet_enqueue_rxbuf(sc, m);
1391         if (error)
1392                 m_freem(m);
1393
1394         return (error);
1395 }
1396
1397 static void
1398 vtnet_discard_merged_rxbuf(struct vtnet_softc *sc, int nbufs)
1399 {
1400         struct virtqueue *vq;
1401         struct mbuf *m;
1402
1403         vq = sc->vtnet_rx_vq;
1404
1405         while (--nbufs > 0) {
1406                 if ((m = virtqueue_dequeue(vq, NULL)) == NULL)
1407                         break;
1408                 vtnet_discard_rxbuf(sc, m);
1409         }
1410 }
1411
1412 static void
1413 vtnet_discard_rxbuf(struct vtnet_softc *sc, struct mbuf *m)
1414 {
1415         int error;
1416
1417         /*
1418          * Requeue the discarded mbuf. This should always be
1419          * successful since it was just dequeued.
1420          */
1421         error = vtnet_enqueue_rxbuf(sc, m);
1422         KASSERT(error == 0, ("cannot requeue discarded mbuf"));
1423 }
1424
1425 static int
1426 vtnet_enqueue_rxbuf(struct vtnet_softc *sc, struct mbuf *m)
1427 {
1428         struct sglist sg;
1429         struct sglist_seg segs[VTNET_MAX_RX_SEGS];
1430         struct vtnet_rx_header *rxhdr;
1431         struct virtio_net_hdr *hdr;
1432         uint8_t *mdata;
1433         int offset, error;
1434
1435         ASSERT_SERIALIZED(&sc->vtnet_slz);
1436         if ((sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) == 0)
1437                 KASSERT(m->m_next == NULL, ("chained Rx mbuf"));
1438
1439         sglist_init(&sg, VTNET_MAX_RX_SEGS, segs);
1440
1441         mdata = mtod(m, uint8_t *);
1442         offset = 0;
1443
1444         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1445                 rxhdr = (struct vtnet_rx_header *) mdata;
1446                 hdr = &rxhdr->vrh_hdr;
1447                 offset += sizeof(struct vtnet_rx_header);
1448
1449                 error = sglist_append(&sg, hdr, sc->vtnet_hdr_size);
1450                 KASSERT(error == 0, ("cannot add header to sglist"));
1451         }
1452
1453         error = sglist_append(&sg, mdata + offset, m->m_len - offset);
1454         if (error)
1455                 return (error);
1456
1457         if (m->m_next != NULL) {
1458                 error = sglist_append_mbuf(&sg, m->m_next);
1459                 if (error)
1460                         return (error);
1461         }
1462
1463         return (virtqueue_enqueue(sc->vtnet_rx_vq, m, &sg, 0, sg.sg_nseg));
1464 }
1465
1466 static void
1467 vtnet_vlan_tag_remove(struct mbuf *m)
1468 {
1469         struct ether_vlan_header *evl;
1470
1471         evl = mtod(m, struct ether_vlan_header *);
1472
1473         m->m_pkthdr.ether_vlantag = ntohs(evl->evl_tag);
1474         m->m_flags |= M_VLANTAG;
1475
1476         /* Strip the 802.1Q header. */
1477         bcopy((char *) evl, (char *) evl + ETHER_VLAN_ENCAP_LEN,
1478             ETHER_HDR_LEN - ETHER_TYPE_LEN);
1479         m_adj(m, ETHER_VLAN_ENCAP_LEN);
1480 }
1481
1482 /*
1483  * Alternative method of doing receive checksum offloading. Rather
1484  * than parsing the received frame down to the IP header, use the
1485  * csum_offset to determine which CSUM_* flags are appropriate. We
1486  * can get by with doing this only because the checksum offsets are
1487  * unique for the things we care about.
1488  */
1489 static int
1490 vtnet_rx_csum(struct vtnet_softc *sc, struct mbuf *m,
1491     struct virtio_net_hdr *hdr)
1492 {
1493         struct ether_header *eh;
1494         struct ether_vlan_header *evh;
1495         struct udphdr *udp;
1496         int csum_len;
1497         uint16_t eth_type;
1498
1499         csum_len = hdr->csum_start + hdr->csum_offset;
1500
1501         if (csum_len < sizeof(struct ether_header) + sizeof(struct ip))
1502                 return (1);
1503         if (m->m_len < csum_len)
1504                 return (1);
1505
1506         eh = mtod(m, struct ether_header *);
1507         eth_type = ntohs(eh->ether_type);
1508         if (eth_type == ETHERTYPE_VLAN) {
1509                 evh = mtod(m, struct ether_vlan_header *);
1510                 eth_type = ntohs(evh->evl_proto);
1511         }
1512
1513         if (eth_type != ETHERTYPE_IP && eth_type != ETHERTYPE_IPV6) {
1514                 sc->vtnet_stats.rx_csum_bad_ethtype++;
1515                 return (1);
1516         }
1517
1518         /* Use the offset to determine the appropriate CSUM_* flags. */
1519         switch (hdr->csum_offset) {
1520         case offsetof(struct udphdr, uh_sum):
1521                 if (m->m_len < hdr->csum_start + sizeof(struct udphdr))
1522                         return (1);
1523                 udp = (struct udphdr *)(mtod(m, uint8_t *) + hdr->csum_start);
1524                 if (udp->uh_sum == 0)
1525                         return (0);
1526
1527                 /* FALLTHROUGH */
1528
1529         case offsetof(struct tcphdr, th_sum):
1530                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1531                 m->m_pkthdr.csum_data = 0xFFFF;
1532                 break;
1533
1534         default:
1535                 sc->vtnet_stats.rx_csum_bad_offset++;
1536                 return (1);
1537         }
1538
1539         sc->vtnet_stats.rx_csum_offloaded++;
1540
1541         return (0);
1542 }
1543
1544 static int
1545 vtnet_rxeof_merged(struct vtnet_softc *sc, struct mbuf *m_head, int nbufs)
1546 {
1547         struct ifnet *ifp;
1548         struct virtqueue *vq;
1549         struct mbuf *m, *m_tail;
1550         int len;
1551
1552         ifp = sc->vtnet_ifp;
1553         vq = sc->vtnet_rx_vq;
1554         m_tail = m_head;
1555
1556         while (--nbufs > 0) {
1557                 m = virtqueue_dequeue(vq, &len);
1558                 if (m == NULL) {
1559                         ifp->if_ierrors++;
1560                         goto fail;
1561                 }
1562
1563                 if (vtnet_newbuf(sc) != 0) {
1564                         ifp->if_iqdrops++;
1565                         vtnet_discard_rxbuf(sc, m);
1566                         if (nbufs > 1)
1567                                 vtnet_discard_merged_rxbuf(sc, nbufs);
1568                         goto fail;
1569                 }
1570
1571                 if (m->m_len < len)
1572                         len = m->m_len;
1573
1574                 m->m_len = len;
1575                 m->m_flags &= ~M_PKTHDR;
1576
1577                 m_head->m_pkthdr.len += len;
1578                 m_tail->m_next = m;
1579                 m_tail = m;
1580         }
1581
1582         return (0);
1583
1584 fail:
1585         sc->vtnet_stats.rx_mergeable_failed++;
1586         m_freem(m_head);
1587
1588         return (1);
1589 }
1590
1591 static int
1592 vtnet_rxeof(struct vtnet_softc *sc, int count, int *rx_npktsp)
1593 {
1594         struct virtio_net_hdr lhdr;
1595         struct ifnet *ifp;
1596         struct virtqueue *vq;
1597         struct mbuf *m;
1598         struct ether_header *eh;
1599         struct virtio_net_hdr *hdr;
1600         struct virtio_net_hdr_mrg_rxbuf *mhdr;
1601         int len, deq, nbufs, adjsz, rx_npkts;
1602
1603         ifp = sc->vtnet_ifp;
1604         vq = sc->vtnet_rx_vq;
1605         hdr = &lhdr;
1606         deq = 0;
1607         rx_npkts = 0;
1608
1609         ASSERT_SERIALIZED(&sc->vtnet_slz);
1610
1611         while (--count >= 0) {
1612                 m = virtqueue_dequeue(vq, &len);
1613                 if (m == NULL)
1614                         break;
1615                 deq++;
1616
1617                 if (len < sc->vtnet_hdr_size + ETHER_HDR_LEN) {
1618                         ifp->if_ierrors++;
1619                         vtnet_discard_rxbuf(sc, m);
1620                         continue;
1621                 }
1622
1623                 if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1624                         nbufs = 1;
1625                         adjsz = sizeof(struct vtnet_rx_header);
1626                         /*
1627                          * Account for our pad between the header and
1628                          * the actual start of the frame.
1629                          */
1630                         len += VTNET_RX_HEADER_PAD;
1631                 } else {
1632                         mhdr = mtod(m, struct virtio_net_hdr_mrg_rxbuf *);
1633                         nbufs = mhdr->num_buffers;
1634                         adjsz = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1635                 }
1636
1637                 if (vtnet_replace_rxbuf(sc, m, len) != 0) {
1638                         ifp->if_iqdrops++;
1639                         vtnet_discard_rxbuf(sc, m);
1640                         if (nbufs > 1)
1641                                 vtnet_discard_merged_rxbuf(sc, nbufs);
1642                         continue;
1643                 }
1644
1645                 m->m_pkthdr.len = len;
1646                 m->m_pkthdr.rcvif = ifp;
1647                 m->m_pkthdr.csum_flags = 0;
1648
1649                 if (nbufs > 1) {
1650                         if (vtnet_rxeof_merged(sc, m, nbufs) != 0)
1651                                 continue;
1652                 }
1653
1654                 ifp->if_ipackets++;
1655
1656                 /*
1657                  * Save copy of header before we strip it. For both mergeable
1658                  * and non-mergeable, the VirtIO header is placed first in the
1659                  * mbuf's data. We no longer need num_buffers, so always use a
1660                  * virtio_net_hdr.
1661                  */
1662                 memcpy(hdr, mtod(m, void *), sizeof(struct virtio_net_hdr));
1663                 m_adj(m, adjsz);
1664
1665                 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1666                         eh = mtod(m, struct ether_header *);
1667                         if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
1668                                 vtnet_vlan_tag_remove(m);
1669
1670                                 /*
1671                                  * With the 802.1Q header removed, update the
1672                                  * checksum starting location accordingly.
1673                                  */
1674                                 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
1675                                         hdr->csum_start -=
1676                                             ETHER_VLAN_ENCAP_LEN;
1677                         }
1678                 }
1679
1680                 if (ifp->if_capenable & IFCAP_RXCSUM &&
1681                     hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
1682                         if (vtnet_rx_csum(sc, m, hdr) != 0)
1683                                 sc->vtnet_stats.rx_csum_failed++;
1684                 }
1685
1686                 lwkt_serialize_exit(&sc->vtnet_slz);
1687                 rx_npkts++;
1688                 ifp->if_input(ifp, m, NULL, -1);
1689                 lwkt_serialize_enter(&sc->vtnet_slz);
1690
1691                 /*
1692                  * The interface may have been stopped while we were
1693                  * passing the packet up the network stack.
1694                  */
1695                 if ((ifp->if_flags & IFF_RUNNING) == 0)
1696                         break;
1697         }
1698
1699         virtqueue_notify(vq, &sc->vtnet_slz);
1700
1701         if (rx_npktsp != NULL)
1702                 *rx_npktsp = rx_npkts;
1703
1704         return (count > 0 ? 0 : EAGAIN);
1705 }
1706
1707 static void
1708 vtnet_rx_intr_task(void *arg)
1709 {
1710         struct vtnet_softc *sc;
1711         struct ifnet *ifp;
1712         int more;
1713
1714         sc = arg;
1715         ifp = sc->vtnet_ifp;
1716
1717 next:
1718 //      lwkt_serialize_enter(&sc->vtnet_slz);
1719
1720         if ((ifp->if_flags & IFF_RUNNING) == 0) {
1721                 vtnet_enable_rx_intr(sc);
1722 //              lwkt_serialize_exit(&sc->vtnet_slz);
1723                 return;
1724         }
1725
1726         more = vtnet_rxeof(sc, sc->vtnet_rx_process_limit, NULL);
1727         if (!more && vtnet_enable_rx_intr(sc) != 0) {
1728                 vtnet_disable_rx_intr(sc);
1729                 more = 1;
1730         }
1731
1732 //      lwkt_serialize_exit(&sc->vtnet_slz);
1733
1734         if (more) {
1735                 sc->vtnet_stats.rx_task_rescheduled++;
1736                 goto next;
1737         }
1738 }
1739
1740 static int
1741 vtnet_rx_vq_intr(void *xsc)
1742 {
1743         struct vtnet_softc *sc;
1744
1745         sc = xsc;
1746
1747         vtnet_disable_rx_intr(sc);
1748         vtnet_rx_intr_task(sc);
1749
1750         return (1);
1751 }
1752
1753 static void
1754 vtnet_txeof(struct vtnet_softc *sc)
1755 {
1756         struct virtqueue *vq;
1757         struct ifnet *ifp;
1758         struct vtnet_tx_header *txhdr;
1759         int deq;
1760
1761         vq = sc->vtnet_tx_vq;
1762         ifp = sc->vtnet_ifp;
1763         deq = 0;
1764
1765         ASSERT_SERIALIZED(&sc->vtnet_slz);
1766
1767         while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) {
1768                 deq++;
1769                 ifp->if_opackets++;
1770                 m_freem(txhdr->vth_mbuf);
1771         }
1772
1773         if (deq > 0) {
1774                 ifq_clr_oactive(&ifp->if_snd);
1775                 if (virtqueue_empty(vq))
1776                         sc->vtnet_watchdog_timer = 0;
1777         }
1778 }
1779
1780 static struct mbuf *
1781 vtnet_tx_offload(struct vtnet_softc *sc, struct mbuf *m,
1782     struct virtio_net_hdr *hdr)
1783 {
1784         struct ifnet *ifp;
1785         struct ether_header *eh;
1786         struct ether_vlan_header *evh;
1787         struct ip *ip;
1788         struct ip6_hdr *ip6;
1789         struct tcphdr *tcp;
1790         int ip_offset;
1791         uint16_t eth_type, csum_start;
1792         uint8_t ip_proto, gso_type;
1793
1794         ifp = sc->vtnet_ifp;
1795         M_ASSERTPKTHDR(m);
1796
1797         ip_offset = sizeof(struct ether_header);
1798         if (m->m_len < ip_offset) {
1799                 if ((m = m_pullup(m, ip_offset)) == NULL)
1800                         return (NULL);
1801         }
1802
1803         eh = mtod(m, struct ether_header *);
1804         eth_type = ntohs(eh->ether_type);
1805         if (eth_type == ETHERTYPE_VLAN) {
1806                 ip_offset = sizeof(struct ether_vlan_header);
1807                 if (m->m_len < ip_offset) {
1808                         if ((m = m_pullup(m, ip_offset)) == NULL)
1809                                 return (NULL);
1810                 }
1811                 evh = mtod(m, struct ether_vlan_header *);
1812                 eth_type = ntohs(evh->evl_proto);
1813         }
1814
1815         switch (eth_type) {
1816         case ETHERTYPE_IP:
1817                 if (m->m_len < ip_offset + sizeof(struct ip)) {
1818                         m = m_pullup(m, ip_offset + sizeof(struct ip));
1819                         if (m == NULL)
1820                                 return (NULL);
1821                 }
1822
1823                 ip = (struct ip *)(mtod(m, uint8_t *) + ip_offset);
1824                 ip_proto = ip->ip_p;
1825                 csum_start = ip_offset + (ip->ip_hl << 2);
1826                 gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
1827                 break;
1828
1829         case ETHERTYPE_IPV6:
1830                 if (m->m_len < ip_offset + sizeof(struct ip6_hdr)) {
1831                         m = m_pullup(m, ip_offset + sizeof(struct ip6_hdr));
1832                         if (m == NULL)
1833                                 return (NULL);
1834                 }
1835
1836                 ip6 = (struct ip6_hdr *)(mtod(m, uint8_t *) + ip_offset);
1837                 /*
1838                  * XXX Assume no extension headers are present. Presently,
1839                  * this will always be true in the case of TSO, and FreeBSD
1840                  * does not perform checksum offloading of IPv6 yet.
1841                  */
1842                 ip_proto = ip6->ip6_nxt;
1843                 csum_start = ip_offset + sizeof(struct ip6_hdr);
1844                 gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
1845                 break;
1846
1847         default:
1848                 return (m);
1849         }
1850
1851         if (m->m_pkthdr.csum_flags & VTNET_CSUM_OFFLOAD) {
1852                 hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM;
1853                 hdr->csum_start = csum_start;
1854                 hdr->csum_offset = m->m_pkthdr.csum_data;
1855
1856                 sc->vtnet_stats.tx_csum_offloaded++;
1857         }
1858
1859         if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1860                 if (ip_proto != IPPROTO_TCP)
1861                         return (m);
1862
1863                 if (m->m_len < csum_start + sizeof(struct tcphdr)) {
1864                         m = m_pullup(m, csum_start + sizeof(struct tcphdr));
1865                         if (m == NULL)
1866                                 return (NULL);
1867                 }
1868
1869                 tcp = (struct tcphdr *)(mtod(m, uint8_t *) + csum_start);
1870                 hdr->gso_type = gso_type;
1871                 hdr->hdr_len = csum_start + (tcp->th_off << 2);
1872                 hdr->gso_size = m->m_pkthdr.tso_segsz;
1873
1874                 if (tcp->th_flags & TH_CWR) {
1875                         /*
1876                          * Drop if we did not negotiate VIRTIO_NET_F_HOST_ECN.
1877                          * ECN support is only configurable globally with the
1878                          * net.inet.tcp.ecn.enable sysctl knob.
1879                          */
1880                         if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) {
1881                                 if_printf(ifp, "TSO with ECN not supported "
1882                                     "by host\n");
1883                                 m_freem(m);
1884                                 return (NULL);
1885                         }
1886
1887                         hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN;
1888                 }
1889
1890                 sc->vtnet_stats.tx_tso_offloaded++;
1891         }
1892
1893         return (m);
1894 }
1895
1896 static int
1897 vtnet_enqueue_txbuf(struct vtnet_softc *sc, struct mbuf **m_head,
1898     struct vtnet_tx_header *txhdr)
1899 {
1900         struct sglist sg;
1901         struct sglist_seg segs[VTNET_MAX_TX_SEGS];
1902         struct virtqueue *vq;
1903         struct mbuf *m;
1904         int collapsed, error;
1905
1906         vq = sc->vtnet_tx_vq;
1907         m = *m_head;
1908         collapsed = 0;
1909
1910         sglist_init(&sg, VTNET_MAX_TX_SEGS, segs);
1911         error = sglist_append(&sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size);
1912         KASSERT(error == 0 && sg.sg_nseg == 1,
1913             ("cannot add header to sglist"));
1914
1915 again:
1916         error = sglist_append_mbuf(&sg, m);
1917         if (error) {
1918                 if (collapsed)
1919                         goto fail;
1920
1921                 //m = m_collapse(m, M_NOWAIT, VTNET_MAX_TX_SEGS - 1);
1922                 m = m_defrag(m, M_NOWAIT);
1923                 if (m == NULL)
1924                         goto fail;
1925
1926                 *m_head = m;
1927                 collapsed = 1;
1928                 goto again;
1929         }
1930
1931         txhdr->vth_mbuf = m;
1932
1933         return (virtqueue_enqueue(vq, txhdr, &sg, sg.sg_nseg, 0));
1934
1935 fail:
1936         m_freem(*m_head);
1937         *m_head = NULL;
1938
1939         return (ENOBUFS);
1940 }
1941
1942 static struct mbuf *
1943 vtnet_vlan_tag_insert(struct mbuf *m)
1944 {
1945         struct mbuf *n;
1946         struct ether_vlan_header *evl;
1947
1948         if (M_WRITABLE(m) == 0) {
1949                 n = m_dup(m, M_NOWAIT);
1950                 m_freem(m);
1951                 if ((m = n) == NULL)
1952                         return (NULL);
1953         }
1954
1955         M_PREPEND(m, ETHER_VLAN_ENCAP_LEN, M_NOWAIT);
1956         if (m == NULL)
1957                 return (NULL);
1958         if (m->m_len < sizeof(struct ether_vlan_header)) {
1959                 m = m_pullup(m, sizeof(struct ether_vlan_header));
1960                 if (m == NULL)
1961                         return (NULL);
1962         }
1963
1964         /* Insert 802.1Q header into the existing Ethernet header. */
1965         evl = mtod(m, struct ether_vlan_header *);
1966         bcopy((char *) evl + ETHER_VLAN_ENCAP_LEN,
1967               (char *) evl, ETHER_HDR_LEN - ETHER_TYPE_LEN);
1968         evl->evl_encap_proto = htons(ETHERTYPE_VLAN);
1969         evl->evl_tag = htons(m->m_pkthdr.ether_vlantag);
1970         m->m_flags &= ~M_VLANTAG;
1971
1972         return (m);
1973 }
1974
1975 static int
1976 vtnet_encap(struct vtnet_softc *sc, struct mbuf **m_head)
1977 {
1978         struct vtnet_tx_header *txhdr;
1979         struct virtio_net_hdr *hdr;
1980         struct mbuf *m;
1981         int error;
1982
1983         txhdr = &sc->vtnet_txhdrarea[sc->vtnet_txhdridx];
1984         memset(txhdr, 0, sizeof(struct vtnet_tx_header));
1985
1986         /*
1987          * Always use the non-mergeable header to simplify things. When
1988          * the mergeable feature is negotiated, the num_buffers field
1989          * must be set to zero. We use vtnet_hdr_size later to enqueue
1990          * the correct header size to the host.
1991          */
1992         hdr = &txhdr->vth_uhdr.hdr;
1993         m = *m_head;
1994
1995         error = ENOBUFS;
1996
1997         if (m->m_flags & M_VLANTAG) {
1998                 //m = ether_vlanencap(m, m->m_pkthdr.ether_vtag);
1999                 m = vtnet_vlan_tag_insert(m);
2000                 if ((*m_head = m) == NULL)
2001                         goto fail;
2002                 m->m_flags &= ~M_VLANTAG;
2003         }
2004
2005         if (m->m_pkthdr.csum_flags != 0) {
2006                 m = vtnet_tx_offload(sc, m, hdr);
2007                 if ((*m_head = m) == NULL)
2008                         goto fail;
2009         }
2010
2011         error = vtnet_enqueue_txbuf(sc, m_head, txhdr);
2012         if (error == 0)
2013                 sc->vtnet_txhdridx =
2014                     (sc->vtnet_txhdridx + 1) % ((sc->vtnet_tx_size / 2) + 1);
2015 fail:
2016         return (error);
2017 }
2018
2019 static void
2020 vtnet_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
2021 {
2022         struct vtnet_softc *sc;
2023
2024         sc = ifp->if_softc;
2025
2026         ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq);
2027         lwkt_serialize_enter(&sc->vtnet_slz);
2028         vtnet_start_locked(ifp, ifsq);
2029         lwkt_serialize_exit(&sc->vtnet_slz);
2030 }
2031
2032 static void
2033 vtnet_start_locked(struct ifnet *ifp, struct ifaltq_subque *ifsq)
2034 {
2035         struct vtnet_softc *sc;
2036         struct virtqueue *vq;
2037         struct mbuf *m0;
2038         int enq;
2039
2040         sc = ifp->if_softc;
2041         vq = sc->vtnet_tx_vq;
2042         enq = 0;
2043
2044         ASSERT_SERIALIZED(&sc->vtnet_slz);
2045
2046         if ((ifp->if_flags & (IFF_RUNNING)) !=
2047             IFF_RUNNING || ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0))
2048                 return;
2049
2050 #ifdef VTNET_TX_INTR_MODERATION
2051         if (virtqueue_nused(vq) >= sc->vtnet_tx_size / 2)
2052                 vtnet_txeof(sc);
2053 #endif
2054
2055         while (!ifsq_is_empty(ifsq)) {
2056                 if (virtqueue_full(vq)) {
2057                         ifq_set_oactive(&ifp->if_snd);
2058                         break;
2059                 }
2060
2061                 m0 = ifq_dequeue(&ifp->if_snd);
2062                 if (m0 == NULL)
2063                         break;
2064
2065                 if (vtnet_encap(sc, &m0) != 0) {
2066                         if (m0 == NULL)
2067                                 break;
2068                         ifq_prepend(&ifp->if_snd, m0);
2069                         ifq_set_oactive(&ifp->if_snd);
2070                         break;
2071                 }
2072
2073                 enq++;
2074                 ETHER_BPF_MTAP(ifp, m0);
2075         }
2076
2077         if (enq > 0) {
2078                 virtqueue_notify(vq, &sc->vtnet_slz);
2079                 sc->vtnet_watchdog_timer = VTNET_WATCHDOG_TIMEOUT;
2080         }
2081 }
2082
2083 static void
2084 vtnet_tick(void *xsc)
2085 {
2086         struct vtnet_softc *sc;
2087
2088         sc = xsc;
2089
2090 #if 0
2091         ASSERT_SERIALIZED(&sc->vtnet_slz);
2092 #ifdef VTNET_DEBUG
2093         virtqueue_dump(sc->vtnet_rx_vq);
2094         virtqueue_dump(sc->vtnet_tx_vq);
2095 #endif
2096
2097         vtnet_watchdog(sc);
2098         callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc);
2099 #endif
2100 }
2101
2102 static void
2103 vtnet_tx_intr_task(void *arg)
2104 {
2105         struct vtnet_softc *sc;
2106         struct ifnet *ifp;
2107         struct ifaltq_subque *ifsq;
2108
2109         sc = arg;
2110         ifp = sc->vtnet_ifp;
2111         ifsq = ifq_get_subq_default(&ifp->if_snd);
2112
2113 next:
2114 //      lwkt_serialize_enter(&sc->vtnet_slz);
2115
2116         if ((ifp->if_flags & IFF_RUNNING) == 0) {
2117                 vtnet_enable_tx_intr(sc);
2118 //              lwkt_serialize_exit(&sc->vtnet_slz);
2119                 return;
2120         }
2121
2122         vtnet_txeof(sc);
2123
2124         if (!ifsq_is_empty(ifsq))
2125                 vtnet_start_locked(ifp, ifsq);
2126
2127         if (vtnet_enable_tx_intr(sc) != 0) {
2128                 vtnet_disable_tx_intr(sc);
2129                 sc->vtnet_stats.tx_task_rescheduled++;
2130 //              lwkt_serialize_exit(&sc->vtnet_slz);
2131                 goto next;
2132         }
2133
2134 //      lwkt_serialize_exit(&sc->vtnet_slz);
2135 }
2136
2137 static int
2138 vtnet_tx_vq_intr(void *xsc)
2139 {
2140         struct vtnet_softc *sc;
2141
2142         sc = xsc;
2143
2144         vtnet_disable_tx_intr(sc);
2145         vtnet_tx_intr_task(sc);
2146
2147         return (1);
2148 }
2149
2150 static void
2151 vtnet_stop(struct vtnet_softc *sc)
2152 {
2153         device_t dev;
2154         struct ifnet *ifp;
2155
2156         dev = sc->vtnet_dev;
2157         ifp = sc->vtnet_ifp;
2158
2159         ASSERT_SERIALIZED(&sc->vtnet_slz);
2160
2161         sc->vtnet_watchdog_timer = 0;
2162         callout_stop(&sc->vtnet_tick_ch);
2163         ifq_clr_oactive(&ifp->if_snd);
2164         ifp->if_flags &= ~(IFF_RUNNING);
2165
2166         vtnet_disable_rx_intr(sc);
2167         vtnet_disable_tx_intr(sc);
2168
2169         /*
2170          * Stop the host VirtIO adapter. Note this will reset the host
2171          * adapter's state back to the pre-initialized state, so in
2172          * order to make the device usable again, we must drive it
2173          * through virtio_reinit() and virtio_reinit_complete().
2174          */
2175         virtio_stop(dev);
2176
2177         sc->vtnet_flags &= ~VTNET_FLAG_LINK;
2178
2179         vtnet_free_rx_mbufs(sc);
2180         vtnet_free_tx_mbufs(sc);
2181 }
2182
2183 static int
2184 vtnet_virtio_reinit(struct vtnet_softc *sc)
2185 {
2186         device_t dev;
2187         struct ifnet *ifp;
2188         uint64_t features;
2189         int error;
2190
2191         dev = sc->vtnet_dev;
2192         ifp = sc->vtnet_ifp;
2193         features = sc->vtnet_features;
2194
2195         /*
2196          * Re-negotiate with the host, removing any disabled receive
2197          * features. Transmit features are disabled only on our side
2198          * via if_capenable and if_hwassist.
2199          */
2200
2201         if (ifp->if_capabilities & IFCAP_RXCSUM) {
2202                 if ((ifp->if_capenable & IFCAP_RXCSUM) == 0)
2203                         features &= ~VIRTIO_NET_F_GUEST_CSUM;
2204         }
2205
2206         if (ifp->if_capabilities & IFCAP_LRO) {
2207                 if ((ifp->if_capenable & IFCAP_LRO) == 0)
2208                         features &= ~VTNET_LRO_FEATURES;
2209         }
2210
2211         if (ifp->if_capabilities & IFCAP_VLAN_HWFILTER) {
2212                 if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
2213                         features &= ~VIRTIO_NET_F_CTRL_VLAN;
2214         }
2215
2216         error = virtio_reinit(dev, features);
2217         if (error)
2218                 device_printf(dev, "virtio reinit error %d\n", error);
2219
2220         return (error);
2221 }
2222
2223 static void
2224 vtnet_init_locked(struct vtnet_softc *sc)
2225 {
2226         device_t dev;
2227         struct ifnet *ifp;
2228         int error;
2229
2230         dev = sc->vtnet_dev;
2231         ifp = sc->vtnet_ifp;
2232
2233         ASSERT_SERIALIZED(&sc->vtnet_slz);
2234
2235         if (ifp->if_flags & IFF_RUNNING)
2236                 return;
2237
2238         /* Stop host's adapter, cancel any pending I/O. */
2239         vtnet_stop(sc);
2240
2241         /* Reinitialize the host device. */
2242         error = vtnet_virtio_reinit(sc);
2243         if (error) {
2244                 device_printf(dev,
2245                     "reinitialization failed, stopping device...\n");
2246                 vtnet_stop(sc);
2247                 return;
2248         }
2249
2250         /* Update host with assigned MAC address. */
2251         bcopy(IF_LLADDR(ifp), sc->vtnet_hwaddr, ETHER_ADDR_LEN);
2252         vtnet_set_hwaddr(sc);
2253
2254         ifp->if_hwassist = 0;
2255         if (ifp->if_capenable & IFCAP_TXCSUM)
2256                 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;
2257         if (ifp->if_capenable & IFCAP_TSO4)
2258                 ifp->if_hwassist |= CSUM_TSO;
2259
2260         error = vtnet_init_rx_vq(sc);
2261         if (error) {
2262                 device_printf(dev,
2263                     "cannot allocate mbufs for Rx virtqueue\n");
2264                 vtnet_stop(sc);
2265                 return;
2266         }
2267
2268         if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
2269                 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
2270                         /* Restore promiscuous and all-multicast modes. */
2271                         vtnet_rx_filter(sc);
2272
2273                         /* Restore filtered MAC addresses. */
2274                         vtnet_rx_filter_mac(sc);
2275                 }
2276
2277                 /* Restore VLAN filters. */
2278                 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
2279                         vtnet_rx_filter_vlan(sc);
2280         }
2281
2282         {
2283                 vtnet_enable_rx_intr(sc);
2284                 vtnet_enable_tx_intr(sc);
2285         }
2286
2287         ifp->if_flags |= IFF_RUNNING;
2288         ifq_clr_oactive(&ifp->if_snd);
2289
2290         virtio_reinit_complete(dev);
2291
2292         vtnet_update_link_status(sc);
2293         callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc);
2294 }
2295
2296 static void
2297 vtnet_init(void *xsc)
2298 {
2299         struct vtnet_softc *sc;
2300
2301         sc = xsc;
2302
2303         lwkt_serialize_enter(&sc->vtnet_slz);
2304         vtnet_init_locked(sc);
2305         lwkt_serialize_exit(&sc->vtnet_slz);
2306 }
2307
2308 static void
2309 vtnet_exec_ctrl_cmd(struct vtnet_softc *sc, void *cookie,
2310     struct sglist *sg, int readable, int writable)
2311 {
2312         struct virtqueue *vq;
2313         void *c;
2314
2315         vq = sc->vtnet_ctrl_vq;
2316
2317         ASSERT_SERIALIZED(&sc->vtnet_slz);
2318         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ,
2319             ("no control virtqueue"));
2320         KASSERT(virtqueue_empty(vq),
2321             ("control command already enqueued"));
2322
2323         if (virtqueue_enqueue(vq, cookie, sg, readable, writable) != 0)
2324                 return;
2325
2326         virtqueue_notify(vq, &sc->vtnet_slz);
2327
2328         /*
2329          * Poll until the command is complete. Previously, we would
2330          * sleep until the control virtqueue interrupt handler woke
2331          * us up, but dropping the VTNET_MTX leads to serialization
2332          * difficulties.
2333          *
2334          * Furthermore, it appears QEMU/KVM only allocates three MSIX
2335          * vectors. Two of those vectors are needed for the Rx and Tx
2336          * virtqueues. We do not support sharing both a Vq and config
2337          * changed notification on the same MSIX vector.
2338          */
2339         c = virtqueue_poll(vq, NULL);
2340         KASSERT(c == cookie, ("unexpected control command response"));
2341 }
2342
2343 static int
2344 vtnet_ctrl_mac_cmd(struct vtnet_softc *sc, uint8_t *hwaddr)
2345 {
2346         struct {
2347                 struct virtio_net_ctrl_hdr hdr __aligned(2);
2348                 uint8_t pad1;
2349                 char aligned_hwaddr[ETHER_ADDR_LEN] __aligned(8);
2350                 uint8_t pad2;
2351                 uint8_t ack;
2352         } s;
2353         struct sglist_seg segs[3];
2354         struct sglist sg;
2355         int error;
2356
2357         s.hdr.class = VIRTIO_NET_CTRL_MAC;
2358         s.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET;
2359         s.ack = VIRTIO_NET_ERR;
2360
2361         /* Copy the mac address into physically contiguous memory */
2362         memcpy(s.aligned_hwaddr, hwaddr, ETHER_ADDR_LEN);
2363
2364         sglist_init(&sg, 3, segs);
2365         error = 0;
2366         error |= sglist_append(&sg, &s.hdr,
2367             sizeof(struct virtio_net_ctrl_hdr));
2368         error |= sglist_append(&sg, s.aligned_hwaddr, ETHER_ADDR_LEN);
2369         error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
2370         KASSERT(error == 0 && sg.sg_nseg == 3,
2371             ("%s: error %d adding set MAC msg to sglist", __func__, error));
2372
2373         vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
2374
2375         return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
2376 }
2377
2378 static void
2379 vtnet_rx_filter(struct vtnet_softc *sc)
2380 {
2381         device_t dev;
2382         struct ifnet *ifp;
2383
2384         dev = sc->vtnet_dev;
2385         ifp = sc->vtnet_ifp;
2386
2387         ASSERT_SERIALIZED(&sc->vtnet_slz);
2388         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
2389             ("CTRL_RX feature not negotiated"));
2390
2391         if (vtnet_set_promisc(sc, ifp->if_flags & IFF_PROMISC) != 0)
2392                 device_printf(dev, "cannot %s promiscuous mode\n",
2393                     ifp->if_flags & IFF_PROMISC ? "enable" : "disable");
2394
2395         if (vtnet_set_allmulti(sc, ifp->if_flags & IFF_ALLMULTI) != 0)
2396                 device_printf(dev, "cannot %s all-multicast mode\n",
2397                     ifp->if_flags & IFF_ALLMULTI ? "enable" : "disable");
2398 }
2399
2400 static int
2401 vtnet_ctrl_rx_cmd(struct vtnet_softc *sc, int cmd, int on)
2402 {
2403         struct sglist_seg segs[3];
2404         struct sglist sg;
2405         struct {
2406                 struct virtio_net_ctrl_hdr hdr __aligned(2);
2407                 uint8_t pad1;
2408                 uint8_t onoff;
2409                 uint8_t pad2;
2410                 uint8_t ack;
2411         } s;
2412         int error;
2413
2414         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
2415             ("%s: CTRL_RX feature not negotiated", __func__));
2416
2417         s.hdr.class = VIRTIO_NET_CTRL_RX;
2418         s.hdr.cmd = cmd;
2419         s.onoff = !!on;
2420         s.ack = VIRTIO_NET_ERR;
2421
2422         sglist_init(&sg, 3, segs);
2423         error = 0;
2424         error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
2425         error |= sglist_append(&sg, &s.onoff, sizeof(uint8_t));
2426         error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
2427         KASSERT(error == 0 && sg.sg_nseg == 3,
2428             ("%s: error %d adding Rx message to sglist", __func__, error));
2429
2430         vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
2431
2432         return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
2433 }
2434
2435 static int
2436 vtnet_set_promisc(struct vtnet_softc *sc, int on)
2437 {
2438
2439         return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_PROMISC, on));
2440 }
2441
2442 static int
2443 vtnet_set_allmulti(struct vtnet_softc *sc, int on)
2444 {
2445
2446         return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, on));
2447 }
2448
2449 static void
2450 vtnet_rx_filter_mac(struct vtnet_softc *sc)
2451 {
2452         struct virtio_net_ctrl_hdr hdr __aligned(2);
2453         struct vtnet_mac_filter *filter;
2454         struct sglist_seg segs[4];
2455         struct sglist sg;
2456         struct ifnet *ifp;
2457         struct ifaddr *ifa;
2458         struct ifaddr_container *ifac;
2459         struct ifmultiaddr *ifma;
2460         int ucnt, mcnt, promisc, allmulti, error;
2461         uint8_t ack;
2462
2463         ifp = sc->vtnet_ifp;
2464         ucnt = 0;
2465         mcnt = 0;
2466         promisc = 0;
2467         allmulti = 0;
2468
2469         ASSERT_SERIALIZED(&sc->vtnet_slz);
2470         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
2471             ("%s: CTRL_RX feature not negotiated", __func__));
2472
2473         /* Use the MAC filtering table allocated in vtnet_attach. */
2474         filter = sc->vtnet_macfilter;
2475         memset(filter, 0, sizeof(struct vtnet_mac_filter));
2476
2477         /* Unicast MAC addresses: */
2478         //if_addr_rlock(ifp);
2479         TAILQ_FOREACH(ifac, &ifp->if_addrheads[mycpuid], ifa_link) {
2480                 ifa = ifac->ifa;
2481                 if (ifa->ifa_addr->sa_family != AF_LINK)
2482                         continue;
2483                 else if (memcmp(LLADDR((struct sockaddr_dl *)ifa->ifa_addr),
2484                     sc->vtnet_hwaddr, ETHER_ADDR_LEN) == 0)
2485                         continue;
2486                 else if (ucnt == VTNET_MAX_MAC_ENTRIES) {
2487                         promisc = 1;
2488                         break;
2489                 }
2490
2491                 bcopy(LLADDR((struct sockaddr_dl *)ifa->ifa_addr),
2492                     &filter->vmf_unicast.macs[ucnt], ETHER_ADDR_LEN);
2493                 ucnt++;
2494         }
2495         //if_addr_runlock(ifp);
2496
2497         if (promisc != 0) {
2498                 filter->vmf_unicast.nentries = 0;
2499                 if_printf(ifp, "more than %d MAC addresses assigned, "
2500                     "falling back to promiscuous mode\n",
2501                     VTNET_MAX_MAC_ENTRIES);
2502         } else
2503                 filter->vmf_unicast.nentries = ucnt;
2504
2505         /* Multicast MAC addresses: */
2506         //if_maddr_rlock(ifp);
2507         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2508                 if (ifma->ifma_addr->sa_family != AF_LINK)
2509                         continue;
2510                 else if (mcnt == VTNET_MAX_MAC_ENTRIES) {
2511                         allmulti = 1;
2512                         break;
2513                 }
2514
2515                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2516                     &filter->vmf_multicast.macs[mcnt], ETHER_ADDR_LEN);
2517                 mcnt++;
2518         }
2519         //if_maddr_runlock(ifp);
2520
2521         if (allmulti != 0) {
2522                 filter->vmf_multicast.nentries = 0;
2523                 if_printf(ifp, "more than %d multicast MAC addresses "
2524                     "assigned, falling back to all-multicast mode\n",
2525                     VTNET_MAX_MAC_ENTRIES);
2526         } else
2527                 filter->vmf_multicast.nentries = mcnt;
2528
2529         if (promisc != 0 && allmulti != 0)
2530                 goto out;
2531
2532         hdr.class = VIRTIO_NET_CTRL_MAC;
2533         hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
2534         ack = VIRTIO_NET_ERR;
2535
2536         sglist_init(&sg, 4, segs);
2537         error = 0;
2538         error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
2539         error |= sglist_append(&sg, &filter->vmf_unicast,
2540             sizeof(uint32_t) + filter->vmf_unicast.nentries * ETHER_ADDR_LEN);
2541         error |= sglist_append(&sg, &filter->vmf_multicast,
2542             sizeof(uint32_t) + filter->vmf_multicast.nentries * ETHER_ADDR_LEN);
2543         error |= sglist_append(&sg, &ack, sizeof(uint8_t));
2544         KASSERT(error == 0 && sg.sg_nseg == 4,
2545             ("%s: error %d adding MAC filter msg to sglist", __func__, error));
2546
2547         vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
2548
2549         if (ack != VIRTIO_NET_OK)
2550                 if_printf(ifp, "error setting host MAC filter table\n");
2551
2552 out:
2553         if (promisc != 0 && vtnet_set_promisc(sc, 1) != 0)
2554                 if_printf(ifp, "cannot enable promiscuous mode\n");
2555         if (allmulti != 0 && vtnet_set_allmulti(sc, 1) != 0)
2556                 if_printf(ifp, "cannot enable all-multicast mode\n");
2557 }
2558
2559 static int
2560 vtnet_exec_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
2561 {
2562         struct sglist_seg segs[3];
2563         struct sglist sg;
2564         struct {
2565                 struct virtio_net_ctrl_hdr hdr __aligned(2);
2566                 uint8_t pad1;
2567                 uint16_t tag;
2568                 uint8_t pad2;
2569                 uint8_t ack;
2570         } s;
2571         int error;
2572
2573         s.hdr.class = VIRTIO_NET_CTRL_VLAN;
2574         s.hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
2575         s.tag = tag;
2576         s.ack = VIRTIO_NET_ERR;
2577
2578         sglist_init(&sg, 3, segs);
2579         error = 0;
2580         error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
2581         error |= sglist_append(&sg, &s.tag, sizeof(uint16_t));
2582         error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
2583         KASSERT(error == 0 && sg.sg_nseg == 3,
2584             ("%s: error %d adding VLAN message to sglist", __func__, error));
2585
2586         vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
2587
2588         return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
2589 }
2590
2591 static void
2592 vtnet_rx_filter_vlan(struct vtnet_softc *sc)
2593 {
2594         uint32_t w;
2595         uint16_t tag;
2596         int i, bit, nvlans;
2597
2598         ASSERT_SERIALIZED(&sc->vtnet_slz);
2599         KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER,
2600             ("%s: VLAN_FILTER feature not negotiated", __func__));
2601
2602         nvlans = sc->vtnet_nvlans;
2603
2604         /* Enable the filter for each configured VLAN. */
2605         for (i = 0; i < VTNET_VLAN_SHADOW_SIZE && nvlans > 0; i++) {
2606                 w = sc->vtnet_vlan_shadow[i];
2607                 while ((bit = ffs(w) - 1) != -1) {
2608                         w &= ~(1 << bit);
2609                         tag = sizeof(w) * CHAR_BIT * i + bit;
2610                         nvlans--;
2611
2612                         if (vtnet_exec_vlan_filter(sc, 1, tag) != 0) {
2613                                 device_printf(sc->vtnet_dev,
2614                                     "cannot enable VLAN %d filter\n", tag);
2615                         }
2616                 }
2617         }
2618
2619         KASSERT(nvlans == 0, ("VLAN count incorrect"));
2620 }
2621
2622 static void
2623 vtnet_update_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
2624 {
2625         struct ifnet *ifp;
2626         int idx, bit;
2627
2628         ifp = sc->vtnet_ifp;
2629         idx = (tag >> 5) & 0x7F;
2630         bit = tag & 0x1F;
2631
2632         if (tag == 0 || tag > 4095)
2633                 return;
2634
2635         lwkt_serialize_enter(&sc->vtnet_slz);
2636
2637         /* Update shadow VLAN table. */
2638         if (add) {
2639                 sc->vtnet_nvlans++;
2640                 sc->vtnet_vlan_shadow[idx] |= (1 << bit);
2641         } else {
2642                 sc->vtnet_nvlans--;
2643                 sc->vtnet_vlan_shadow[idx] &= ~(1 << bit);
2644         }
2645
2646         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER &&
2647             vtnet_exec_vlan_filter(sc, add, tag) != 0) {
2648                 device_printf(sc->vtnet_dev,
2649                     "cannot %s VLAN %d %s the host filter table\n",
2650                     add ? "add" : "remove", tag, add ? "to" : "from");
2651         }
2652
2653         lwkt_serialize_exit(&sc->vtnet_slz);
2654 }
2655
2656 static void
2657 vtnet_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
2658 {
2659
2660         if (ifp->if_softc != arg)
2661                 return;
2662
2663         vtnet_update_vlan_filter(arg, 1, tag);
2664 }
2665
2666 static void
2667 vtnet_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
2668 {
2669
2670         if (ifp->if_softc != arg)
2671                 return;
2672
2673         vtnet_update_vlan_filter(arg, 0, tag);
2674 }
2675
2676 static int
2677 vtnet_ifmedia_upd(struct ifnet *ifp)
2678 {
2679         struct vtnet_softc *sc;
2680         struct ifmedia *ifm;
2681
2682         sc = ifp->if_softc;
2683         ifm = &sc->vtnet_media;
2684
2685         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
2686                 return (EINVAL);
2687
2688         return (0);
2689 }
2690
2691 static void
2692 vtnet_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
2693 {
2694         struct vtnet_softc *sc;
2695
2696         sc = ifp->if_softc;
2697
2698         ifmr->ifm_status = IFM_AVALID;
2699         ifmr->ifm_active = IFM_ETHER;
2700
2701         lwkt_serialize_enter(&sc->vtnet_slz);
2702         if (vtnet_is_link_up(sc) != 0) {
2703                 ifmr->ifm_status |= IFM_ACTIVE;
2704                 ifmr->ifm_active |= VTNET_MEDIATYPE;
2705         } else
2706                 ifmr->ifm_active |= IFM_NONE;
2707         lwkt_serialize_exit(&sc->vtnet_slz);
2708 }
2709
2710 static void
2711 vtnet_add_statistics(struct vtnet_softc *sc)
2712 {
2713         device_t dev;
2714         struct vtnet_statistics *stats;
2715         struct sysctl_ctx_list *ctx;
2716         struct sysctl_oid *tree;
2717         struct sysctl_oid_list *child;
2718
2719         dev = sc->vtnet_dev;
2720         stats = &sc->vtnet_stats;
2721         ctx = device_get_sysctl_ctx(dev);
2722         tree = device_get_sysctl_tree(dev);
2723         child = SYSCTL_CHILDREN(tree);
2724
2725         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "mbuf_alloc_failed",
2726             CTLFLAG_RD, &stats->mbuf_alloc_failed, 0,
2727             "Mbuf cluster allocation failures");
2728
2729         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_frame_too_large",
2730             CTLFLAG_RD, &stats->rx_frame_too_large, 0,
2731             "Received frame larger than the mbuf chain");
2732         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_enq_replacement_failed",
2733             CTLFLAG_RD, &stats->rx_enq_replacement_failed, 0,
2734             "Enqueuing the replacement receive mbuf failed");
2735         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_mergeable_failed",
2736             CTLFLAG_RD, &stats->rx_mergeable_failed, 0,
2737             "Mergeable buffers receive failures");
2738         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ethtype",
2739             CTLFLAG_RD, &stats->rx_csum_bad_ethtype, 0,
2740             "Received checksum offloaded buffer with unsupported "
2741             "Ethernet type");
2742         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ipproto",
2743             CTLFLAG_RD, &stats->rx_csum_bad_ipproto, 0,
2744             "Received checksum offloaded buffer with incorrect IP protocol");
2745         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_offset",
2746             CTLFLAG_RD, &stats->rx_csum_bad_offset, 0,
2747             "Received checksum offloaded buffer with incorrect offset");
2748         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_failed",
2749             CTLFLAG_RD, &stats->rx_csum_failed, 0,
2750             "Received buffer checksum offload failed");
2751         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_offloaded",
2752             CTLFLAG_RD, &stats->rx_csum_offloaded, 0,
2753             "Received buffer checksum offload succeeded");
2754         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_task_rescheduled",
2755             CTLFLAG_RD, &stats->rx_task_rescheduled, 0,
2756             "Times the receive interrupt task rescheduled itself");
2757
2758         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_bad_ethtype",
2759             CTLFLAG_RD, &stats->tx_csum_bad_ethtype, 0,
2760             "Aborted transmit of checksum offloaded buffer with unknown "
2761             "Ethernet type");
2762         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_bad_ethtype",
2763             CTLFLAG_RD, &stats->tx_tso_bad_ethtype, 0,
2764             "Aborted transmit of TSO buffer with unknown Ethernet type");
2765         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_offloaded",
2766             CTLFLAG_RD, &stats->tx_csum_offloaded, 0,
2767             "Offloaded checksum of transmitted buffer");
2768         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_offloaded",
2769             CTLFLAG_RD, &stats->tx_tso_offloaded, 0,
2770             "Segmentation offload of transmitted buffer");
2771         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_task_rescheduled",
2772             CTLFLAG_RD, &stats->tx_task_rescheduled, 0,
2773             "Times the transmit interrupt task rescheduled itself");
2774 }
2775
2776 static int
2777 vtnet_enable_rx_intr(struct vtnet_softc *sc)
2778 {
2779
2780         return (virtqueue_enable_intr(sc->vtnet_rx_vq));
2781 }
2782
2783 static void
2784 vtnet_disable_rx_intr(struct vtnet_softc *sc)
2785 {
2786
2787         virtqueue_disable_intr(sc->vtnet_rx_vq);
2788 }
2789
2790 static int
2791 vtnet_enable_tx_intr(struct vtnet_softc *sc)
2792 {
2793
2794 #ifdef VTNET_TX_INTR_MODERATION
2795         return (0);
2796 #else
2797         return (virtqueue_enable_intr(sc->vtnet_tx_vq));
2798 #endif
2799 }
2800
2801 static void
2802 vtnet_disable_tx_intr(struct vtnet_softc *sc)
2803 {
2804
2805         virtqueue_disable_intr(sc->vtnet_tx_vq);
2806 }