sys/dev/virtual/virtio/net/if_vtnet.c

   1 /*-
   2  * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice unmodified, this list of conditions, and the following
  10  *    disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 /* Driver for VirtIO network devices. */
  28
  29 #include <sys/cdefs.h>
  30
  31 #include <sys/param.h>
  32 #include <sys/systm.h>
  33 #include <sys/kernel.h>
  34 #include <sys/sockio.h>
  35 #include <sys/mbuf.h>
  36 #include <sys/malloc.h>
  37 #include <sys/module.h>
  38 #include <sys/socket.h>
  39 #include <sys/sysctl.h>
  40 #include <sys/taskqueue.h>
  41 #include <sys/random.h>
  42 #include <sys/sglist.h>
  43 #include <sys/serialize.h>
  44 #include <sys/bus.h>
  45 #include <sys/rman.h>
  46
  47 #include <net/ethernet.h>
  48 #include <net/if.h>
  49 #include <net/if_arp.h>
  50 #include <net/if_dl.h>
  51 #include <net/if_types.h>
  52 #include <net/if_media.h>
  53 #include <net/vlan/if_vlan_var.h>
  54 #include <net/vlan/if_vlan_ether.h>
  55 #include <net/ifq_var.h>
  56
  57 #include <net/bpf.h>
  58
  59 #include <netinet/in_systm.h>
  60 #include <netinet/in.h>
  61 #include <netinet/ip.h>
  62 #include <netinet/ip6.h>
  63 #include <netinet/udp.h>
  64 #include <netinet/tcp.h>
  65
  66 #include <dev/virtual/virtio/virtio/virtio.h>
  67 #include <dev/virtual/virtio/virtio/virtqueue.h>
  68
  69 #include "virtio_net.h"
  70 #include "virtio_if.h"
  71
  72 struct vtnet_statistics {
  73         unsigned long           mbuf_alloc_failed;
  74
  75         unsigned long           rx_frame_too_large;
  76         unsigned long           rx_enq_replacement_failed;
  77         unsigned long           rx_mergeable_failed;
  78         unsigned long           rx_csum_bad_ethtype;
  79         unsigned long           rx_csum_bad_start;
  80         unsigned long           rx_csum_bad_ipproto;
  81         unsigned long           rx_csum_bad_offset;
  82         unsigned long           rx_csum_failed;
  83         unsigned long           rx_csum_offloaded;
  84         unsigned long           rx_task_rescheduled;
  85
  86         unsigned long           tx_csum_offloaded;
  87         unsigned long           tx_tso_offloaded;
  88         unsigned long           tx_csum_bad_ethtype;
  89         unsigned long           tx_tso_bad_ethtype;
  90         unsigned long           tx_task_rescheduled;
  91 };
  92
  93 struct vtnet_softc {
  94         device_t                vtnet_dev;
  95         struct ifnet            *vtnet_ifp;
  96         struct lwkt_serialize   vtnet_slz;
  97
  98         uint32_t                vtnet_flags;
  99 #define VTNET_FLAG_LINK         0x0001
 100 #define VTNET_FLAG_SUSPENDED    0x0002
 101 #define VTNET_FLAG_MAC          0x0004
 102 #define VTNET_FLAG_CTRL_VQ      0x0008
 103 #define VTNET_FLAG_CTRL_RX      0x0010
 104 #define VTNET_FLAG_CTRL_MAC     0x0020
 105 #define VTNET_FLAG_VLAN_FILTER  0x0040
 106 #define VTNET_FLAG_TSO_ECN      0x0080
 107 #define VTNET_FLAG_MRG_RXBUFS   0x0100
 108 #define VTNET_FLAG_LRO_NOMRG    0x0200
 109
 110         struct virtqueue        *vtnet_rx_vq;
 111         struct virtqueue        *vtnet_tx_vq;
 112         struct virtqueue        *vtnet_ctrl_vq;
 113
 114         struct vtnet_tx_header  *vtnet_txhdrarea;
 115         uint32_t                vtnet_txhdridx;
 116         struct vtnet_mac_filter *vtnet_macfilter;
 117
 118         int                     vtnet_hdr_size;
 119         int                     vtnet_tx_size;
 120         int                     vtnet_rx_size;
 121         int                     vtnet_rx_process_limit;
 122         int                     vtnet_rx_mbuf_size;
 123         int                     vtnet_rx_mbuf_count;
 124         int                     vtnet_if_flags;
 125         int                     vtnet_watchdog_timer;
 126         uint64_t                vtnet_features;
 127
 128         struct task             vtnet_cfgchg_task;
 129
 130         struct vtnet_statistics vtnet_stats;
 131
 132         struct callout          vtnet_tick_ch;
 133
 134         eventhandler_tag        vtnet_vlan_attach;
 135         eventhandler_tag        vtnet_vlan_detach;
 136
 137         struct ifmedia          vtnet_media;
 138         /*
 139          * Fake media type; the host does not provide us with
 140          * any real media information.
 141          */
 142 #define VTNET_MEDIATYPE         (IFM_ETHER | IFM_1000_T | IFM_FDX)
 143         char                    vtnet_hwaddr[ETHER_ADDR_LEN];
 144
 145         /*
 146          * During reset, the host's VLAN filtering table is lost. The
 147          * array below is used to restore all the VLANs configured on
 148          * this interface after a reset.
 149          */
 150 #define VTNET_VLAN_SHADOW_SIZE  (4096 / 32)
 151         int                     vtnet_nvlans;
 152         uint32_t                vtnet_vlan_shadow[VTNET_VLAN_SHADOW_SIZE];
 153
 154         char                    vtnet_mtx_name[16];
 155 };
 156
 157 /*
 158  * When mergeable buffers are not negotiated, the vtnet_rx_header structure
 159  * below is placed at the beginning of the mbuf data. Use 4 bytes of pad to
 160  * both keep the VirtIO header and the data non-contiguous and to keep the
 161  * frame's payload 4 byte aligned.
 162  *
 163  * When mergeable buffers are negotiated, the host puts the VirtIO header in
 164  * the beginning of the first mbuf's data.
 165  */
 166 #define VTNET_RX_HEADER_PAD     4
 167 struct vtnet_rx_header {
 168         struct virtio_net_hdr   vrh_hdr;
 169         char                    vrh_pad[VTNET_RX_HEADER_PAD];
 170 } __packed;
 171
 172 /*
 173  * For each outgoing frame, the vtnet_tx_header below is allocated from
 174  * the vtnet_tx_header_zone.
 175  */
 176 struct vtnet_tx_header {
 177         union {
 178                 struct virtio_net_hdr           hdr;
 179                 struct virtio_net_hdr_mrg_rxbuf mhdr;
 180         } vth_uhdr;
 181
 182         struct mbuf             *vth_mbuf;
 183 };
 184
 185 MALLOC_DEFINE(M_VTNET, "VTNET_TX", "Outgoing VTNET TX frame header");
 186
 187 /*
 188  * The VirtIO specification does not place a limit on the number of MAC
 189  * addresses the guest driver may request to be filtered. In practice,
 190  * the host is constrained by available resources. To simplify this driver,
 191  * impose a reasonably high limit of MAC addresses we will filter before
 192  * falling back to promiscuous or all-multicast modes.
 193  */
 194 #define VTNET_MAX_MAC_ENTRIES   128
 195
 196 struct vtnet_mac_table {
 197         uint32_t                nentries;
 198         uint8_t                 macs[VTNET_MAX_MAC_ENTRIES][ETHER_ADDR_LEN];
 199 } __packed;
 200
 201 struct vtnet_mac_filter {
 202         struct vtnet_mac_table  vmf_unicast;
 203         uint32_t                vmf_pad; /* Make tables non-contiguous. */
 204         struct vtnet_mac_table  vmf_multicast;
 205 };
 206
 207 #define VTNET_WATCHDOG_TIMEOUT  5
 208 #define VTNET_CSUM_OFFLOAD      (CSUM_TCP | CSUM_UDP)
 209
 210 /* Features desired/implemented by this driver. */
 211 #define VTNET_FEATURES          \
 212     (VIRTIO_NET_F_MAC           | \
 213      VIRTIO_NET_F_STATUS        | \
 214      VIRTIO_NET_F_CTRL_VQ       | \
 215      VIRTIO_NET_F_CTRL_RX       | \
 216      VIRTIO_NET_F_CTRL_MAC_ADDR | \
 217      VIRTIO_NET_F_CTRL_VLAN     | \
 218      VIRTIO_NET_F_CSUM          | \
 219      VIRTIO_NET_F_HOST_TSO4     | \
 220      VIRTIO_NET_F_HOST_TSO6     | \
 221      VIRTIO_NET_F_HOST_ECN      | \
 222      VIRTIO_NET_F_GUEST_CSUM    | \
 223      VIRTIO_NET_F_GUEST_TSO4    | \
 224      VIRTIO_NET_F_GUEST_TSO6    | \
 225      VIRTIO_NET_F_GUEST_ECN     | \
 226      VIRTIO_NET_F_MRG_RXBUF)
 227
 228 /*
 229  * The VIRTIO_NET_F_GUEST_TSO[46] features permit the host to send us
 230  * frames larger than 1514 bytes. We do not yet support software LRO
 231  * via tcp_lro_rx().
 232  */
 233 #define VTNET_LRO_FEATURES (VIRTIO_NET_F_GUEST_TSO4 | \
 234                             VIRTIO_NET_F_GUEST_TSO6 | VIRTIO_NET_F_GUEST_ECN)
 235
 236 #define VTNET_MAX_MTU           65536
 237 #define VTNET_MAX_RX_SIZE       65550
 238
 239 /*
 240  * Used to preallocate the Vq indirect descriptors. The first segment
 241  * is reserved for the header.
 242  */
 243 #define VTNET_MIN_RX_SEGS       2
 244 #define VTNET_MAX_RX_SEGS       34
 245 #define VTNET_MAX_TX_SEGS       34
 246
 247 #define IFCAP_TSO4              0x00100 /* can do TCP Segmentation Offload */
 248 #define IFCAP_TSO6              0x00200 /* can do TCP6 Segmentation Offload */
 249 #define IFCAP_LRO               0x00400 /* can do Large Receive Offload */
 250 #define IFCAP_VLAN_HWFILTER     0x10000 /* interface hw can filter vlan tag */
 251 #define IFCAP_VLAN_HWTSO        0x40000 /* can do IFCAP_TSO on VLANs */
 252
 253
 254 /*
 255  * Assert we can receive and transmit the maximum with regular
 256  * size clusters.
 257  */
 258 CTASSERT(((VTNET_MAX_RX_SEGS - 1) * MCLBYTES) >= VTNET_MAX_RX_SIZE);
 259 CTASSERT(((VTNET_MAX_TX_SEGS - 1) * MCLBYTES) >= VTNET_MAX_MTU);
 260
 261 /*
 262  * Determine how many mbufs are in each receive buffer. For LRO without
 263  * mergeable descriptors, we must allocate an mbuf chain large enough to
 264  * hold both the vtnet_rx_header and the maximum receivable data.
 265  */
 266 #define VTNET_NEEDED_RX_MBUFS(_sc)                                      \
 267         ((_sc)->vtnet_flags & VTNET_FLAG_LRO_NOMRG) == 0 ? 1 :          \
 268         howmany(sizeof(struct vtnet_rx_header) + VTNET_MAX_RX_SIZE,     \
 269         (_sc)->vtnet_rx_mbuf_size)
 270
 271 static int      vtnet_modevent(module_t, int, void *);
 272
 273 static int      vtnet_probe(device_t);
 274 static int      vtnet_attach(device_t);
 275 static int      vtnet_detach(device_t);
 276 static int      vtnet_suspend(device_t);
 277 static int      vtnet_resume(device_t);
 278 static int      vtnet_shutdown(device_t);
 279 static int      vtnet_config_change(device_t);
 280
 281 static void     vtnet_negotiate_features(struct vtnet_softc *);
 282 static int      vtnet_alloc_virtqueues(struct vtnet_softc *);
 283 static void     vtnet_get_hwaddr(struct vtnet_softc *);
 284 static void     vtnet_set_hwaddr(struct vtnet_softc *);
 285 static int      vtnet_is_link_up(struct vtnet_softc *);
 286 static void     vtnet_update_link_status(struct vtnet_softc *);
 287 #if 0
 288 static void     vtnet_watchdog(struct vtnet_softc *);
 289 #endif
 290 static void     vtnet_config_change_task(void *, int);
 291 static int      vtnet_change_mtu(struct vtnet_softc *, int);
 292 static int      vtnet_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 293
 294 static int      vtnet_init_rx_vq(struct vtnet_softc *);
 295 static void     vtnet_free_rx_mbufs(struct vtnet_softc *);
 296 static void     vtnet_free_tx_mbufs(struct vtnet_softc *);
 297 static void     vtnet_free_ctrl_vq(struct vtnet_softc *);
 298
 299 static struct mbuf * vtnet_alloc_rxbuf(struct vtnet_softc *, int,
 300                     struct mbuf **);
 301 static int      vtnet_replace_rxbuf(struct vtnet_softc *,
 302                     struct mbuf *, int);
 303 static int      vtnet_newbuf(struct vtnet_softc *);
 304 static void     vtnet_discard_merged_rxbuf(struct vtnet_softc *, int);
 305 static void     vtnet_discard_rxbuf(struct vtnet_softc *, struct mbuf *);
 306 static int      vtnet_enqueue_rxbuf(struct vtnet_softc *, struct mbuf *);
 307 static void     vtnet_vlan_tag_remove(struct mbuf *);
 308 static int      vtnet_rx_csum(struct vtnet_softc *, struct mbuf *,
 309                     struct virtio_net_hdr *);
 310 static int      vtnet_rxeof_merged(struct vtnet_softc *, struct mbuf *, int);
 311 static int      vtnet_rxeof(struct vtnet_softc *, int, int *);
 312 static void     vtnet_rx_intr_task(void *);
 313 static int      vtnet_rx_vq_intr(void *);
 314
 315 static void     vtnet_txeof(struct vtnet_softc *);
 316 static struct mbuf * vtnet_tx_offload(struct vtnet_softc *, struct mbuf *,
 317                     struct virtio_net_hdr *);
 318 static int      vtnet_enqueue_txbuf(struct vtnet_softc *, struct mbuf **,
 319                     struct vtnet_tx_header *);
 320 static int      vtnet_encap(struct vtnet_softc *, struct mbuf **);
 321 static void     vtnet_start_locked(struct ifnet *, struct ifaltq_subque *);
 322 static void     vtnet_start(struct ifnet *, struct ifaltq_subque *);
 323 static void     vtnet_tick(void *);
 324 static void     vtnet_tx_intr_task(void *);
 325 static int      vtnet_tx_vq_intr(void *);
 326
 327 static void     vtnet_stop(struct vtnet_softc *);
 328 static int      vtnet_reinit(struct vtnet_softc *);
 329 static void     vtnet_init_locked(struct vtnet_softc *);
 330 static void     vtnet_init(void *);
 331
 332 static void     vtnet_exec_ctrl_cmd(struct vtnet_softc *, void *,
 333                     struct sglist *, int, int);
 334
 335 static int      vtnet_ctrl_mac_cmd(struct vtnet_softc *, uint8_t *);
 336 static int      vtnet_ctrl_rx_cmd(struct vtnet_softc *, int, int);
 337 static int      vtnet_set_promisc(struct vtnet_softc *, int);
 338 static int      vtnet_set_allmulti(struct vtnet_softc *, int);
 339 static void     vtnet_rx_filter(struct vtnet_softc *sc);
 340 static void     vtnet_rx_filter_mac(struct vtnet_softc *);
 341
 342 static int      vtnet_exec_vlan_filter(struct vtnet_softc *, int, uint16_t);
 343 static void     vtnet_rx_filter_vlan(struct vtnet_softc *);
 344 static void     vtnet_set_vlan_filter(struct vtnet_softc *, int, uint16_t);
 345 static void     vtnet_register_vlan(void *, struct ifnet *, uint16_t);
 346 static void     vtnet_unregister_vlan(void *, struct ifnet *, uint16_t);
 347
 348 static int      vtnet_ifmedia_upd(struct ifnet *);
 349 static void     vtnet_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 350
 351 static void     vtnet_add_statistics(struct vtnet_softc *);
 352
 353 static int      vtnet_enable_rx_intr(struct vtnet_softc *);
 354 static int      vtnet_enable_tx_intr(struct vtnet_softc *);
 355 static void     vtnet_disable_rx_intr(struct vtnet_softc *);
 356 static void     vtnet_disable_tx_intr(struct vtnet_softc *);
 357
 358 /* Tunables. */
 359 static int vtnet_csum_disable = 0;
 360 TUNABLE_INT("hw.vtnet.csum_disable", &vtnet_csum_disable);
 361 static int vtnet_tso_disable = 1;
 362 TUNABLE_INT("hw.vtnet.tso_disable", &vtnet_tso_disable);
 363 static int vtnet_lro_disable = 1;
 364 TUNABLE_INT("hw.vtnet.lro_disable", &vtnet_lro_disable);
 365
 366 /*
 367  * Reducing the number of transmit completed interrupts can
 368  * improve performance. To do so, the define below keeps the
 369  * Tx vq interrupt disabled and adds calls to vtnet_txeof()
 370  * in the start and watchdog paths. The price to pay for this
 371  * is the m_free'ing of transmitted mbufs may be delayed until
 372  * the watchdog fires.
 373  */
 374 #define VTNET_TX_INTR_MODERATION
 375
 376 static struct virtio_feature_desc vtnet_feature_desc[] = {
 377         { VIRTIO_NET_F_CSUM,            "TxChecksum"    },
 378         { VIRTIO_NET_F_GUEST_CSUM,      "RxChecksum"    },
 379         { VIRTIO_NET_F_MAC,             "MacAddress"    },
 380         { VIRTIO_NET_F_GSO,             "TxAllGSO"      },
 381         { VIRTIO_NET_F_GUEST_TSO4,      "RxTSOv4"       },
 382         { VIRTIO_NET_F_GUEST_TSO6,      "RxTSOv6"       },
 383         { VIRTIO_NET_F_GUEST_ECN,       "RxECN"         },
 384         { VIRTIO_NET_F_GUEST_UFO,       "RxUFO"         },
 385         { VIRTIO_NET_F_HOST_TSO4,       "TxTSOv4"       },
 386         { VIRTIO_NET_F_HOST_TSO6,       "TxTSOv6"       },
 387         { VIRTIO_NET_F_HOST_ECN,        "TxTSOECN"      },
 388         { VIRTIO_NET_F_HOST_UFO,        "TxUFO"         },
 389         { VIRTIO_NET_F_MRG_RXBUF,       "MrgRxBuf"      },
 390         { VIRTIO_NET_F_STATUS,          "Status"        },
 391         { VIRTIO_NET_F_CTRL_VQ,         "ControlVq"     },
 392         { VIRTIO_NET_F_CTRL_RX,         "RxMode"        },
 393         { VIRTIO_NET_F_CTRL_VLAN,       "VLanFilter"    },
 394         { VIRTIO_NET_F_CTRL_RX_EXTRA,   "RxModeExtra"   },
 395         { VIRTIO_NET_F_GUEST_ANNOUNCE,  "GuestAnnounce" },
 396         { VIRTIO_NET_F_MQ,              "RFS"           },
 397         { VIRTIO_NET_F_CTRL_MAC_ADDR,   "SetMacAddress" },
 398         { 0, NULL }
 399 };
 400
 401 static device_method_t vtnet_methods[] = {
 402         /* Device methods. */
 403         DEVMETHOD(device_probe,         vtnet_probe),
 404         DEVMETHOD(device_attach,        vtnet_attach),
 405         DEVMETHOD(device_detach,        vtnet_detach),
 406         DEVMETHOD(device_suspend,       vtnet_suspend),
 407         DEVMETHOD(device_resume,        vtnet_resume),
 408         DEVMETHOD(device_shutdown,      vtnet_shutdown),
 409
 410         /* VirtIO methods. */
 411         DEVMETHOD(virtio_config_change, vtnet_config_change),
 412
 413         { 0, 0 }
 414 };
 415
 416 static driver_t vtnet_driver = {
 417         "vtnet",
 418         vtnet_methods,
 419         sizeof(struct vtnet_softc)
 420 };
 421
 422 static devclass_t vtnet_devclass;
 423
 424 DRIVER_MODULE(vtnet, virtio_pci, vtnet_driver, vtnet_devclass,
 425     vtnet_modevent, 0);
 426 MODULE_VERSION(vtnet, 1);
 427 MODULE_DEPEND(vtnet, virtio, 1, 1, 1);
 428
 429 static int
 430 vtnet_modevent(module_t mod, int type, void *unused)
 431 {
 432         int error;
 433
 434         error = 0;
 435
 436         switch (type) {
 437         case MOD_LOAD:
 438                 break;
 439         case MOD_UNLOAD:
 440                 break;
 441         case MOD_SHUTDOWN:
 442                 break;
 443         default:
 444                 error = EOPNOTSUPP;
 445                 break;
 446         }
 447
 448         return (error);
 449 }
 450
 451 static int
 452 vtnet_probe(device_t dev)
 453 {
 454         if (virtio_get_device_type(dev) != VIRTIO_ID_NETWORK)
 455                 return (ENXIO);
 456
 457         device_set_desc(dev, "VirtIO Networking Adapter");
 458
 459         return (BUS_PROBE_DEFAULT);
 460 }
 461
 462 static int
 463 vtnet_attach(device_t dev)
 464 {
 465         struct vtnet_softc *sc;
 466         struct ifnet *ifp;
 467         int tx_size, error;
 468
 469         sc = device_get_softc(dev);
 470         sc->vtnet_dev = dev;
 471
 472         lwkt_serialize_init(&sc->vtnet_slz);
 473         callout_init(&sc->vtnet_tick_ch);
 474
 475         ifmedia_init(&sc->vtnet_media, IFM_IMASK, vtnet_ifmedia_upd,
 476                      vtnet_ifmedia_sts);
 477         ifmedia_add(&sc->vtnet_media, VTNET_MEDIATYPE, 0, NULL);
 478         ifmedia_set(&sc->vtnet_media, VTNET_MEDIATYPE);
 479
 480         vtnet_add_statistics(sc);
 481
 482         /* Register our feature descriptions. */
 483         virtio_set_feature_desc(dev, vtnet_feature_desc);
 484         vtnet_negotiate_features(sc);
 485
 486         if (virtio_with_feature(dev, VIRTIO_NET_F_MAC)) {
 487                 /* This feature should always be negotiated. */
 488                 sc->vtnet_flags |= VTNET_FLAG_MAC;
 489         }
 490
 491         if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) {
 492                 sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS;
 493                 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 494         } else {
 495                 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
 496         }
 497
 498         sc->vtnet_rx_mbuf_size = MCLBYTES;
 499         sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc);
 500
 501         if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VQ)) {
 502                 sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ;
 503
 504                 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX))
 505                         sc->vtnet_flags |= VTNET_FLAG_CTRL_RX;
 506                 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VLAN))
 507                         sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER;
 508                 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_MAC_ADDR) &&
 509                     virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX))
 510                         sc->vtnet_flags |= VTNET_FLAG_CTRL_MAC;
 511         }
 512
 513         /* Read (or generate) the MAC address for the adapter. */
 514         vtnet_get_hwaddr(sc);
 515
 516         error = vtnet_alloc_virtqueues(sc);
 517         if (error) {
 518                 device_printf(dev, "cannot allocate virtqueues\n");
 519                 goto fail;
 520         }
 521
 522         ifp = sc->vtnet_ifp = if_alloc(IFT_ETHER);
 523         if (ifp == NULL) {
 524                 device_printf(dev, "cannot allocate ifnet structure\n");
 525                 error = ENOSPC;
 526                 goto fail;
 527         }
 528
 529         ifp->if_softc = sc;
 530         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 531         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 532         ifp->if_init = vtnet_init;
 533         ifp->if_start = vtnet_start;
 534         ifp->if_ioctl = vtnet_ioctl;
 535
 536         sc->vtnet_rx_size = virtqueue_size(sc->vtnet_rx_vq);
 537         sc->vtnet_rx_process_limit = sc->vtnet_rx_size;
 538
 539         tx_size = virtqueue_size(sc->vtnet_tx_vq);
 540         sc->vtnet_tx_size = tx_size;
 541         sc->vtnet_txhdridx = 0;
 542         sc->vtnet_txhdrarea = contigmalloc(
 543             ((sc->vtnet_tx_size / 2) + 1) * sizeof(struct vtnet_tx_header),
 544             M_VTNET, M_WAITOK, 0, BUS_SPACE_MAXADDR, 4, 0);
 545         if (sc->vtnet_txhdrarea == NULL) {
 546                 device_printf(dev, "cannot contigmalloc the tx headers\n");
 547                 goto fail;
 548         }
 549         sc->vtnet_macfilter = contigmalloc(
 550             sizeof(struct vtnet_mac_filter),
 551             M_DEVBUF, M_WAITOK, 0, BUS_SPACE_MAXADDR, 4, 0);
 552         if (sc->vtnet_macfilter == NULL) {
 553                 device_printf(dev,
 554                     "cannot contigmalloc the mac filter table\n");
 555                 goto fail;
 556         }
 557         ifq_set_maxlen(&ifp->if_snd, tx_size - 1);
 558         ifq_set_ready(&ifp->if_snd);
 559
 560         ether_ifattach(ifp, sc->vtnet_hwaddr, NULL);
 561
 562         if (virtio_with_feature(dev, VIRTIO_NET_F_STATUS)){
 563                 //ifp->if_capabilities |= IFCAP_LINKSTATE;
 564                  kprintf("add dynamic link state\n");
 565         }
 566
 567         /* Tell the upper layer(s) we support long frames. */
 568         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
 569         ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU;
 570
 571         if (virtio_with_feature(dev, VIRTIO_NET_F_CSUM)) {
 572                 ifp->if_capabilities |= IFCAP_TXCSUM;
 573
 574                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4))
 575                         ifp->if_capabilities |= IFCAP_TSO4;
 576                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
 577                         ifp->if_capabilities |= IFCAP_TSO6;
 578                 if (ifp->if_capabilities & IFCAP_TSO)
 579                         ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
 580
 581                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN))
 582                         sc->vtnet_flags |= VTNET_FLAG_TSO_ECN;
 583         }
 584
 585         if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_CSUM)) {
 586                 ifp->if_capabilities |= IFCAP_RXCSUM;
 587
 588                 if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) ||
 589                     virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6))
 590                         ifp->if_capabilities |= IFCAP_LRO;
 591         }
 592
 593         if (ifp->if_capabilities & IFCAP_HWCSUM) {
 594                 /*
 595                  * VirtIO does not support VLAN tagging, but we can fake
 596                  * it by inserting and removing the 802.1Q header during
 597                  * transmit and receive. We are then able to do checksum
 598                  * offloading of VLAN frames.
 599                  */
 600                 ifp->if_capabilities |=
 601                         IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM;
 602         }
 603
 604         ifp->if_capenable = ifp->if_capabilities;
 605
 606         /*
 607          * Capabilities after here are not enabled by default.
 608          */
 609
 610         if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) {
 611                 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
 612
 613                 sc->vtnet_vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 614                     vtnet_register_vlan, sc, EVENTHANDLER_PRI_FIRST);
 615                 sc->vtnet_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 616                     vtnet_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST);
 617         }
 618
 619         TASK_INIT(&sc->vtnet_cfgchg_task, 0, vtnet_config_change_task, sc);
 620
 621         error = virtio_setup_intr(dev, &sc->vtnet_slz);
 622         if (error) {
 623                 device_printf(dev, "cannot setup virtqueue interrupts\n");
 624                 ether_ifdetach(ifp);
 625                 goto fail;
 626         }
 627
 628         /*
 629          * Device defaults to promiscuous mode for backwards
 630          * compatibility. Turn it off if possible.
 631          */
 632         if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
 633                 lwkt_serialize_enter(&sc->vtnet_slz);
 634                 if (vtnet_set_promisc(sc, 0) != 0) {
 635                         ifp->if_flags |= IFF_PROMISC;
 636                         device_printf(dev,
 637                             "cannot disable promiscuous mode\n");
 638                 }
 639                 lwkt_serialize_exit(&sc->vtnet_slz);
 640         } else
 641                 ifp->if_flags |= IFF_PROMISC;
 642
 643 fail:
 644         if (error)
 645                 vtnet_detach(dev);
 646
 647         return (error);
 648 }
 649
 650 static int
 651 vtnet_detach(device_t dev)
 652 {
 653         struct vtnet_softc *sc;
 654         struct ifnet *ifp;
 655
 656         sc = device_get_softc(dev);
 657         ifp = sc->vtnet_ifp;
 658
 659         if (device_is_attached(dev)) {
 660                 lwkt_serialize_enter(&sc->vtnet_slz);
 661                 vtnet_stop(sc);
 662                 lwkt_serialize_exit(&sc->vtnet_slz);
 663
 664                 callout_stop(&sc->vtnet_tick_ch);
 665                 taskqueue_drain(taskqueue_swi, &sc->vtnet_cfgchg_task);
 666
 667                 ether_ifdetach(ifp);
 668         }
 669
 670         if (sc->vtnet_vlan_attach != NULL) {
 671                 EVENTHANDLER_DEREGISTER(vlan_config, sc->vtnet_vlan_attach);
 672                 sc->vtnet_vlan_attach = NULL;
 673         }
 674         if (sc->vtnet_vlan_detach != NULL) {
 675                 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vtnet_vlan_detach);
 676                 sc->vtnet_vlan_detach = NULL;
 677         }
 678
 679         if (ifp) {
 680                 if_free(ifp);
 681                 sc->vtnet_ifp = NULL;
 682         }
 683
 684         if (sc->vtnet_rx_vq != NULL)
 685                 vtnet_free_rx_mbufs(sc);
 686         if (sc->vtnet_tx_vq != NULL)
 687                 vtnet_free_tx_mbufs(sc);
 688         if (sc->vtnet_ctrl_vq != NULL)
 689                 vtnet_free_ctrl_vq(sc);
 690
 691         if (sc->vtnet_txhdrarea != NULL) {
 692                 contigfree(sc->vtnet_txhdrarea,
 693                     ((sc->vtnet_tx_size / 2) + 1) *
 694                     sizeof(struct vtnet_tx_header), M_VTNET);
 695                 sc->vtnet_txhdrarea = NULL;
 696         }
 697         if (sc->vtnet_macfilter != NULL) {
 698                 contigfree(sc->vtnet_macfilter,
 699                     sizeof(struct vtnet_mac_filter), M_DEVBUF);
 700                 sc->vtnet_macfilter = NULL;
 701         }
 702
 703         ifmedia_removeall(&sc->vtnet_media);
 704
 705         return (0);
 706 }
 707
 708 static int
 709 vtnet_suspend(device_t dev)
 710 {
 711         struct vtnet_softc *sc;
 712
 713         sc = device_get_softc(dev);
 714
 715         lwkt_serialize_enter(&sc->vtnet_slz);
 716         vtnet_stop(sc);
 717         sc->vtnet_flags |= VTNET_FLAG_SUSPENDED;
 718         lwkt_serialize_exit(&sc->vtnet_slz);
 719
 720         return (0);
 721 }
 722
 723 static int
 724 vtnet_resume(device_t dev)
 725 {
 726         struct vtnet_softc *sc;
 727         struct ifnet *ifp;
 728
 729         sc = device_get_softc(dev);
 730         ifp = sc->vtnet_ifp;
 731
 732         lwkt_serialize_enter(&sc->vtnet_slz);
 733         if (ifp->if_flags & IFF_UP)
 734                 vtnet_init_locked(sc);
 735         sc->vtnet_flags &= ~VTNET_FLAG_SUSPENDED;
 736         lwkt_serialize_exit(&sc->vtnet_slz);
 737
 738         return (0);
 739 }
 740
 741 static int
 742 vtnet_shutdown(device_t dev)
 743 {
 744
 745         /*
 746          * Suspend already does all of what we need to
 747          * do here; we just never expect to be resumed.
 748          */
 749         return (vtnet_suspend(dev));
 750 }
 751
 752 static int
 753 vtnet_config_change(device_t dev)
 754 {
 755         struct vtnet_softc *sc;
 756
 757         sc = device_get_softc(dev);
 758
 759         taskqueue_enqueue(taskqueue_thread[mycpuid], &sc->vtnet_cfgchg_task);
 760
 761         return (1);
 762 }
 763
 764 static void
 765 vtnet_negotiate_features(struct vtnet_softc *sc)
 766 {
 767         device_t dev;
 768         uint64_t mask, features;
 769
 770         dev = sc->vtnet_dev;
 771         mask = 0;
 772
 773         if (vtnet_csum_disable)
 774                 mask |= VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM;
 775
 776         /*
 777          * TSO and LRO are only available when their corresponding checksum
 778          * offload feature is also negotiated.
 779          */
 780
 781         if (vtnet_csum_disable || vtnet_tso_disable)
 782                 mask |= VIRTIO_NET_F_HOST_TSO4 | VIRTIO_NET_F_HOST_TSO6 |
 783                     VIRTIO_NET_F_HOST_ECN;
 784
 785         if (vtnet_csum_disable || vtnet_lro_disable)
 786                 mask |= VTNET_LRO_FEATURES;
 787
 788         features = VTNET_FEATURES & ~mask;
 789         features |= VIRTIO_F_NOTIFY_ON_EMPTY;
 790         sc->vtnet_features = virtio_negotiate_features(dev, features);
 791
 792         if (virtio_with_feature(dev, VTNET_LRO_FEATURES) &&
 793             virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF) == 0) {
 794                 /*
 795                  * LRO without mergeable buffers requires special care. This
 796                  * is not ideal because every receive buffer must be large
 797                  * enough to hold the maximum TCP packet, the Ethernet header,
 798                  * and the header. This requires up to 34 descriptors with
 799                  * MCLBYTES clusters. If we do not have indirect descriptors,
 800                  * LRO is disabled since the virtqueue will not contain very
 801                  * many receive buffers.
 802                  */
 803                 if (!virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) {
 804                         device_printf(dev,
 805                             "LRO disabled due to both mergeable buffers and "
 806                             "indirect descriptors not negotiated\n");
 807
 808                         features &= ~VTNET_LRO_FEATURES;
 809                         sc->vtnet_features =
 810                             virtio_negotiate_features(dev, features);
 811                 } else
 812                         sc->vtnet_flags |= VTNET_FLAG_LRO_NOMRG;
 813         }
 814 }
 815
 816 static int
 817 vtnet_alloc_virtqueues(struct vtnet_softc *sc)
 818 {
 819         device_t dev;
 820         struct vq_alloc_info vq_info[3];
 821         int nvqs, rxsegs;
 822
 823         dev = sc->vtnet_dev;
 824         nvqs = 2;
 825
 826         /*
 827          * Indirect descriptors are not needed for the Rx
 828          * virtqueue when mergeable buffers are negotiated.
 829          * The header is placed inline with the data, not
 830          * in a separate descriptor, and mbuf clusters are
 831          * always physically contiguous.
 832          */
 833         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
 834                 rxsegs = sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG ?
 835                     VTNET_MAX_RX_SEGS : VTNET_MIN_RX_SEGS;
 836         } else
 837                 rxsegs = 0;
 838
 839         VQ_ALLOC_INFO_INIT(&vq_info[0], rxsegs,
 840             vtnet_rx_vq_intr, sc, &sc->vtnet_rx_vq,
 841             "%s receive", device_get_nameunit(dev));
 842
 843         VQ_ALLOC_INFO_INIT(&vq_info[1], VTNET_MAX_TX_SEGS,
 844             vtnet_tx_vq_intr, sc, &sc->vtnet_tx_vq,
 845             "%s transmit", device_get_nameunit(dev));
 846
 847         if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
 848                 nvqs++;
 849
 850                 VQ_ALLOC_INFO_INIT(&vq_info[2], 0, NULL, NULL,
 851                     &sc->vtnet_ctrl_vq, "%s control",
 852                     device_get_nameunit(dev));
 853         }
 854
 855         return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
 856 }
 857
 858 static void
 859 vtnet_set_hwaddr(struct vtnet_softc *sc)
 860 {
 861         device_t dev;
 862
 863         dev = sc->vtnet_dev;
 864
 865         if ((sc->vtnet_flags & VTNET_FLAG_CTRL_MAC) &&
 866             (sc->vtnet_flags & VTNET_FLAG_CTRL_RX)) {
 867                 if (vtnet_ctrl_mac_cmd(sc, sc->vtnet_hwaddr) != 0)
 868                         device_printf(dev, "unable to set MAC address\n");
 869         } else if (sc->vtnet_flags & VTNET_FLAG_MAC) {
 870                 virtio_write_device_config(dev,
 871                     offsetof(struct virtio_net_config, mac),
 872                     sc->vtnet_hwaddr, ETHER_ADDR_LEN);
 873         }
 874 }
 875
 876 static void
 877 vtnet_get_hwaddr(struct vtnet_softc *sc)
 878 {
 879         device_t dev;
 880
 881         dev = sc->vtnet_dev;
 882
 883         if ((sc->vtnet_flags & VTNET_FLAG_MAC) == 0) {
 884                 /*
 885                  * Generate a random locally administered unicast address.
 886                  *
 887                  * It would be nice to generate the same MAC address across
 888                  * reboots, but it seems all the hosts currently available
 889                  * support the MAC feature, so this isn't too important.
 890                  */
 891                 sc->vtnet_hwaddr[0] = 0xB2;
 892                 karc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1);
 893                 vtnet_set_hwaddr(sc);
 894                 return;
 895         }
 896
 897         virtio_read_device_config(dev,
 898             offsetof(struct virtio_net_config, mac),
 899             sc->vtnet_hwaddr, ETHER_ADDR_LEN);
 900 }
 901
 902 static int
 903 vtnet_is_link_up(struct vtnet_softc *sc)
 904 {
 905         device_t dev;
 906         struct ifnet *ifp;
 907         uint16_t status;
 908
 909         dev = sc->vtnet_dev;
 910         ifp = sc->vtnet_ifp;
 911
 912         ASSERT_SERIALIZED(&sc->vtnet_slz);
 913
 914         status = virtio_read_dev_config_2(dev,
 915                         offsetof(struct virtio_net_config, status));
 916
 917         return ((status & VIRTIO_NET_S_LINK_UP) != 0);
 918 }
 919
 920 static void
 921 vtnet_update_link_status(struct vtnet_softc *sc)
 922 {
 923         device_t dev;
 924         struct ifnet *ifp;
 925         struct ifaltq_subque *ifsq;
 926         int link;
 927
 928         dev = sc->vtnet_dev;
 929         ifp = sc->vtnet_ifp;
 930         ifsq = ifq_get_subq_default(&ifp->if_snd);
 931
 932         link = vtnet_is_link_up(sc);
 933
 934         if (link && ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0)) {
 935                 sc->vtnet_flags |= VTNET_FLAG_LINK;
 936                 if (bootverbose)
 937                         device_printf(dev, "Link is up\n");
 938                 ifp->if_link_state = LINK_STATE_UP;
 939                 if_link_state_change(ifp);
 940                 if (!ifsq_is_empty(ifsq))
 941                         vtnet_start_locked(ifp, ifsq);
 942         } else if (!link && (sc->vtnet_flags & VTNET_FLAG_LINK)) {
 943                 sc->vtnet_flags &= ~VTNET_FLAG_LINK;
 944                 if (bootverbose)
 945                         device_printf(dev, "Link is down\n");
 946
 947                 ifp->if_link_state = LINK_STATE_DOWN;
 948                 if_link_state_change(ifp);
 949         }
 950 }
 951
 952 #if 0
 953 static void
 954 vtnet_watchdog(struct vtnet_softc *sc)
 955 {
 956         struct ifnet *ifp;
 957
 958         ifp = sc->vtnet_ifp;
 959
 960 #ifdef VTNET_TX_INTR_MODERATION
 961         vtnet_txeof(sc);
 962 #endif
 963
 964         if (sc->vtnet_watchdog_timer == 0 || --sc->vtnet_watchdog_timer)
 965                 return;
 966
 967         if_printf(ifp, "watchdog timeout -- resetting\n");
 968 #ifdef VTNET_DEBUG
 969         virtqueue_dump(sc->vtnet_tx_vq);
 970 #endif
 971         ifp->if_oerrors++;
 972         ifp->if_flags &= ~IFF_RUNNING;
 973         vtnet_init_locked(sc);
 974 }
 975 #endif
 976
 977 static void
 978 vtnet_config_change_task(void *arg, int pending)
 979 {
 980         struct vtnet_softc *sc;
 981
 982         sc = arg;
 983
 984         lwkt_serialize_enter(&sc->vtnet_slz);
 985         vtnet_update_link_status(sc);
 986         lwkt_serialize_exit(&sc->vtnet_slz);
 987 }
 988
 989 static int
 990 vtnet_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data,struct ucred *cr)
 991 {
 992         struct vtnet_softc *sc;
 993         struct ifreq *ifr;
 994         int reinit, mask, error;
 995
 996         sc = ifp->if_softc;
 997         ifr = (struct ifreq *) data;
 998         reinit = 0;
 999         error = 0;
1000
1001         switch (cmd) {
1002         case SIOCSIFMTU:
1003                 if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > VTNET_MAX_MTU)
1004                         error = EINVAL;
1005                 else if (ifp->if_mtu != ifr->ifr_mtu) {
1006                         lwkt_serialize_enter(&sc->vtnet_slz);
1007                         error = vtnet_change_mtu(sc, ifr->ifr_mtu);
1008                         lwkt_serialize_exit(&sc->vtnet_slz);
1009                 }
1010                 break;
1011
1012         case SIOCSIFFLAGS:
1013                 lwkt_serialize_enter(&sc->vtnet_slz);
1014                 if ((ifp->if_flags & IFF_UP) == 0) {
1015                         if (ifp->if_flags & IFF_RUNNING)
1016                                 vtnet_stop(sc);
1017                 } else if (ifp->if_flags & IFF_RUNNING) {
1018                         if ((ifp->if_flags ^ sc->vtnet_if_flags) &
1019                             (IFF_PROMISC | IFF_ALLMULTI)) {
1020                                 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX)
1021                                         vtnet_rx_filter(sc);
1022                                 else
1023                                         error = ENOTSUP;
1024                         }
1025                 } else
1026                         vtnet_init_locked(sc);
1027
1028                 if (error == 0)
1029                         sc->vtnet_if_flags = ifp->if_flags;
1030                 lwkt_serialize_exit(&sc->vtnet_slz);
1031                 break;
1032
1033         case SIOCADDMULTI:
1034         case SIOCDELMULTI:
1035                 lwkt_serialize_enter(&sc->vtnet_slz);
1036                 if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) &&
1037                     (ifp->if_flags & IFF_RUNNING))
1038                         vtnet_rx_filter_mac(sc);
1039                 lwkt_serialize_exit(&sc->vtnet_slz);
1040                 break;
1041
1042         case SIOCSIFMEDIA:
1043         case SIOCGIFMEDIA:
1044                 error = ifmedia_ioctl(ifp, ifr, &sc->vtnet_media, cmd);
1045                 break;
1046
1047         case SIOCSIFCAP:
1048                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1049
1050                 lwkt_serialize_enter(&sc->vtnet_slz);
1051
1052                 if (mask & IFCAP_TXCSUM) {
1053                         ifp->if_capenable ^= IFCAP_TXCSUM;
1054                         if (ifp->if_capenable & IFCAP_TXCSUM)
1055                                 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;
1056                         else
1057                                 ifp->if_hwassist &= ~VTNET_CSUM_OFFLOAD;
1058                 }
1059
1060                 if (mask & IFCAP_TSO4) {
1061                         ifp->if_capenable ^= IFCAP_TSO4;
1062                         if (ifp->if_capenable & IFCAP_TSO4)
1063                                 ifp->if_hwassist |= CSUM_TSO;
1064                         else
1065                                 ifp->if_hwassist &= ~CSUM_TSO;
1066                 }
1067
1068                 if (mask & IFCAP_RXCSUM) {
1069                         ifp->if_capenable ^= IFCAP_RXCSUM;
1070                         reinit = 1;
1071                 }
1072
1073                 if (mask & IFCAP_LRO) {
1074                         ifp->if_capenable ^= IFCAP_LRO;
1075                         reinit = 1;
1076                 }
1077
1078                 if (mask & IFCAP_VLAN_HWFILTER) {
1079                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1080                         reinit = 1;
1081                 }
1082
1083                 if (mask & IFCAP_VLAN_HWTSO)
1084                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1085
1086                 if (mask & IFCAP_VLAN_HWTAGGING)
1087                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1088
1089                 if (reinit && (ifp->if_flags & IFF_RUNNING)) {
1090                         ifp->if_flags &= ~IFF_RUNNING;
1091                         vtnet_init_locked(sc);
1092                 }
1093                 //VLAN_CAPABILITIES(ifp);
1094
1095                 lwkt_serialize_exit(&sc->vtnet_slz);
1096                 break;
1097
1098         default:
1099                 error = ether_ioctl(ifp, cmd, data);
1100                 break;
1101         }
1102
1103         return (error);
1104 }
1105
1106 static int
1107 vtnet_change_mtu(struct vtnet_softc *sc, int new_mtu)
1108 {
1109         struct ifnet *ifp;
1110         int new_frame_size, clsize;
1111
1112         ifp = sc->vtnet_ifp;
1113
1114         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1115                 new_frame_size = sizeof(struct vtnet_rx_header) +
1116                     sizeof(struct ether_vlan_header) + new_mtu;
1117
1118                 if (new_frame_size > MJUM9BYTES)
1119                         return (EINVAL);
1120
1121                 if (new_frame_size <= MCLBYTES)
1122                         clsize = MCLBYTES;
1123                 else
1124                         clsize = MJUM9BYTES;
1125         } else {
1126                 new_frame_size = sizeof(struct virtio_net_hdr_mrg_rxbuf) +
1127                     sizeof(struct ether_vlan_header) + new_mtu;
1128
1129                 if (new_frame_size <= MCLBYTES)
1130                         clsize = MCLBYTES;
1131                 else
1132                         clsize = MJUMPAGESIZE;
1133         }
1134
1135         sc->vtnet_rx_mbuf_size = clsize;
1136         sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc);
1137         KASSERT(sc->vtnet_rx_mbuf_count < VTNET_MAX_RX_SEGS,
1138             ("too many rx mbufs: %d", sc->vtnet_rx_mbuf_count));
1139
1140         ifp->if_mtu = new_mtu;
1141
1142         if (ifp->if_flags & IFF_RUNNING) {
1143                 ifp->if_flags &= ~IFF_RUNNING;
1144                 vtnet_init_locked(sc);
1145         }
1146
1147         return (0);
1148 }
1149
1150 static int
1151 vtnet_init_rx_vq(struct vtnet_softc *sc)
1152 {
1153         struct virtqueue *vq;
1154         int nbufs, error;
1155
1156         vq = sc->vtnet_rx_vq;
1157         nbufs = 0;
1158         error = ENOSPC;
1159
1160         while (!virtqueue_full(vq)) {
1161                 if ((error = vtnet_newbuf(sc)) != 0)
1162                         break;
1163                 nbufs++;
1164         }
1165
1166         if (nbufs > 0) {
1167                 virtqueue_notify(vq, &sc->vtnet_slz);
1168
1169                 /*
1170                  * EMSGSIZE signifies the virtqueue did not have enough
1171                  * entries available to hold the last mbuf. This is not
1172                  * an error. We should not get ENOSPC since we check if
1173                  * the virtqueue is full before attempting to add a
1174                  * buffer.
1175                  */
1176                 if (error == EMSGSIZE)
1177                         error = 0;
1178         }
1179
1180         return (error);
1181 }
1182
1183 static void
1184 vtnet_free_rx_mbufs(struct vtnet_softc *sc)
1185 {
1186         struct virtqueue *vq;
1187         struct mbuf *m;
1188         int last;
1189
1190         vq = sc->vtnet_rx_vq;
1191         last = 0;
1192
1193         while ((m = virtqueue_drain(vq, &last)) != NULL)
1194                 m_freem(m);
1195
1196         KASSERT(virtqueue_empty(vq), ("mbufs remaining in Rx Vq"));
1197 }
1198
1199 static void
1200 vtnet_free_tx_mbufs(struct vtnet_softc *sc)
1201 {
1202         struct virtqueue *vq;
1203         struct vtnet_tx_header *txhdr;
1204         int last;
1205
1206         vq = sc->vtnet_tx_vq;
1207         last = 0;
1208
1209         while ((txhdr = virtqueue_drain(vq, &last)) != NULL) {
1210                 m_freem(txhdr->vth_mbuf);
1211         }
1212
1213         KASSERT(virtqueue_empty(vq), ("mbufs remaining in Tx Vq"));
1214 }
1215
1216 static void
1217 vtnet_free_ctrl_vq(struct vtnet_softc *sc)
1218 {
1219         /*
1220          * The control virtqueue is only polled, therefore
1221          * it should already be empty.
1222          */
1223         KASSERT(virtqueue_empty(sc->vtnet_ctrl_vq),
1224                 ("Ctrl Vq not empty"));
1225 }
1226
1227 static struct mbuf *
1228 vtnet_alloc_rxbuf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp)
1229 {
1230         struct mbuf *m_head, *m_tail, *m;
1231         int i, clsize;
1232
1233         clsize = sc->vtnet_rx_mbuf_size;
1234
1235         /*use getcl instead of getjcl. see  if_mxge.c comment line 2398*/
1236         //m_head = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, clsize);
1237         m_head = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR );
1238         if (m_head == NULL)
1239                 goto fail;
1240
1241         m_head->m_len = clsize;
1242         m_tail = m_head;
1243
1244         if (nbufs > 1) {
1245                 KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
1246                         ("chained Rx mbuf requested without LRO_NOMRG"));
1247
1248                 for (i = 0; i < nbufs - 1; i++) {
1249                         //m = m_getjcl(M_DONTWAIT, MT_DATA, 0, clsize);
1250                         m = m_getcl(M_NOWAIT, MT_DATA, 0);
1251                         if (m == NULL)
1252                                 goto fail;
1253
1254                         m->m_len = clsize;
1255                         m_tail->m_next = m;
1256                         m_tail = m;
1257                 }
1258         }
1259
1260         if (m_tailp != NULL)
1261                 *m_tailp = m_tail;
1262
1263         return (m_head);
1264
1265 fail:
1266         sc->vtnet_stats.mbuf_alloc_failed++;
1267         m_freem(m_head);
1268
1269         return (NULL);
1270 }
1271
1272 static int
1273 vtnet_replace_rxbuf(struct vtnet_softc *sc, struct mbuf *m0, int len0)
1274 {
1275         struct mbuf *m, *m_prev;
1276         struct mbuf *m_new, *m_tail;
1277         int len, clsize, nreplace, error;
1278
1279         m = m0;
1280         m_prev = NULL;
1281         len = len0;
1282
1283         m_tail = NULL;
1284         clsize = sc->vtnet_rx_mbuf_size;
1285         nreplace = 0;
1286
1287         if (m->m_next != NULL)
1288                 KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
1289                     ("chained Rx mbuf without LRO_NOMRG"));
1290
1291         /*
1292          * Since LRO_NOMRG mbuf chains are so large, we want to avoid
1293          * allocating an entire chain for each received frame. When
1294          * the received frame's length is less than that of the chain,
1295          * the unused mbufs are reassigned to the new chain.
1296          */
1297         while (len > 0) {
1298                 /*
1299                  * Something is seriously wrong if we received
1300                  * a frame larger than the mbuf chain. Drop it.
1301                  */
1302                 if (m == NULL) {
1303                         sc->vtnet_stats.rx_frame_too_large++;
1304                         return (EMSGSIZE);
1305                 }
1306
1307                 KASSERT(m->m_len == clsize,
1308                     ("mbuf length not expected cluster size: %d",
1309                     m->m_len));
1310
1311                 m->m_len = MIN(m->m_len, len);
1312                 len -= m->m_len;
1313
1314                 m_prev = m;
1315                 m = m->m_next;
1316                 nreplace++;
1317         }
1318
1319         KASSERT(m_prev != NULL, ("m_prev == NULL"));
1320         KASSERT(nreplace <= sc->vtnet_rx_mbuf_count,
1321                 ("too many replacement mbufs: %d/%d", nreplace,
1322                 sc->vtnet_rx_mbuf_count));
1323
1324         m_new = vtnet_alloc_rxbuf(sc, nreplace, &m_tail);
1325         if (m_new == NULL) {
1326                 m_prev->m_len = clsize;
1327                 return (ENOBUFS);
1328         }
1329
1330         /*
1331          * Move unused mbufs, if any, from the original chain
1332          * onto the end of the new chain.
1333          */
1334         if (m_prev->m_next != NULL) {
1335                 m_tail->m_next = m_prev->m_next;
1336                 m_prev->m_next = NULL;
1337         }
1338
1339         error = vtnet_enqueue_rxbuf(sc, m_new);
1340         if (error) {
1341                 /*
1342                  * BAD! We could not enqueue the replacement mbuf chain. We
1343                  * must restore the m0 chain to the original state if it was
1344                  * modified so we can subsequently discard it.
1345                  *
1346                  * NOTE: The replacement is suppose to be an identical copy
1347                  * to the one just dequeued so this is an unexpected error.
1348                  */
1349                 sc->vtnet_stats.rx_enq_replacement_failed++;
1350
1351                 if (m_tail->m_next != NULL) {
1352                         m_prev->m_next = m_tail->m_next;
1353                         m_tail->m_next = NULL;
1354                 }
1355
1356                 m_prev->m_len = clsize;
1357                 m_freem(m_new);
1358         }
1359
1360         return (error);
1361 }
1362
1363 static int
1364 vtnet_newbuf(struct vtnet_softc *sc)
1365 {
1366         struct mbuf *m;
1367         int error;
1368
1369         m = vtnet_alloc_rxbuf(sc, sc->vtnet_rx_mbuf_count, NULL);
1370         if (m == NULL)
1371                 return (ENOBUFS);
1372
1373         error = vtnet_enqueue_rxbuf(sc, m);
1374         if (error)
1375                 m_freem(m);
1376
1377         return (error);
1378 }
1379
1380 static void
1381 vtnet_discard_merged_rxbuf(struct vtnet_softc *sc, int nbufs)
1382 {
1383         struct virtqueue *vq;
1384         struct mbuf *m;
1385
1386         vq = sc->vtnet_rx_vq;
1387
1388         while (--nbufs > 0) {
1389                 if ((m = virtqueue_dequeue(vq, NULL)) == NULL)
1390                         break;
1391                 vtnet_discard_rxbuf(sc, m);
1392         }
1393 }
1394
1395 static void
1396 vtnet_discard_rxbuf(struct vtnet_softc *sc, struct mbuf *m)
1397 {
1398         int error;
1399
1400         /*
1401          * Requeue the discarded mbuf. This should always be
1402          * successful since it was just dequeued.
1403          */
1404         error = vtnet_enqueue_rxbuf(sc, m);
1405         KASSERT(error == 0, ("cannot requeue discarded mbuf"));
1406 }
1407
1408 static int
1409 vtnet_enqueue_rxbuf(struct vtnet_softc *sc, struct mbuf *m)
1410 {
1411         struct sglist sg;
1412         struct sglist_seg segs[VTNET_MAX_RX_SEGS];
1413         struct vtnet_rx_header *rxhdr;
1414         struct virtio_net_hdr *hdr;
1415         uint8_t *mdata;
1416         int offset, error;
1417
1418         ASSERT_SERIALIZED(&sc->vtnet_slz);
1419         if ((sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) == 0)
1420                 KASSERT(m->m_next == NULL, ("chained Rx mbuf"));
1421
1422         sglist_init(&sg, VTNET_MAX_RX_SEGS, segs);
1423
1424         mdata = mtod(m, uint8_t *);
1425         offset = 0;
1426
1427         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1428                 rxhdr = (struct vtnet_rx_header *) mdata;
1429                 hdr = &rxhdr->vrh_hdr;
1430                 offset += sizeof(struct vtnet_rx_header);
1431
1432                 error = sglist_append(&sg, hdr, sc->vtnet_hdr_size);
1433                 KASSERT(error == 0, ("cannot add header to sglist"));
1434         }
1435
1436         error = sglist_append(&sg, mdata + offset, m->m_len - offset);
1437         if (error)
1438                 return (error);
1439
1440         if (m->m_next != NULL) {
1441                 error = sglist_append_mbuf(&sg, m->m_next);
1442                 if (error)
1443                         return (error);
1444         }
1445
1446         return (virtqueue_enqueue(sc->vtnet_rx_vq, m, &sg, 0, sg.sg_nseg));
1447 }
1448
1449 static void
1450 vtnet_vlan_tag_remove(struct mbuf *m)
1451 {
1452         struct ether_vlan_header *evl;
1453
1454         evl = mtod(m, struct ether_vlan_header *);
1455
1456         m->m_pkthdr.ether_vlantag = ntohs(evl->evl_tag);
1457         m->m_flags |= M_VLANTAG;
1458
1459         /* Strip the 802.1Q header. */
1460         bcopy((char *) evl, (char *) evl + ETHER_VLAN_ENCAP_LEN,
1461             ETHER_HDR_LEN - ETHER_TYPE_LEN);
1462         m_adj(m, ETHER_VLAN_ENCAP_LEN);
1463 }
1464
1465 /*
1466  * Alternative method of doing receive checksum offloading. Rather
1467  * than parsing the received frame down to the IP header, use the
1468  * csum_offset to determine which CSUM_* flags are appropriate. We
1469  * can get by with doing this only because the checksum offsets are
1470  * unique for the things we care about.
1471  */
1472 static int
1473 vtnet_rx_csum(struct vtnet_softc *sc, struct mbuf *m,
1474     struct virtio_net_hdr *hdr)
1475 {
1476         struct ether_header *eh;
1477         struct ether_vlan_header *evh;
1478         struct udphdr *udp;
1479         int csum_len;
1480         uint16_t eth_type;
1481
1482         csum_len = hdr->csum_start + hdr->csum_offset;
1483
1484         if (csum_len < sizeof(struct ether_header) + sizeof(struct ip))
1485                 return (1);
1486         if (m->m_len < csum_len)
1487                 return (1);
1488
1489         eh = mtod(m, struct ether_header *);
1490         eth_type = ntohs(eh->ether_type);
1491         if (eth_type == ETHERTYPE_VLAN) {
1492                 evh = mtod(m, struct ether_vlan_header *);
1493                 eth_type = ntohs(evh->evl_proto);
1494         }
1495
1496         if (eth_type != ETHERTYPE_IP && eth_type != ETHERTYPE_IPV6) {
1497                 sc->vtnet_stats.rx_csum_bad_ethtype++;
1498                 return (1);
1499         }
1500
1501         /* Use the offset to determine the appropriate CSUM_* flags. */
1502         switch (hdr->csum_offset) {
1503         case offsetof(struct udphdr, uh_sum):
1504                 if (m->m_len < hdr->csum_start + sizeof(struct udphdr))
1505                         return (1);
1506                 udp = (struct udphdr *)(mtod(m, uint8_t *) + hdr->csum_start);
1507                 if (udp->uh_sum == 0)
1508                         return (0);
1509
1510                 /* FALLTHROUGH */
1511
1512         case offsetof(struct tcphdr, th_sum):
1513                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1514                 m->m_pkthdr.csum_data = 0xFFFF;
1515                 break;
1516
1517         default:
1518                 sc->vtnet_stats.rx_csum_bad_offset++;
1519                 return (1);
1520         }
1521
1522         sc->vtnet_stats.rx_csum_offloaded++;
1523
1524         return (0);
1525 }
1526
1527 static int
1528 vtnet_rxeof_merged(struct vtnet_softc *sc, struct mbuf *m_head, int nbufs)
1529 {
1530         struct ifnet *ifp;
1531         struct virtqueue *vq;
1532         struct mbuf *m, *m_tail;
1533         int len;
1534
1535         ifp = sc->vtnet_ifp;
1536         vq = sc->vtnet_rx_vq;
1537         m_tail = m_head;
1538
1539         while (--nbufs > 0) {
1540                 m = virtqueue_dequeue(vq, &len);
1541                 if (m == NULL) {
1542                         ifp->if_ierrors++;
1543                         goto fail;
1544                 }
1545
1546                 if (vtnet_newbuf(sc) != 0) {
1547                         ifp->if_iqdrops++;
1548                         vtnet_discard_rxbuf(sc, m);
1549                         if (nbufs > 1)
1550                                 vtnet_discard_merged_rxbuf(sc, nbufs);
1551                         goto fail;
1552                 }
1553
1554                 if (m->m_len < len)
1555                         len = m->m_len;
1556
1557                 m->m_len = len;
1558                 m->m_flags &= ~M_PKTHDR;
1559
1560                 m_head->m_pkthdr.len += len;
1561                 m_tail->m_next = m;
1562                 m_tail = m;
1563         }
1564
1565         return (0);
1566
1567 fail:
1568         sc->vtnet_stats.rx_mergeable_failed++;
1569         m_freem(m_head);
1570
1571         return (1);
1572 }
1573
1574 static int
1575 vtnet_rxeof(struct vtnet_softc *sc, int count, int *rx_npktsp)
1576 {
1577         struct virtio_net_hdr lhdr;
1578         struct ifnet *ifp;
1579         struct virtqueue *vq;
1580         struct mbuf *m;
1581         struct ether_header *eh;
1582         struct virtio_net_hdr *hdr;
1583         struct virtio_net_hdr_mrg_rxbuf *mhdr;
1584         int len, deq, nbufs, adjsz, rx_npkts;
1585
1586         ifp = sc->vtnet_ifp;
1587         vq = sc->vtnet_rx_vq;
1588         hdr = &lhdr;
1589         deq = 0;
1590         rx_npkts = 0;
1591
1592         ASSERT_SERIALIZED(&sc->vtnet_slz);
1593
1594         while (--count >= 0) {
1595                 m = virtqueue_dequeue(vq, &len);
1596                 if (m == NULL)
1597                         break;
1598                 deq++;
1599
1600                 if (len < sc->vtnet_hdr_size + ETHER_HDR_LEN) {
1601                         ifp->if_ierrors++;
1602                         vtnet_discard_rxbuf(sc, m);
1603                         continue;
1604                 }
1605
1606                 if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1607                         nbufs = 1;
1608                         adjsz = sizeof(struct vtnet_rx_header);
1609                         /*
1610                          * Account for our pad between the header and
1611                          * the actual start of the frame.
1612                          */
1613                         len += VTNET_RX_HEADER_PAD;
1614                 } else {
1615                         mhdr = mtod(m, struct virtio_net_hdr_mrg_rxbuf *);
1616                         nbufs = mhdr->num_buffers;
1617                         adjsz = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1618                 }
1619
1620                 if (vtnet_replace_rxbuf(sc, m, len) != 0) {
1621                         ifp->if_iqdrops++;
1622                         vtnet_discard_rxbuf(sc, m);
1623                         if (nbufs > 1)
1624                                 vtnet_discard_merged_rxbuf(sc, nbufs);
1625                         continue;
1626                 }
1627
1628                 m->m_pkthdr.len = len;
1629                 m->m_pkthdr.rcvif = ifp;
1630                 m->m_pkthdr.csum_flags = 0;
1631
1632                 if (nbufs > 1) {
1633                         if (vtnet_rxeof_merged(sc, m, nbufs) != 0)
1634                                 continue;
1635                 }
1636
1637                 ifp->if_ipackets++;
1638
1639                 /*
1640                  * Save copy of header before we strip it. For both mergeable
1641                  * and non-mergeable, the VirtIO header is placed first in the
1642                  * mbuf's data. We no longer need num_buffers, so always use a
1643                  * virtio_net_hdr.
1644                  */
1645                 memcpy(hdr, mtod(m, void *), sizeof(struct virtio_net_hdr));
1646                 m_adj(m, adjsz);
1647
1648                 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1649                         eh = mtod(m, struct ether_header *);
1650                         if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
1651                                 vtnet_vlan_tag_remove(m);
1652
1653                                 /*
1654                                  * With the 802.1Q header removed, update the
1655                                  * checksum starting location accordingly.
1656                                  */
1657                                 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
1658                                         hdr->csum_start -=
1659                                             ETHER_VLAN_ENCAP_LEN;
1660                         }
1661                 }
1662
1663                 if (ifp->if_capenable & IFCAP_RXCSUM &&
1664                     hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
1665                         if (vtnet_rx_csum(sc, m, hdr) != 0)
1666                                 sc->vtnet_stats.rx_csum_failed++;
1667                 }
1668
1669                 lwkt_serialize_exit(&sc->vtnet_slz);
1670                 rx_npkts++;
1671                 ifp->if_input(ifp, m, NULL, -1);
1672                 lwkt_serialize_enter(&sc->vtnet_slz);
1673
1674                 /*
1675                  * The interface may have been stopped while we were
1676                  * passing the packet up the network stack.
1677                  */
1678                 if ((ifp->if_flags & IFF_RUNNING) == 0)
1679                         break;
1680         }
1681
1682         virtqueue_notify(vq, &sc->vtnet_slz);
1683
1684         if (rx_npktsp != NULL)
1685                 *rx_npktsp = rx_npkts;
1686
1687         return (count > 0 ? 0 : EAGAIN);
1688 }
1689
1690 static void
1691 vtnet_rx_intr_task(void *arg)
1692 {
1693         struct vtnet_softc *sc;
1694         struct ifnet *ifp;
1695         int more;
1696
1697         sc = arg;
1698         ifp = sc->vtnet_ifp;
1699
1700 next:
1701 //      lwkt_serialize_enter(&sc->vtnet_slz);
1702
1703         if ((ifp->if_flags & IFF_RUNNING) == 0) {
1704                 vtnet_enable_rx_intr(sc);
1705 //              lwkt_serialize_exit(&sc->vtnet_slz);
1706                 return;
1707         }
1708
1709         more = vtnet_rxeof(sc, sc->vtnet_rx_process_limit, NULL);
1710         if (!more && vtnet_enable_rx_intr(sc) != 0) {
1711                 vtnet_disable_rx_intr(sc);
1712                 more = 1;
1713         }
1714
1715 //      lwkt_serialize_exit(&sc->vtnet_slz);
1716
1717         if (more) {
1718                 sc->vtnet_stats.rx_task_rescheduled++;
1719                 goto next;
1720         }
1721 }
1722
1723 static int
1724 vtnet_rx_vq_intr(void *xsc)
1725 {
1726         struct vtnet_softc *sc;
1727
1728         sc = xsc;
1729
1730         vtnet_disable_rx_intr(sc);
1731         vtnet_rx_intr_task(sc);
1732
1733         return (1);
1734 }
1735
1736 static void
1737 vtnet_txeof(struct vtnet_softc *sc)
1738 {
1739         struct virtqueue *vq;
1740         struct ifnet *ifp;
1741         struct vtnet_tx_header *txhdr;
1742         int deq;
1743
1744         vq = sc->vtnet_tx_vq;
1745         ifp = sc->vtnet_ifp;
1746         deq = 0;
1747
1748         ASSERT_SERIALIZED(&sc->vtnet_slz);
1749
1750         while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) {
1751                 deq++;
1752                 ifp->if_opackets++;
1753                 m_freem(txhdr->vth_mbuf);
1754         }
1755
1756         if (deq > 0) {
1757                 ifq_clr_oactive(&ifp->if_snd);
1758                 if (virtqueue_empty(vq))
1759                         sc->vtnet_watchdog_timer = 0;
1760         }
1761 }
1762
1763 static struct mbuf *
1764 vtnet_tx_offload(struct vtnet_softc *sc, struct mbuf *m,
1765     struct virtio_net_hdr *hdr)
1766 {
1767         struct ifnet *ifp;
1768         struct ether_header *eh;
1769         struct ether_vlan_header *evh;
1770         struct ip *ip;
1771         struct ip6_hdr *ip6;
1772         struct tcphdr *tcp;
1773         int ip_offset;
1774         uint16_t eth_type, csum_start;
1775         uint8_t ip_proto, gso_type;
1776
1777         ifp = sc->vtnet_ifp;
1778         M_ASSERTPKTHDR(m);
1779
1780         ip_offset = sizeof(struct ether_header);
1781         if (m->m_len < ip_offset) {
1782                 if ((m = m_pullup(m, ip_offset)) == NULL)
1783                         return (NULL);
1784         }
1785
1786         eh = mtod(m, struct ether_header *);
1787         eth_type = ntohs(eh->ether_type);
1788         if (eth_type == ETHERTYPE_VLAN) {
1789                 ip_offset = sizeof(struct ether_vlan_header);
1790                 if (m->m_len < ip_offset) {
1791                         if ((m = m_pullup(m, ip_offset)) == NULL)
1792                                 return (NULL);
1793                 }
1794                 evh = mtod(m, struct ether_vlan_header *);
1795                 eth_type = ntohs(evh->evl_proto);
1796         }
1797
1798         switch (eth_type) {
1799         case ETHERTYPE_IP:
1800                 if (m->m_len < ip_offset + sizeof(struct ip)) {
1801                         m = m_pullup(m, ip_offset + sizeof(struct ip));
1802                         if (m == NULL)
1803                                 return (NULL);
1804                 }
1805
1806                 ip = (struct ip *)(mtod(m, uint8_t *) + ip_offset);
1807                 ip_proto = ip->ip_p;
1808                 csum_start = ip_offset + (ip->ip_hl << 2);
1809                 gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
1810                 break;
1811
1812         case ETHERTYPE_IPV6:
1813                 if (m->m_len < ip_offset + sizeof(struct ip6_hdr)) {
1814                         m = m_pullup(m, ip_offset + sizeof(struct ip6_hdr));
1815                         if (m == NULL)
1816                                 return (NULL);
1817                 }
1818
1819                 ip6 = (struct ip6_hdr *)(mtod(m, uint8_t *) + ip_offset);
1820                 /*
1821                  * XXX Assume no extension headers are present. Presently,
1822                  * this will always be true in the case of TSO, and FreeBSD
1823                  * does not perform checksum offloading of IPv6 yet.
1824                  */
1825                 ip_proto = ip6->ip6_nxt;
1826                 csum_start = ip_offset + sizeof(struct ip6_hdr);
1827                 gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
1828                 break;
1829
1830         default:
1831                 return (m);
1832         }
1833
1834         if (m->m_pkthdr.csum_flags & VTNET_CSUM_OFFLOAD) {
1835                 hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM;
1836                 hdr->csum_start = csum_start;
1837                 hdr->csum_offset = m->m_pkthdr.csum_data;
1838
1839                 sc->vtnet_stats.tx_csum_offloaded++;
1840         }
1841
1842         if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1843                 if (ip_proto != IPPROTO_TCP)
1844                         return (m);
1845
1846                 if (m->m_len < csum_start + sizeof(struct tcphdr)) {
1847                         m = m_pullup(m, csum_start + sizeof(struct tcphdr));
1848                         if (m == NULL)
1849                                 return (NULL);
1850                 }
1851
1852                 tcp = (struct tcphdr *)(mtod(m, uint8_t *) + csum_start);
1853                 hdr->gso_type = gso_type;
1854                 hdr->hdr_len = csum_start + (tcp->th_off << 2);
1855                 hdr->gso_size = m->m_pkthdr.tso_segsz;
1856
1857                 if (tcp->th_flags & TH_CWR) {
1858                         /*
1859                          * Drop if we did not negotiate VIRTIO_NET_F_HOST_ECN.
1860                          * ECN support is only configurable globally with the
1861                          * net.inet.tcp.ecn.enable sysctl knob.
1862                          */
1863                         if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) {
1864                                 if_printf(ifp, "TSO with ECN not supported "
1865                                     "by host\n");
1866                                 m_freem(m);
1867                                 return (NULL);
1868                         }
1869
1870                         hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN;
1871                 }
1872
1873                 sc->vtnet_stats.tx_tso_offloaded++;
1874         }
1875
1876         return (m);
1877 }
1878
1879 static int
1880 vtnet_enqueue_txbuf(struct vtnet_softc *sc, struct mbuf **m_head,
1881     struct vtnet_tx_header *txhdr)
1882 {
1883         struct sglist sg;
1884         struct sglist_seg segs[VTNET_MAX_TX_SEGS];
1885         struct virtqueue *vq;
1886         struct mbuf *m;
1887         int collapsed, error;
1888
1889         vq = sc->vtnet_tx_vq;
1890         m = *m_head;
1891         collapsed = 0;
1892
1893         sglist_init(&sg, VTNET_MAX_TX_SEGS, segs);
1894         error = sglist_append(&sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size);
1895         KASSERT(error == 0 && sg.sg_nseg == 1,
1896             ("cannot add header to sglist"));
1897
1898 again:
1899         error = sglist_append_mbuf(&sg, m);
1900         if (error) {
1901                 if (collapsed)
1902                         goto fail;
1903
1904                 //m = m_collapse(m, M_NOWAIT, VTNET_MAX_TX_SEGS - 1);
1905                 m = m_defrag(m, M_NOWAIT);
1906                 if (m == NULL)
1907                         goto fail;
1908
1909                 *m_head = m;
1910                 collapsed = 1;
1911                 goto again;
1912         }
1913
1914         txhdr->vth_mbuf = m;
1915
1916         return (virtqueue_enqueue(vq, txhdr, &sg, sg.sg_nseg, 0));
1917
1918 fail:
1919         m_freem(*m_head);
1920         *m_head = NULL;
1921
1922         return (ENOBUFS);
1923 }
1924
1925 static struct mbuf *
1926 vtnet_vlan_tag_insert(struct mbuf *m)
1927 {
1928         struct mbuf *n;
1929         struct ether_vlan_header *evl;
1930
1931         if (M_WRITABLE(m) == 0) {
1932                 n = m_dup(m, M_NOWAIT);
1933                 m_freem(m);
1934                 if ((m = n) == NULL)
1935                         return (NULL);
1936         }
1937
1938         M_PREPEND(m, ETHER_VLAN_ENCAP_LEN, M_NOWAIT);
1939         if (m == NULL)
1940                 return (NULL);
1941         if (m->m_len < sizeof(struct ether_vlan_header)) {
1942                 m = m_pullup(m, sizeof(struct ether_vlan_header));
1943                 if (m == NULL)
1944                         return (NULL);
1945         }
1946
1947         /* Insert 802.1Q header into the existing Ethernet header. */
1948         evl = mtod(m, struct ether_vlan_header *);
1949         bcopy((char *) evl + ETHER_VLAN_ENCAP_LEN,
1950               (char *) evl, ETHER_HDR_LEN - ETHER_TYPE_LEN);
1951         evl->evl_encap_proto = htons(ETHERTYPE_VLAN);
1952         evl->evl_tag = htons(m->m_pkthdr.ether_vlantag);
1953         m->m_flags &= ~M_VLANTAG;
1954
1955         return (m);
1956 }
1957
1958 static int
1959 vtnet_encap(struct vtnet_softc *sc, struct mbuf **m_head)
1960 {
1961         struct vtnet_tx_header *txhdr;
1962         struct virtio_net_hdr *hdr;
1963         struct mbuf *m;
1964         int error;
1965
1966         txhdr = &sc->vtnet_txhdrarea[sc->vtnet_txhdridx];
1967         memset(txhdr, 0, sizeof(struct vtnet_tx_header));
1968
1969         /*
1970          * Always use the non-mergeable header to simplify things. When
1971          * the mergeable feature is negotiated, the num_buffers field
1972          * must be set to zero. We use vtnet_hdr_size later to enqueue
1973          * the correct header size to the host.
1974          */
1975         hdr = &txhdr->vth_uhdr.hdr;
1976         m = *m_head;
1977
1978         error = ENOBUFS;
1979
1980         if (m->m_flags & M_VLANTAG) {
1981                 //m = ether_vlanencap(m, m->m_pkthdr.ether_vtag);
1982                 m = vtnet_vlan_tag_insert(m);
1983                 if ((*m_head = m) == NULL)
1984                         goto fail;
1985                 m->m_flags &= ~M_VLANTAG;
1986         }
1987
1988         if (m->m_pkthdr.csum_flags != 0) {
1989                 m = vtnet_tx_offload(sc, m, hdr);
1990                 if ((*m_head = m) == NULL)
1991                         goto fail;
1992         }
1993
1994         error = vtnet_enqueue_txbuf(sc, m_head, txhdr);
1995         if (error == 0)
1996                 sc->vtnet_txhdridx =
1997                     (sc->vtnet_txhdridx + 1) % ((sc->vtnet_tx_size / 2) + 1);
1998 fail:
1999         return (error);
2000 }
2001
2002 static void
2003 vtnet_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
2004 {
2005         struct vtnet_softc *sc;
2006
2007         sc = ifp->if_softc;
2008
2009         ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq);
2010         lwkt_serialize_enter(&sc->vtnet_slz);
2011         vtnet_start_locked(ifp, ifsq);
2012         lwkt_serialize_exit(&sc->vtnet_slz);
2013 }
2014
2015 static void
2016 vtnet_start_locked(struct ifnet *ifp, struct ifaltq_subque *ifsq)
2017 {
2018         struct vtnet_softc *sc;
2019         struct virtqueue *vq;
2020         struct mbuf *m0;
2021         int enq;
2022
2023         sc = ifp->if_softc;
2024         vq = sc->vtnet_tx_vq;
2025         enq = 0;
2026
2027         ASSERT_SERIALIZED(&sc->vtnet_slz);
2028
2029         if ((ifp->if_flags & (IFF_RUNNING)) !=
2030             IFF_RUNNING || ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0))
2031                 return;
2032
2033 #ifdef VTNET_TX_INTR_MODERATION
2034         if (virtqueue_nused(vq) >= sc->vtnet_tx_size / 2)
2035                 vtnet_txeof(sc);
2036 #endif
2037
2038         while (!ifsq_is_empty(ifsq)) {
2039                 if (virtqueue_full(vq)) {
2040                         ifq_set_oactive(&ifp->if_snd);
2041                         break;
2042                 }
2043
2044                 m0 = ifq_dequeue(&ifp->if_snd);
2045                 if (m0 == NULL)
2046                         break;
2047
2048                 if (vtnet_encap(sc, &m0) != 0) {
2049                         if (m0 == NULL)
2050                                 break;
2051                         ifq_prepend(&ifp->if_snd, m0);
2052                         ifq_set_oactive(&ifp->if_snd);
2053                         break;
2054                 }
2055
2056                 enq++;
2057                 ETHER_BPF_MTAP(ifp, m0);
2058         }
2059
2060         if (enq > 0) {
2061                 virtqueue_notify(vq, &sc->vtnet_slz);
2062                 sc->vtnet_watchdog_timer = VTNET_WATCHDOG_TIMEOUT;
2063         }
2064 }
2065
2066 static void
2067 vtnet_tick(void *xsc)
2068 {
2069         struct vtnet_softc *sc;
2070
2071         sc = xsc;
2072
2073 #if 0
2074         ASSERT_SERIALIZED(&sc->vtnet_slz);
2075 #ifdef VTNET_DEBUG
2076         virtqueue_dump(sc->vtnet_rx_vq);
2077         virtqueue_dump(sc->vtnet_tx_vq);
2078 #endif
2079
2080         vtnet_watchdog(sc);
2081         callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc);
2082 #endif
2083 }
2084
2085 static void
2086 vtnet_tx_intr_task(void *arg)
2087 {
2088         struct vtnet_softc *sc;
2089         struct ifnet *ifp;
2090         struct ifaltq_subque *ifsq;
2091
2092         sc = arg;
2093         ifp = sc->vtnet_ifp;
2094         ifsq = ifq_get_subq_default(&ifp->if_snd);
2095
2096 next:
2097 //      lwkt_serialize_enter(&sc->vtnet_slz);
2098
2099         if ((ifp->if_flags & IFF_RUNNING) == 0) {
2100                 vtnet_enable_tx_intr(sc);
2101 //              lwkt_serialize_exit(&sc->vtnet_slz);
2102                 return;
2103         }
2104
2105         vtnet_txeof(sc);
2106
2107         if (!ifsq_is_empty(ifsq))
2108                 vtnet_start_locked(ifp, ifsq);
2109
2110         if (vtnet_enable_tx_intr(sc) != 0) {
2111                 vtnet_disable_tx_intr(sc);
2112                 sc->vtnet_stats.tx_task_rescheduled++;
2113 //              lwkt_serialize_exit(&sc->vtnet_slz);
2114                 goto next;
2115         }
2116
2117 //      lwkt_serialize_exit(&sc->vtnet_slz);
2118 }
2119
2120 static int
2121 vtnet_tx_vq_intr(void *xsc)
2122 {
2123         struct vtnet_softc *sc;
2124
2125         sc = xsc;
2126
2127         vtnet_disable_tx_intr(sc);
2128         vtnet_tx_intr_task(sc);
2129
2130         return (1);
2131 }
2132
2133 static void
2134 vtnet_stop(struct vtnet_softc *sc)
2135 {
2136         device_t dev;
2137         struct ifnet *ifp;
2138
2139         dev = sc->vtnet_dev;
2140         ifp = sc->vtnet_ifp;
2141
2142         ASSERT_SERIALIZED(&sc->vtnet_slz);
2143
2144         sc->vtnet_watchdog_timer = 0;
2145         callout_stop(&sc->vtnet_tick_ch);
2146         ifq_clr_oactive(&ifp->if_snd);
2147         ifp->if_flags &= ~(IFF_RUNNING);
2148
2149         vtnet_disable_rx_intr(sc);
2150         vtnet_disable_tx_intr(sc);
2151
2152         /*
2153          * Stop the host VirtIO adapter. Note this will reset the host
2154          * adapter's state back to the pre-initialized state, so in
2155          * order to make the device usable again, we must drive it
2156          * through virtio_reinit() and virtio_reinit_complete().
2157          */
2158         virtio_stop(dev);
2159
2160         sc->vtnet_flags &= ~VTNET_FLAG_LINK;
2161
2162         vtnet_free_rx_mbufs(sc);
2163         vtnet_free_tx_mbufs(sc);
2164 }
2165
2166 static int
2167 vtnet_reinit(struct vtnet_softc *sc)
2168 {
2169         struct ifnet *ifp;
2170         uint64_t features;
2171
2172         ifp = sc->vtnet_ifp;
2173         features = sc->vtnet_features;
2174
2175         /*
2176          * Re-negotiate with the host, removing any disabled receive
2177          * features. Transmit features are disabled only on our side
2178          * via if_capenable and if_hwassist.
2179          */
2180
2181         if (ifp->if_capabilities & IFCAP_RXCSUM) {
2182                 if ((ifp->if_capenable & IFCAP_RXCSUM) == 0)
2183                         features &= ~VIRTIO_NET_F_GUEST_CSUM;
2184         }
2185
2186         if (ifp->if_capabilities & IFCAP_LRO) {
2187                 if ((ifp->if_capenable & IFCAP_LRO) == 0)
2188                         features &= ~VTNET_LRO_FEATURES;
2189         }
2190
2191         if (ifp->if_capabilities & IFCAP_VLAN_HWFILTER) {
2192                 if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
2193                         features &= ~VIRTIO_NET_F_CTRL_VLAN;
2194         }
2195
2196         return (virtio_reinit(sc->vtnet_dev, features));
2197 }
2198
2199 static void
2200 vtnet_init_locked(struct vtnet_softc *sc)
2201 {
2202         device_t dev;
2203         struct ifnet *ifp;
2204         int error;
2205
2206         dev = sc->vtnet_dev;
2207         ifp = sc->vtnet_ifp;
2208
2209         ASSERT_SERIALIZED(&sc->vtnet_slz);
2210
2211         if (ifp->if_flags & IFF_RUNNING)
2212                 return;
2213
2214         /* Stop host's adapter, cancel any pending I/O. */
2215         vtnet_stop(sc);
2216
2217         /* Reinitialize the host device. */
2218         error = vtnet_reinit(sc);
2219         if (error) {
2220                 device_printf(dev,
2221                     "reinitialization failed, stopping device...\n");
2222                 vtnet_stop(sc);
2223                 return;
2224         }
2225
2226         /* Update host with assigned MAC address. */
2227         bcopy(IF_LLADDR(ifp), sc->vtnet_hwaddr, ETHER_ADDR_LEN);
2228         vtnet_set_hwaddr(sc);
2229
2230         ifp->if_hwassist = 0;
2231         if (ifp->if_capenable & IFCAP_TXCSUM)
2232                 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;
2233         if (ifp->if_capenable & IFCAP_TSO4)
2234                 ifp->if_hwassist |= CSUM_TSO;
2235
2236         error = vtnet_init_rx_vq(sc);
2237         if (error) {
2238                 device_printf(dev,
2239                     "cannot allocate mbufs for Rx virtqueue\n");
2240                 vtnet_stop(sc);
2241                 return;
2242         }
2243
2244         if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
2245                 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
2246                         /* Restore promiscuous and all-multicast modes. */
2247                         vtnet_rx_filter(sc);
2248
2249                         /* Restore filtered MAC addresses. */
2250                         vtnet_rx_filter_mac(sc);
2251                 }
2252
2253                 /* Restore VLAN filters. */
2254                 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
2255                         vtnet_rx_filter_vlan(sc);
2256         }
2257
2258         {
2259                 vtnet_enable_rx_intr(sc);
2260                 vtnet_enable_tx_intr(sc);
2261         }
2262
2263         ifp->if_flags |= IFF_RUNNING;
2264         ifq_clr_oactive(&ifp->if_snd);
2265
2266         virtio_reinit_complete(dev);
2267
2268         vtnet_update_link_status(sc);
2269         callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc);
2270 }
2271
2272 static void
2273 vtnet_init(void *xsc)
2274 {
2275         struct vtnet_softc *sc;
2276
2277         sc = xsc;
2278
2279         lwkt_serialize_enter(&sc->vtnet_slz);
2280         vtnet_init_locked(sc);
2281         lwkt_serialize_exit(&sc->vtnet_slz);
2282 }
2283
2284 static void
2285 vtnet_exec_ctrl_cmd(struct vtnet_softc *sc, void *cookie,
2286     struct sglist *sg, int readable, int writable)
2287 {
2288         struct virtqueue *vq;
2289         void *c;
2290
2291         vq = sc->vtnet_ctrl_vq;
2292
2293         ASSERT_SERIALIZED(&sc->vtnet_slz);
2294         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ,
2295             ("no control virtqueue"));
2296         KASSERT(virtqueue_empty(vq),
2297             ("control command already enqueued"));
2298
2299         if (virtqueue_enqueue(vq, cookie, sg, readable, writable) != 0)
2300                 return;
2301
2302         virtqueue_notify(vq, &sc->vtnet_slz);
2303
2304         /*
2305          * Poll until the command is complete. Previously, we would
2306          * sleep until the control virtqueue interrupt handler woke
2307          * us up, but dropping the VTNET_MTX leads to serialization
2308          * difficulties.
2309          *
2310          * Furthermore, it appears QEMU/KVM only allocates three MSIX
2311          * vectors. Two of those vectors are needed for the Rx and Tx
2312          * virtqueues. We do not support sharing both a Vq and config
2313          * changed notification on the same MSIX vector.
2314          */
2315         c = virtqueue_poll(vq, NULL);
2316         KASSERT(c == cookie, ("unexpected control command response"));
2317 }
2318
2319 static int
2320 vtnet_ctrl_mac_cmd(struct vtnet_softc *sc, uint8_t *hwaddr)
2321 {
2322         struct {
2323                 struct virtio_net_ctrl_hdr hdr __aligned(2);
2324                 uint8_t pad1;
2325                 char aligned_hwaddr[ETHER_ADDR_LEN] __aligned(8);
2326                 uint8_t pad2;
2327                 uint8_t ack;
2328         } s;
2329         struct sglist_seg segs[3];
2330         struct sglist sg;
2331         int error;
2332
2333         s.hdr.class = VIRTIO_NET_CTRL_MAC;
2334         s.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET;
2335         s.ack = VIRTIO_NET_ERR;
2336
2337         /* Copy the mac address into physically contiguous memory */
2338         memcpy(s.aligned_hwaddr, hwaddr, ETHER_ADDR_LEN);
2339
2340         sglist_init(&sg, 3, segs);
2341         error = 0;
2342         error |= sglist_append(&sg, &s.hdr,
2343             sizeof(struct virtio_net_ctrl_hdr));
2344         error |= sglist_append(&sg, s.aligned_hwaddr, ETHER_ADDR_LEN);
2345         error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
2346         KASSERT(error == 0 && sg.sg_nseg == 3,
2347             ("%s: error %d adding set MAC msg to sglist", __func__, error));
2348
2349         vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
2350
2351         return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
2352 }
2353
2354 static void
2355 vtnet_rx_filter(struct vtnet_softc *sc)
2356 {
2357         device_t dev;
2358         struct ifnet *ifp;
2359
2360         dev = sc->vtnet_dev;
2361         ifp = sc->vtnet_ifp;
2362
2363         ASSERT_SERIALIZED(&sc->vtnet_slz);
2364         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
2365             ("CTRL_RX feature not negotiated"));
2366
2367         if (vtnet_set_promisc(sc, ifp->if_flags & IFF_PROMISC) != 0)
2368                 device_printf(dev, "cannot %s promiscuous mode\n",
2369                     ifp->if_flags & IFF_PROMISC ? "enable" : "disable");
2370
2371         if (vtnet_set_allmulti(sc, ifp->if_flags & IFF_ALLMULTI) != 0)
2372                 device_printf(dev, "cannot %s all-multicast mode\n",
2373                     ifp->if_flags & IFF_ALLMULTI ? "enable" : "disable");
2374 }
2375
2376 static int
2377 vtnet_ctrl_rx_cmd(struct vtnet_softc *sc, int cmd, int on)
2378 {
2379         struct virtio_net_ctrl_hdr hdr __aligned(2);
2380         struct sglist_seg segs[3];
2381         struct sglist sg;
2382         uint8_t onoff, ack;
2383         int error;
2384
2385         if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0)
2386                 return (ENOTSUP);
2387
2388         error = 0;
2389
2390         hdr.class = VIRTIO_NET_CTRL_RX;
2391         hdr.cmd = cmd;
2392         onoff = !!on;
2393         ack = VIRTIO_NET_ERR;
2394
2395         sglist_init(&sg, 3, segs);
2396         error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
2397         error |= sglist_append(&sg, &onoff, sizeof(uint8_t));
2398         error |= sglist_append(&sg, &ack, sizeof(uint8_t));
2399         KASSERT(error == 0 && sg.sg_nseg == 3,
2400             ("error adding Rx filter message to sglist"));
2401
2402         vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
2403
2404         return (ack == VIRTIO_NET_OK ? 0 : EIO);
2405 }
2406
2407 static int
2408 vtnet_set_promisc(struct vtnet_softc *sc, int on)
2409 {
2410
2411         return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_PROMISC, on));
2412 }
2413
2414 static int
2415 vtnet_set_allmulti(struct vtnet_softc *sc, int on)
2416 {
2417
2418         return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, on));
2419 }
2420
2421 static void
2422 vtnet_rx_filter_mac(struct vtnet_softc *sc)
2423 {
2424         struct virtio_net_ctrl_hdr hdr __aligned(2);
2425         struct vtnet_mac_filter *filter;
2426         struct sglist_seg segs[4];
2427         struct sglist sg;
2428         struct ifnet *ifp;
2429         struct ifaddr *ifa;
2430         struct ifaddr_container *ifac;
2431         struct ifmultiaddr *ifma;
2432         int ucnt, mcnt, promisc, allmulti, error;
2433         uint8_t ack;
2434
2435         ifp = sc->vtnet_ifp;
2436         ucnt = 0;
2437         mcnt = 0;
2438         promisc = 0;
2439         allmulti = 0;
2440
2441         ASSERT_SERIALIZED(&sc->vtnet_slz);
2442         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
2443             ("%s: CTRL_RX feature not negotiated", __func__));
2444
2445         /* Use the MAC filtering table allocated in vtnet_attach. */
2446         filter = sc->vtnet_macfilter;
2447         memset(filter, 0, sizeof(struct vtnet_mac_filter));
2448
2449         /* Unicast MAC addresses: */
2450         //if_addr_rlock(ifp);
2451         TAILQ_FOREACH(ifac, &ifp->if_addrheads[mycpuid], ifa_link) {
2452                 ifa = ifac->ifa;
2453                 if (ifa->ifa_addr->sa_family != AF_LINK)
2454                         continue;
2455                 else if (memcmp(LLADDR((struct sockaddr_dl *)ifa->ifa_addr),
2456                     sc->vtnet_hwaddr, ETHER_ADDR_LEN) == 0)
2457                         continue;
2458                 else if (ucnt == VTNET_MAX_MAC_ENTRIES) {
2459                         promisc = 1;
2460                         break;
2461                 }
2462
2463                 bcopy(LLADDR((struct sockaddr_dl *)ifa->ifa_addr),
2464                     &filter->vmf_unicast.macs[ucnt], ETHER_ADDR_LEN);
2465                 ucnt++;
2466         }
2467         //if_addr_runlock(ifp);
2468
2469         if (promisc != 0) {
2470                 filter->vmf_unicast.nentries = 0;
2471                 if_printf(ifp, "more than %d MAC addresses assigned, "
2472                     "falling back to promiscuous mode\n",
2473                     VTNET_MAX_MAC_ENTRIES);
2474         } else
2475                 filter->vmf_unicast.nentries = ucnt;
2476
2477         /* Multicast MAC addresses: */
2478         //if_maddr_rlock(ifp);
2479         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2480                 if (ifma->ifma_addr->sa_family != AF_LINK)
2481                         continue;
2482                 else if (mcnt == VTNET_MAX_MAC_ENTRIES) {
2483                         allmulti = 1;
2484                         break;
2485                 }
2486
2487                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2488                     &filter->vmf_multicast.macs[mcnt], ETHER_ADDR_LEN);
2489                 mcnt++;
2490         }
2491         //if_maddr_runlock(ifp);
2492
2493         if (allmulti != 0) {
2494                 filter->vmf_multicast.nentries = 0;
2495                 if_printf(ifp, "more than %d multicast MAC addresses "
2496                     "assigned, falling back to all-multicast mode\n",
2497                     VTNET_MAX_MAC_ENTRIES);
2498         } else
2499                 filter->vmf_multicast.nentries = mcnt;
2500
2501         if (promisc != 0 && allmulti != 0)
2502                 goto out;
2503
2504         hdr.class = VIRTIO_NET_CTRL_MAC;
2505         hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
2506         ack = VIRTIO_NET_ERR;
2507
2508         sglist_init(&sg, 4, segs);
2509         error = 0;
2510         error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
2511         error |= sglist_append(&sg, &filter->vmf_unicast,
2512             sizeof(uint32_t) + filter->vmf_unicast.nentries * ETHER_ADDR_LEN);
2513         error |= sglist_append(&sg, &filter->vmf_multicast,
2514             sizeof(uint32_t) + filter->vmf_multicast.nentries * ETHER_ADDR_LEN);
2515         error |= sglist_append(&sg, &ack, sizeof(uint8_t));
2516         KASSERT(error == 0 && sg.sg_nseg == 4,
2517             ("%s: error %d adding MAC filter msg to sglist", __func__, error));
2518
2519         vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
2520
2521         if (ack != VIRTIO_NET_OK)
2522                 if_printf(ifp, "error setting host MAC filter table\n");
2523
2524 out:
2525         if (promisc != 0 && vtnet_set_promisc(sc, 1) != 0)
2526                 if_printf(ifp, "cannot enable promiscuous mode\n");
2527         if (allmulti != 0 && vtnet_set_allmulti(sc, 1) != 0)
2528                 if_printf(ifp, "cannot enable all-multicast mode\n");
2529 }
2530
2531 static int
2532 vtnet_exec_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
2533 {
2534         struct sglist_seg segs[3];
2535         struct sglist sg;
2536         struct {
2537                 struct virtio_net_ctrl_hdr hdr __aligned(2);
2538                 uint8_t pad1;
2539                 uint16_t tag;
2540                 uint8_t pad2;
2541                 uint8_t ack;
2542         } s;
2543         int error;
2544
2545         s.hdr.class = VIRTIO_NET_CTRL_VLAN;
2546         s.hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
2547         s.tag = tag;
2548         s.ack = VIRTIO_NET_ERR;
2549
2550         sglist_init(&sg, 3, segs);
2551         error = 0;
2552         error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
2553         error |= sglist_append(&sg, &s.tag, sizeof(uint16_t));
2554         error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
2555         KASSERT(error == 0 && sg.sg_nseg == 3,
2556             ("%s: error %d adding VLAN message to sglist", __func__, error));
2557
2558         vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
2559
2560         return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
2561 }
2562
2563 static void
2564 vtnet_rx_filter_vlan(struct vtnet_softc *sc)
2565 {
2566         device_t dev;
2567         uint32_t w, mask;
2568         uint16_t tag;
2569         int i, nvlans, error;
2570
2571         ASSERT_SERIALIZED(&sc->vtnet_slz);
2572         KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER,
2573             ("VLAN_FILTER feature not negotiated"));
2574
2575         dev = sc->vtnet_dev;
2576         nvlans = sc->vtnet_nvlans;
2577         error = 0;
2578
2579         /* Enable filtering for each configured VLAN. */
2580         for (i = 0; i < VTNET_VLAN_SHADOW_SIZE && nvlans > 0; i++) {
2581                 w = sc->vtnet_vlan_shadow[i];
2582                 for (mask = 1, tag = i * 32; w != 0; mask <<= 1, tag++) {
2583                         if ((w & mask) != 0) {
2584                                 w &= ~mask;
2585                                 nvlans--;
2586                                 if (vtnet_exec_vlan_filter(sc, 1, tag) != 0)
2587                                         error++;
2588                         }
2589                 }
2590         }
2591
2592         KASSERT(nvlans == 0, ("VLAN count incorrect"));
2593         if (error)
2594                 device_printf(dev, "cannot restore VLAN filter table\n");
2595 }
2596
2597 static void
2598 vtnet_set_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
2599 {
2600         struct ifnet *ifp;
2601         int idx, bit;
2602
2603         KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER,
2604             ("VLAN_FILTER feature not negotiated"));
2605
2606         if ((tag == 0) || (tag > 4095))
2607                 return;
2608
2609         ifp = sc->vtnet_ifp;
2610         idx = (tag >> 5) & 0x7F;
2611         bit = tag & 0x1F;
2612
2613         lwkt_serialize_enter(&sc->vtnet_slz);
2614
2615         /* Update shadow VLAN table. */
2616         if (add) {
2617                 sc->vtnet_nvlans++;
2618                 sc->vtnet_vlan_shadow[idx] |= (1 << bit);
2619         } else {
2620                 sc->vtnet_nvlans--;
2621                 sc->vtnet_vlan_shadow[idx] &= ~(1 << bit);
2622         }
2623
2624         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
2625                 if (vtnet_exec_vlan_filter(sc, add, tag) != 0) {
2626                         device_printf(sc->vtnet_dev,
2627                             "cannot %s VLAN %d %s the host filter table\n",
2628                             add ? "add" : "remove", tag,
2629                             add ? "to" : "from");
2630                 }
2631         }
2632
2633         lwkt_serialize_exit(&sc->vtnet_slz);
2634 }
2635
2636 static void
2637 vtnet_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
2638 {
2639
2640         if (ifp->if_softc != arg)
2641                 return;
2642
2643         vtnet_set_vlan_filter(arg, 1, tag);
2644 }
2645
2646 static void
2647 vtnet_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
2648 {
2649
2650         if (ifp->if_softc != arg)
2651                 return;
2652
2653         vtnet_set_vlan_filter(arg, 0, tag);
2654 }
2655
2656 static int
2657 vtnet_ifmedia_upd(struct ifnet *ifp)
2658 {
2659         struct vtnet_softc *sc;
2660         struct ifmedia *ifm;
2661
2662         sc = ifp->if_softc;
2663         ifm = &sc->vtnet_media;
2664
2665         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
2666                 return (EINVAL);
2667
2668         return (0);
2669 }
2670
2671 static void
2672 vtnet_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
2673 {
2674         struct vtnet_softc *sc;
2675
2676         sc = ifp->if_softc;
2677
2678         ifmr->ifm_status = IFM_AVALID;
2679         ifmr->ifm_active = IFM_ETHER;
2680
2681         lwkt_serialize_enter(&sc->vtnet_slz);
2682         if (vtnet_is_link_up(sc) != 0) {
2683                 ifmr->ifm_status |= IFM_ACTIVE;
2684                 ifmr->ifm_active |= VTNET_MEDIATYPE;
2685         } else
2686                 ifmr->ifm_active |= IFM_NONE;
2687         lwkt_serialize_exit(&sc->vtnet_slz);
2688 }
2689
2690 static void
2691 vtnet_add_statistics(struct vtnet_softc *sc)
2692 {
2693         device_t dev;
2694         struct vtnet_statistics *stats;
2695         struct sysctl_ctx_list *ctx;
2696         struct sysctl_oid *tree;
2697         struct sysctl_oid_list *child;
2698
2699         dev = sc->vtnet_dev;
2700         stats = &sc->vtnet_stats;
2701         ctx = device_get_sysctl_ctx(dev);
2702         tree = device_get_sysctl_tree(dev);
2703         child = SYSCTL_CHILDREN(tree);
2704
2705         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_alloc_failed",
2706             CTLFLAG_RD, &stats->mbuf_alloc_failed,
2707             "Mbuf cluster allocation failures");
2708         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_frame_too_large",
2709             CTLFLAG_RD, &stats->rx_frame_too_large,
2710             "Received frame larger than the mbuf chain");
2711         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_enq_replacement_failed",
2712             CTLFLAG_RD, &stats->rx_enq_replacement_failed,
2713             "Enqueuing the replacement receive mbuf failed");
2714         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_mergeable_failed",
2715             CTLFLAG_RD, &stats->rx_mergeable_failed,
2716             "Mergeable buffers receive failures");
2717         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_bad_ethtype",
2718             CTLFLAG_RD, &stats->rx_csum_bad_ethtype,
2719             "Received checksum offloaded buffer with unsupported "
2720             "Ethernet type");
2721         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_bad_start",
2722             CTLFLAG_RD, &stats->rx_csum_bad_start,
2723             "Received checksum offloaded buffer with incorrect start offset");
2724         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_bad_ipproto",
2725             CTLFLAG_RD, &stats->rx_csum_bad_ipproto,
2726             "Received checksum offloaded buffer with incorrect IP protocol");
2727         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_bad_offset",
2728             CTLFLAG_RD, &stats->rx_csum_bad_offset,
2729             "Received checksum offloaded buffer with incorrect offset");
2730         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_failed",
2731             CTLFLAG_RD, &stats->rx_csum_failed,
2732             "Received buffer checksum offload failed");
2733         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_offloaded",
2734             CTLFLAG_RD, &stats->rx_csum_offloaded,
2735             "Received buffer checksum offload succeeded");
2736         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_task_rescheduled",
2737             CTLFLAG_RD, &stats->rx_task_rescheduled,
2738             "Times the receive interrupt task rescheduled itself");
2739
2740         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_csum_offloaded",
2741             CTLFLAG_RD, &stats->tx_csum_offloaded,
2742             "Offloaded checksum of transmitted buffer");
2743         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_tso_offloaded",
2744             CTLFLAG_RD, &stats->tx_tso_offloaded,
2745             "Segmentation offload of transmitted buffer");
2746         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_csum_bad_ethtype",
2747             CTLFLAG_RD, &stats->tx_csum_bad_ethtype,
2748             "Aborted transmit of checksum offloaded buffer with unknown "
2749             "Ethernet type");
2750         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_tso_bad_ethtype",
2751             CTLFLAG_RD, &stats->tx_tso_bad_ethtype,
2752             "Aborted transmit of TSO buffer with unknown Ethernet type");
2753         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_task_rescheduled",
2754             CTLFLAG_RD, &stats->tx_task_rescheduled,
2755             "Times the transmit interrupt task rescheduled itself");
2756 }
2757
2758 static int
2759 vtnet_enable_rx_intr(struct vtnet_softc *sc)
2760 {
2761
2762         return (virtqueue_enable_intr(sc->vtnet_rx_vq));
2763 }
2764
2765 static void
2766 vtnet_disable_rx_intr(struct vtnet_softc *sc)
2767 {
2768
2769         virtqueue_disable_intr(sc->vtnet_rx_vq);
2770 }
2771
2772 static int
2773 vtnet_enable_tx_intr(struct vtnet_softc *sc)
2774 {
2775
2776 #ifdef VTNET_TX_INTR_MODERATION
2777         return (0);
2778 #else
2779         return (virtqueue_enable_intr(sc->vtnet_tx_vq));
2780 #endif
2781 }
2782
2783 static void
2784 vtnet_disable_tx_intr(struct vtnet_softc *sc)
2785 {
2786
2787         virtqueue_disable_intr(sc->vtnet_tx_vq);
2788 }