sys/dev/virtual/virtio/net/if_vtnet.c

   1 /*-
   2  * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice unmodified, this list of conditions, and the following
  10  *    disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 /* Driver for VirtIO network devices. */
  28
  29 #include <sys/cdefs.h>
  30
  31 #include <sys/param.h>
  32 #include <sys/systm.h>
  33 #include <sys/kernel.h>
  34 #include <sys/sockio.h>
  35 #include <sys/mbuf.h>
  36 #include <sys/malloc.h>
  37 #include <sys/module.h>
  38 #include <sys/socket.h>
  39 #include <sys/sysctl.h>
  40 #include <sys/taskqueue.h>
  41 #include <sys/random.h>
  42 #include <sys/sglist.h>
  43 #include <sys/serialize.h>
  44 #include <sys/bus.h>
  45 #include <sys/rman.h>
  46
  47 #include <net/ethernet.h>
  48 #include <net/if.h>
  49 #include <net/if_arp.h>
  50 #include <net/if_dl.h>
  51 #include <net/if_types.h>
  52 #include <net/if_media.h>
  53 #include <net/vlan/if_vlan_var.h>
  54 #include <net/vlan/if_vlan_ether.h>
  55 #include <net/ifq_var.h>
  56
  57 #include <net/bpf.h>
  58
  59 #include <netinet/in_systm.h>
  60 #include <netinet/in.h>
  61 #include <netinet/ip.h>
  62 #include <netinet/ip6.h>
  63 #include <netinet/udp.h>
  64 #include <netinet/tcp.h>
  65 #include <netinet/sctp.h>
  66
  67 #include <dev/virtual/virtio/virtio/virtio.h>
  68 #include <dev/virtual/virtio/virtio/virtqueue.h>
  69
  70 #include "virtio_net.h"
  71 #include "virtio_if.h"
  72
  73 struct vtnet_statistics {
  74         unsigned long           mbuf_alloc_failed;
  75
  76         unsigned long           rx_frame_too_large;
  77         unsigned long           rx_enq_replacement_failed;
  78         unsigned long           rx_mergeable_failed;
  79         unsigned long           rx_csum_bad_ethtype;
  80         unsigned long           rx_csum_bad_start;
  81         unsigned long           rx_csum_bad_ipproto;
  82         unsigned long           rx_csum_bad_offset;
  83         unsigned long           rx_csum_failed;
  84         unsigned long           rx_csum_offloaded;
  85         unsigned long           rx_task_rescheduled;
  86
  87         unsigned long           tx_csum_offloaded;
  88         unsigned long           tx_tso_offloaded;
  89         unsigned long           tx_csum_bad_ethtype;
  90         unsigned long           tx_tso_bad_ethtype;
  91         unsigned long           tx_task_rescheduled;
  92 };
  93
  94 struct vtnet_softc {
  95         device_t                vtnet_dev;
  96         struct ifnet            *vtnet_ifp;
  97         struct lwkt_serialize   vtnet_slz;
  98
  99         uint32_t                vtnet_flags;
 100 #define VTNET_FLAG_LINK         0x0001
 101 #define VTNET_FLAG_SUSPENDED    0x0002
 102 #define VTNET_FLAG_CTRL_VQ      0x0004
 103 #define VTNET_FLAG_CTRL_RX      0x0008
 104 #define VTNET_FLAG_VLAN_FILTER  0x0010
 105 #define VTNET_FLAG_TSO_ECN      0x0020
 106 #define VTNET_FLAG_MRG_RXBUFS   0x0040
 107 #define VTNET_FLAG_LRO_NOMRG    0x0080
 108
 109         struct virtqueue        *vtnet_rx_vq;
 110         struct virtqueue        *vtnet_tx_vq;
 111         struct virtqueue        *vtnet_ctrl_vq;
 112
 113         struct vtnet_tx_header  *vtnet_txhdrarea;
 114         uint32_t                vtnet_txhdridx;
 115         struct vtnet_mac_filter *vtnet_macfilter;
 116
 117         int                     vtnet_hdr_size;
 118         int                     vtnet_tx_size;
 119         int                     vtnet_rx_size;
 120         int                     vtnet_rx_process_limit;
 121         int                     vtnet_rx_mbuf_size;
 122         int                     vtnet_rx_mbuf_count;
 123         int                     vtnet_if_flags;
 124         int                     vtnet_watchdog_timer;
 125         uint64_t                vtnet_features;
 126
 127         struct task             vtnet_cfgchg_task;
 128
 129         struct vtnet_statistics vtnet_stats;
 130
 131         struct sysctl_ctx_list  vtnet_sysctl_ctx;
 132         struct sysctl_oid       *vtnet_sysctl_tree;
 133
 134         struct callout          vtnet_tick_ch;
 135
 136         eventhandler_tag        vtnet_vlan_attach;
 137         eventhandler_tag        vtnet_vlan_detach;
 138
 139         struct ifmedia          vtnet_media;
 140         /*
 141          * Fake media type; the host does not provide us with
 142          * any real media information.
 143          */
 144 #define VTNET_MEDIATYPE         (IFM_ETHER | IFM_1000_T | IFM_FDX)
 145         char                    vtnet_hwaddr[ETHER_ADDR_LEN];
 146
 147         /*
 148          * During reset, the host's VLAN filtering table is lost. The
 149          * array below is used to restore all the VLANs configured on
 150          * this interface after a reset.
 151          */
 152 #define VTNET_VLAN_SHADOW_SIZE  (4096 / 32)
 153         int                     vtnet_nvlans;
 154         uint32_t                vtnet_vlan_shadow[VTNET_VLAN_SHADOW_SIZE];
 155
 156         char                    vtnet_mtx_name[16];
 157 };
 158
 159 /*
 160  * When mergeable buffers are not negotiated, the vtnet_rx_header structure
 161  * below is placed at the beginning of the mbuf data. Use 4 bytes of pad to
 162  * both keep the VirtIO header and the data non-contiguous and to keep the
 163  * frame's payload 4 byte aligned.
 164  *
 165  * When mergeable buffers are negotiated, the host puts the VirtIO header in
 166  * the beginning of the first mbuf's data.
 167  */
 168 #define VTNET_RX_HEADER_PAD     4
 169 struct vtnet_rx_header {
 170         struct virtio_net_hdr   vrh_hdr;
 171         char                    vrh_pad[VTNET_RX_HEADER_PAD];
 172 } __packed;
 173
 174 /*
 175  * For each outgoing frame, the vtnet_tx_header below is allocated from
 176  * the vtnet_tx_header_zone.
 177  */
 178 struct vtnet_tx_header {
 179         union {
 180                 struct virtio_net_hdr           hdr;
 181                 struct virtio_net_hdr_mrg_rxbuf mhdr;
 182         } vth_uhdr;
 183
 184         struct mbuf             *vth_mbuf;
 185 };
 186
 187 MALLOC_DEFINE(M_VTNET, "VTNET_TX", "Outgoing VTNET TX frame header");
 188
 189 /*
 190  * The VirtIO specification does not place a limit on the number of MAC
 191  * addresses the guest driver may request to be filtered. In practice,
 192  * the host is constrained by available resources. To simplify this driver,
 193  * impose a reasonably high limit of MAC addresses we will filter before
 194  * falling back to promiscuous or all-multicast modes.
 195  */
 196 #define VTNET_MAX_MAC_ENTRIES   128
 197
 198 struct vtnet_mac_table {
 199         uint32_t                nentries;
 200         uint8_t                 macs[VTNET_MAX_MAC_ENTRIES][ETHER_ADDR_LEN];
 201 } __packed;
 202
 203 struct vtnet_mac_filter {
 204         struct vtnet_mac_table  vmf_unicast;
 205         uint32_t                vmf_pad; /* Make tables non-contiguous. */
 206         struct vtnet_mac_table  vmf_multicast;
 207 };
 208
 209 /*
 210  * The MAC filter table is malloc(9)'d when needed. Ensure it will
 211  * always fit in one segment.
 212  */
 213 CTASSERT(sizeof(struct vtnet_mac_filter) <= PAGE_SIZE);
 214
 215 #define VTNET_WATCHDOG_TIMEOUT  5
 216 #define VTNET_CSUM_OFFLOAD      (CSUM_TCP | CSUM_UDP)// | CSUM_SCTP)
 217
 218 /* Features desired/implemented by this driver. */
 219 #define VTNET_FEATURES          \
 220     (VIRTIO_NET_F_MAC           | \
 221      VIRTIO_NET_F_STATUS        | \
 222      VIRTIO_NET_F_CTRL_VQ       | \
 223      VIRTIO_NET_F_CTRL_RX       | \
 224      VIRTIO_NET_F_CTRL_VLAN     | \
 225      VIRTIO_NET_F_CSUM          | \
 226      VIRTIO_NET_F_HOST_TSO4     | \
 227      VIRTIO_NET_F_HOST_TSO6     | \
 228      VIRTIO_NET_F_HOST_ECN      | \
 229      VIRTIO_NET_F_GUEST_CSUM    | \
 230      VIRTIO_NET_F_GUEST_TSO4    | \
 231      VIRTIO_NET_F_GUEST_TSO6    | \
 232      VIRTIO_NET_F_GUEST_ECN     | \
 233      VIRTIO_NET_F_MRG_RXBUF)
 234
 235 /*
 236  * The VIRTIO_NET_F_GUEST_TSO[46] features permit the host to send us
 237  * frames larger than 1514 bytes. We do not yet support software LRO
 238  * via tcp_lro_rx().
 239  */
 240 #define VTNET_LRO_FEATURES (VIRTIO_NET_F_GUEST_TSO4 | \
 241                             VIRTIO_NET_F_GUEST_TSO6 | VIRTIO_NET_F_GUEST_ECN)
 242
 243 #define VTNET_MAX_MTU           65536
 244 #define VTNET_MAX_RX_SIZE       65550
 245
 246 /*
 247  * Used to preallocate the Vq indirect descriptors. The first segment
 248  * is reserved for the header.
 249  */
 250 #define VTNET_MIN_RX_SEGS       2
 251 #define VTNET_MAX_RX_SEGS       34
 252 #define VTNET_MAX_TX_SEGS       34
 253
 254 #ifndef CSUM_TSO
 255 #define CSUM_TSO                0
 256 #endif
 257
 258 #define IFCAP_TSO4              0x00100 /* can do TCP Segmentation Offload */
 259 #define IFCAP_TSO6              0x00200 /* can do TCP6 Segmentation Offload */
 260 #define IFCAP_LRO               0x00400 /* can do Large Receive Offload */
 261 #define IFCAP_VLAN_HWFILTER     0x10000 /* interface hw can filter vlan tag */
 262 #define IFCAP_VLAN_HWTSO        0x40000 /* can do IFCAP_TSO on VLANs */
 263
 264
 265 /*
 266  * Assert we can receive and transmit the maximum with regular
 267  * size clusters.
 268  */
 269 CTASSERT(((VTNET_MAX_RX_SEGS - 1) * MCLBYTES) >= VTNET_MAX_RX_SIZE);
 270 CTASSERT(((VTNET_MAX_TX_SEGS - 1) * MCLBYTES) >= VTNET_MAX_MTU);
 271
 272 /*
 273  * Determine how many mbufs are in each receive buffer. For LRO without
 274  * mergeable descriptors, we must allocate an mbuf chain large enough to
 275  * hold both the vtnet_rx_header and the maximum receivable data.
 276  */
 277 #define VTNET_NEEDED_RX_MBUFS(_sc)                                      \
 278         ((_sc)->vtnet_flags & VTNET_FLAG_LRO_NOMRG) == 0 ? 1 :          \
 279         howmany(sizeof(struct vtnet_rx_header) + VTNET_MAX_RX_SIZE,     \
 280         (_sc)->vtnet_rx_mbuf_size)
 281
 282 static int      vtnet_modevent(module_t, int, void *);
 283
 284 static int      vtnet_probe(device_t);
 285 static int      vtnet_attach(device_t);
 286 static int      vtnet_detach(device_t);
 287 static int      vtnet_suspend(device_t);
 288 static int      vtnet_resume(device_t);
 289 static int      vtnet_shutdown(device_t);
 290 static int      vtnet_config_change(device_t);
 291
 292 static void     vtnet_negotiate_features(struct vtnet_softc *);
 293 static int      vtnet_alloc_virtqueues(struct vtnet_softc *);
 294 static void     vtnet_get_hwaddr(struct vtnet_softc *);
 295 static void     vtnet_set_hwaddr(struct vtnet_softc *);
 296 static int      vtnet_is_link_up(struct vtnet_softc *);
 297 static void     vtnet_update_link_status(struct vtnet_softc *);
 298 #if 0
 299 static void     vtnet_watchdog(struct vtnet_softc *);
 300 #endif
 301 static void     vtnet_config_change_task(void *, int);
 302 static int      vtnet_change_mtu(struct vtnet_softc *, int);
 303 static int      vtnet_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 304
 305 static int      vtnet_init_rx_vq(struct vtnet_softc *);
 306 static void     vtnet_free_rx_mbufs(struct vtnet_softc *);
 307 static void     vtnet_free_tx_mbufs(struct vtnet_softc *);
 308 static void     vtnet_free_ctrl_vq(struct vtnet_softc *);
 309
 310 static struct mbuf * vtnet_alloc_rxbuf(struct vtnet_softc *, int,
 311                     struct mbuf **);
 312 static int      vtnet_replace_rxbuf(struct vtnet_softc *,
 313                     struct mbuf *, int);
 314 static int      vtnet_newbuf(struct vtnet_softc *);
 315 static void     vtnet_discard_merged_rxbuf(struct vtnet_softc *, int);
 316 static void     vtnet_discard_rxbuf(struct vtnet_softc *, struct mbuf *);
 317 static int      vtnet_enqueue_rxbuf(struct vtnet_softc *, struct mbuf *);
 318 static void     vtnet_vlan_tag_remove(struct mbuf *);
 319 static int      vtnet_rx_csum(struct vtnet_softc *, struct mbuf *,
 320                     struct virtio_net_hdr *);
 321 static int      vtnet_rxeof_merged(struct vtnet_softc *, struct mbuf *, int);
 322 static int      vtnet_rxeof(struct vtnet_softc *, int, int *);
 323 static void     vtnet_rx_intr_task(void *);
 324 static int      vtnet_rx_vq_intr(void *);
 325
 326 static void     vtnet_txeof(struct vtnet_softc *);
 327 static struct mbuf * vtnet_tx_offload(struct vtnet_softc *, struct mbuf *,
 328                     struct virtio_net_hdr *);
 329 static int      vtnet_enqueue_txbuf(struct vtnet_softc *, struct mbuf **,
 330                     struct vtnet_tx_header *);
 331 static int      vtnet_encap(struct vtnet_softc *, struct mbuf **);
 332 static void     vtnet_start_locked(struct ifnet *, struct ifaltq_subque *);
 333 static void     vtnet_start(struct ifnet *, struct ifaltq_subque *);
 334 static void     vtnet_tick(void *);
 335 static void     vtnet_tx_intr_task(void *);
 336 static int      vtnet_tx_vq_intr(void *);
 337
 338 static void     vtnet_stop(struct vtnet_softc *);
 339 static int      vtnet_reinit(struct vtnet_softc *);
 340 static void     vtnet_init_locked(struct vtnet_softc *);
 341 static void     vtnet_init(void *);
 342
 343 static void     vtnet_exec_ctrl_cmd(struct vtnet_softc *, void *,
 344                     struct sglist *, int, int);
 345
 346 static void     vtnet_rx_filter(struct vtnet_softc *sc);
 347 static int      vtnet_ctrl_rx_cmd(struct vtnet_softc *, int, int);
 348 static int      vtnet_set_promisc(struct vtnet_softc *, int);
 349 static int      vtnet_set_allmulti(struct vtnet_softc *, int);
 350 static void     vtnet_rx_filter_mac(struct vtnet_softc *);
 351
 352 static int      vtnet_exec_vlan_filter(struct vtnet_softc *, int, uint16_t);
 353 static void     vtnet_rx_filter_vlan(struct vtnet_softc *);
 354 static void     vtnet_set_vlan_filter(struct vtnet_softc *, int, uint16_t);
 355 static void     vtnet_register_vlan(void *, struct ifnet *, uint16_t);
 356 static void     vtnet_unregister_vlan(void *, struct ifnet *, uint16_t);
 357
 358 static int      vtnet_ifmedia_upd(struct ifnet *);
 359 static void     vtnet_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 360
 361 static void     vtnet_add_statistics(struct vtnet_softc *);
 362
 363 static int      vtnet_enable_rx_intr(struct vtnet_softc *);
 364 static int      vtnet_enable_tx_intr(struct vtnet_softc *);
 365 static void     vtnet_disable_rx_intr(struct vtnet_softc *);
 366 static void     vtnet_disable_tx_intr(struct vtnet_softc *);
 367
 368 /* Tunables. */
 369 static int vtnet_csum_disable = 0;
 370 TUNABLE_INT("hw.vtnet.csum_disable", &vtnet_csum_disable);
 371 static int vtnet_tso_disable = 1;
 372 TUNABLE_INT("hw.vtnet.tso_disable", &vtnet_tso_disable);
 373 static int vtnet_lro_disable = 1;
 374 TUNABLE_INT("hw.vtnet.lro_disable", &vtnet_lro_disable);
 375
 376 /*
 377  * Reducing the number of transmit completed interrupts can
 378  * improve performance. To do so, the define below keeps the
 379  * Tx vq interrupt disabled and adds calls to vtnet_txeof()
 380  * in the start and watchdog paths. The price to pay for this
 381  * is the m_free'ing of transmitted mbufs may be delayed until
 382  * the watchdog fires.
 383  */
 384 #define VTNET_TX_INTR_MODERATION
 385
 386 static struct virtio_feature_desc vtnet_feature_desc[] = {
 387         { VIRTIO_NET_F_CSUM,            "TxChecksum"    },
 388         { VIRTIO_NET_F_GUEST_CSUM,      "RxChecksum"    },
 389         { VIRTIO_NET_F_MAC,             "MacAddress"    },
 390         { VIRTIO_NET_F_GSO,             "TxAllGSO"      },
 391         { VIRTIO_NET_F_GUEST_TSO4,      "RxTSOv4"       },
 392         { VIRTIO_NET_F_GUEST_TSO6,      "RxTSOv6"       },
 393         { VIRTIO_NET_F_GUEST_ECN,       "RxECN"         },
 394         { VIRTIO_NET_F_GUEST_UFO,       "RxUFO"         },
 395         { VIRTIO_NET_F_HOST_TSO4,       "TxTSOv4"       },
 396         { VIRTIO_NET_F_HOST_TSO6,       "TxTSOv6"       },
 397         { VIRTIO_NET_F_HOST_ECN,        "TxTSOECN"      },
 398         { VIRTIO_NET_F_HOST_UFO,        "TxUFO"         },
 399         { VIRTIO_NET_F_MRG_RXBUF,       "MrgRxBuf"      },
 400         { VIRTIO_NET_F_STATUS,          "Status"        },
 401         { VIRTIO_NET_F_CTRL_VQ,         "ControlVq"     },
 402         { VIRTIO_NET_F_CTRL_RX,         "RxMode"        },
 403         { VIRTIO_NET_F_CTRL_VLAN,       "VLanFilter"    },
 404         { VIRTIO_NET_F_CTRL_RX_EXTRA,   "RxModeExtra"   },
 405         { VIRTIO_NET_F_MQ,              "RFS"           },
 406         { 0, NULL }
 407 };
 408
 409 static device_method_t vtnet_methods[] = {
 410         /* Device methods. */
 411         DEVMETHOD(device_probe,         vtnet_probe),
 412         DEVMETHOD(device_attach,        vtnet_attach),
 413         DEVMETHOD(device_detach,        vtnet_detach),
 414         DEVMETHOD(device_suspend,       vtnet_suspend),
 415         DEVMETHOD(device_resume,        vtnet_resume),
 416         DEVMETHOD(device_shutdown,      vtnet_shutdown),
 417
 418         /* VirtIO methods. */
 419         DEVMETHOD(virtio_config_change, vtnet_config_change),
 420
 421         { 0, 0 }
 422 };
 423
 424 static driver_t vtnet_driver = {
 425         "vtnet",
 426         vtnet_methods,
 427         sizeof(struct vtnet_softc)
 428 };
 429
 430 static devclass_t vtnet_devclass;
 431
 432 DRIVER_MODULE(vtnet, virtio_pci, vtnet_driver, vtnet_devclass,
 433               vtnet_modevent, 0);
 434 MODULE_VERSION(vtnet, 1);
 435 MODULE_DEPEND(vtnet, virtio, 1, 1, 1);
 436
 437 static int
 438 vtnet_modevent(module_t mod, int type, void *unused)
 439 {
 440         int error;
 441
 442         error = 0;
 443
 444         switch (type) {
 445         case MOD_LOAD:
 446                 break;
 447         case MOD_UNLOAD:
 448                 break;
 449         case MOD_SHUTDOWN:
 450                 break;
 451         default:
 452                 error = EOPNOTSUPP;
 453                 break;
 454         }
 455
 456         return (error);
 457 }
 458
 459 static int
 460 vtnet_probe(device_t dev)
 461 {
 462         if (virtio_get_device_type(dev) != VIRTIO_ID_NETWORK)
 463                 return (ENXIO);
 464
 465         device_set_desc(dev, "VirtIO Networking Adapter");
 466
 467         return (BUS_PROBE_DEFAULT);
 468 }
 469
 470 static int
 471 vtnet_attach(device_t dev)
 472 {
 473         struct vtnet_softc *sc;
 474         struct ifnet *ifp;
 475         int tx_size, error;
 476
 477         sc = device_get_softc(dev);
 478         sc->vtnet_dev = dev;
 479
 480         lwkt_serialize_init(&sc->vtnet_slz);
 481         callout_init(&sc->vtnet_tick_ch);
 482
 483         ifmedia_init(&sc->vtnet_media, IFM_IMASK, vtnet_ifmedia_upd,
 484                      vtnet_ifmedia_sts);
 485         ifmedia_add(&sc->vtnet_media, VTNET_MEDIATYPE, 0, NULL);
 486         ifmedia_set(&sc->vtnet_media, VTNET_MEDIATYPE);
 487
 488         vtnet_add_statistics(sc);
 489
 490         virtio_set_feature_desc(dev, vtnet_feature_desc);
 491         vtnet_negotiate_features(sc);
 492
 493         if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) {
 494                 sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS;
 495                 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 496         } else {
 497                 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
 498         }
 499
 500         sc->vtnet_rx_mbuf_size = MCLBYTES;
 501         sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc);
 502
 503         if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VQ)) {
 504                 sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ;
 505
 506                 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX))
 507                         sc->vtnet_flags |= VTNET_FLAG_CTRL_RX;
 508                 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VLAN))
 509                         sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER;
 510         }
 511
 512         vtnet_get_hwaddr(sc);
 513
 514         error = vtnet_alloc_virtqueues(sc);
 515         if (error) {
 516                 device_printf(dev, "cannot allocate virtqueues\n");
 517                 goto fail;
 518         }
 519
 520         ifp = sc->vtnet_ifp = if_alloc(IFT_ETHER);
 521         if (ifp == NULL) {
 522                 device_printf(dev, "cannot allocate ifnet structure\n");
 523                 error = ENOSPC;
 524                 goto fail;
 525         }
 526
 527         ifp->if_softc = sc;
 528         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 529         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 530         ifp->if_init = vtnet_init;
 531         ifp->if_start = vtnet_start;
 532         ifp->if_ioctl = vtnet_ioctl;
 533
 534         sc->vtnet_rx_size = virtqueue_size(sc->vtnet_rx_vq);
 535         sc->vtnet_rx_process_limit = sc->vtnet_rx_size;
 536
 537         tx_size = virtqueue_size(sc->vtnet_tx_vq);
 538         sc->vtnet_tx_size = tx_size;
 539         sc->vtnet_txhdridx = 0;
 540         sc->vtnet_txhdrarea = contigmalloc(
 541             ((sc->vtnet_tx_size / 2) + 1) * sizeof(struct vtnet_tx_header),
 542             M_VTNET, M_WAITOK, 0, BUS_SPACE_MAXADDR, 4, 0);
 543         if (sc->vtnet_txhdrarea == NULL) {
 544                 device_printf(dev, "cannot contigmalloc the tx headers\n");
 545                 goto fail;
 546         }
 547         sc->vtnet_macfilter = contigmalloc(
 548             sizeof(struct vtnet_mac_filter),
 549             M_VTNET, M_WAITOK, 0, BUS_SPACE_MAXADDR, 4, 0);
 550         if (sc->vtnet_macfilter == NULL) {
 551                 device_printf(dev,
 552                     "cannot contigmalloc the mac filter table\n");
 553                 goto fail;
 554         }
 555         ifq_set_maxlen(&ifp->if_snd, tx_size - 1);
 556         ifq_set_ready(&ifp->if_snd);
 557
 558         ether_ifattach(ifp, sc->vtnet_hwaddr, NULL);
 559
 560         if (virtio_with_feature(dev, VIRTIO_NET_F_STATUS)){
 561                 //ifp->if_capabilities |= IFCAP_LINKSTATE;
 562                  kprintf("add dynamic link state\n");
 563         }
 564
 565         /* Tell the upper layer(s) we support long frames. */
 566         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
 567         ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU;
 568
 569         if (virtio_with_feature(dev, VIRTIO_NET_F_CSUM)) {
 570                 ifp->if_capabilities |= IFCAP_TXCSUM;
 571
 572                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4))
 573                         ifp->if_capabilities |= IFCAP_TSO4;
 574                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
 575                         ifp->if_capabilities |= IFCAP_TSO6;
 576                 if (ifp->if_capabilities & IFCAP_TSO)
 577                         ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
 578
 579                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN))
 580                         sc->vtnet_flags |= VTNET_FLAG_TSO_ECN;
 581         }
 582
 583         if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_CSUM)) {
 584                 ifp->if_capabilities |= IFCAP_RXCSUM;
 585
 586                 if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) ||
 587                     virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6))
 588                         ifp->if_capabilities |= IFCAP_LRO;
 589         }
 590
 591         if (ifp->if_capabilities & IFCAP_HWCSUM) {
 592                 /*
 593                  * VirtIO does not support VLAN tagging, but we can fake
 594                  * it by inserting and removing the 802.1Q header during
 595                  * transmit and receive. We are then able to do checksum
 596                  * offloading of VLAN frames.
 597                  */
 598                 ifp->if_capabilities |=
 599                         IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM;
 600         }
 601
 602         ifp->if_capenable = ifp->if_capabilities;
 603
 604         /*
 605          * Capabilities after here are not enabled by default.
 606          */
 607
 608         if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) {
 609                 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
 610
 611                 sc->vtnet_vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 612                     vtnet_register_vlan, sc, EVENTHANDLER_PRI_FIRST);
 613                 sc->vtnet_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 614                     vtnet_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST);
 615         }
 616
 617         TASK_INIT(&sc->vtnet_cfgchg_task, 0, vtnet_config_change_task, sc);
 618
 619         error = virtio_setup_intr(dev, &sc->vtnet_slz);
 620         if (error) {
 621                 device_printf(dev, "cannot setup virtqueue interrupts\n");
 622                 ether_ifdetach(ifp);
 623                 goto fail;
 624         }
 625
 626         /*
 627          * Device defaults to promiscuous mode for backwards
 628          * compatibility. Turn it off if possible.
 629          */
 630         if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
 631                 lwkt_serialize_enter(&sc->vtnet_slz);
 632                 if (vtnet_set_promisc(sc, 0) != 0) {
 633                         ifp->if_flags |= IFF_PROMISC;
 634                         device_printf(dev,
 635                             "cannot disable promiscuous mode\n");
 636                 }
 637                 lwkt_serialize_exit(&sc->vtnet_slz);
 638         } else
 639                 ifp->if_flags |= IFF_PROMISC;
 640
 641 fail:
 642         if (error)
 643                 vtnet_detach(dev);
 644
 645         return (error);
 646 }
 647
 648 static int
 649 vtnet_detach(device_t dev)
 650 {
 651         struct vtnet_softc *sc;
 652         struct ifnet *ifp;
 653
 654         sc = device_get_softc(dev);
 655         ifp = sc->vtnet_ifp;
 656
 657         if (device_is_attached(dev)) {
 658                 lwkt_serialize_enter(&sc->vtnet_slz);
 659                 vtnet_stop(sc);
 660                 lwkt_serialize_exit(&sc->vtnet_slz);
 661
 662                 callout_stop(&sc->vtnet_tick_ch);
 663                 taskqueue_drain(taskqueue_swi, &sc->vtnet_cfgchg_task);
 664
 665                 ether_ifdetach(ifp);
 666         }
 667
 668         if (sc->vtnet_vlan_attach != NULL) {
 669                 EVENTHANDLER_DEREGISTER(vlan_config, sc->vtnet_vlan_attach);
 670                 sc->vtnet_vlan_attach = NULL;
 671         }
 672         if (sc->vtnet_vlan_detach != NULL) {
 673                 EVENTHANDLER_DEREGISTER(vlan_unconfg, sc->vtnet_vlan_detach);
 674                 sc->vtnet_vlan_detach = NULL;
 675         }
 676
 677         if (ifp) {
 678                 if_free(ifp);
 679                 sc->vtnet_ifp = NULL;
 680         }
 681
 682         if (sc->vtnet_rx_vq != NULL)
 683                 vtnet_free_rx_mbufs(sc);
 684         if (sc->vtnet_tx_vq != NULL)
 685                 vtnet_free_tx_mbufs(sc);
 686         if (sc->vtnet_ctrl_vq != NULL)
 687                 vtnet_free_ctrl_vq(sc);
 688
 689         if (sc->vtnet_txhdrarea != NULL) {
 690                 contigfree(sc->vtnet_txhdrarea,
 691                     ((sc->vtnet_tx_size / 2) + 1) *
 692                     sizeof(struct vtnet_tx_header), M_VTNET);
 693                 sc->vtnet_txhdrarea = NULL;
 694         }
 695         if (sc->vtnet_macfilter != NULL) {
 696                 contigfree(sc->vtnet_macfilter,
 697                     sizeof(struct vtnet_mac_filter), M_VTNET);
 698                 sc->vtnet_macfilter = NULL;
 699         }
 700
 701         ifmedia_removeall(&sc->vtnet_media);
 702
 703         return (0);
 704 }
 705
 706 static int
 707 vtnet_suspend(device_t dev)
 708 {
 709         struct vtnet_softc *sc;
 710
 711         sc = device_get_softc(dev);
 712
 713         lwkt_serialize_enter(&sc->vtnet_slz);
 714         vtnet_stop(sc);
 715         sc->vtnet_flags |= VTNET_FLAG_SUSPENDED;
 716         lwkt_serialize_exit(&sc->vtnet_slz);
 717
 718         return (0);
 719 }
 720
 721 static int
 722 vtnet_resume(device_t dev)
 723 {
 724         struct vtnet_softc *sc;
 725         struct ifnet *ifp;
 726
 727         sc = device_get_softc(dev);
 728         ifp = sc->vtnet_ifp;
 729
 730         lwkt_serialize_enter(&sc->vtnet_slz);
 731         if (ifp->if_flags & IFF_UP)
 732                 vtnet_init_locked(sc);
 733         sc->vtnet_flags &= ~VTNET_FLAG_SUSPENDED;
 734         lwkt_serialize_exit(&sc->vtnet_slz);
 735
 736         return (0);
 737 }
 738
 739 static int
 740 vtnet_shutdown(device_t dev)
 741 {
 742
 743         /*
 744          * Suspend already does all of what we need to
 745          * do here; we just never expect to be resumed.
 746          */
 747         return (vtnet_suspend(dev));
 748 }
 749
 750 static int
 751 vtnet_config_change(device_t dev)
 752 {
 753         struct vtnet_softc *sc;
 754
 755         sc = device_get_softc(dev);
 756
 757         taskqueue_enqueue(taskqueue_thread[mycpuid], &sc->vtnet_cfgchg_task);
 758
 759         return (1);
 760 }
 761
 762 static void
 763 vtnet_negotiate_features(struct vtnet_softc *sc)
 764 {
 765         device_t dev;
 766         uint64_t mask, features;
 767
 768         dev = sc->vtnet_dev;
 769         mask = 0;
 770
 771         if (vtnet_csum_disable)
 772                 mask |= VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM;
 773
 774         /*
 775          * TSO and LRO are only available when their corresponding
 776          * checksum offload feature is also negotiated.
 777          */
 778
 779         if (vtnet_csum_disable || vtnet_tso_disable)
 780                 mask |= VIRTIO_NET_F_HOST_TSO4 | VIRTIO_NET_F_HOST_TSO6 |
 781                     VIRTIO_NET_F_HOST_ECN;
 782
 783         if (vtnet_csum_disable || vtnet_lro_disable)
 784                 mask |= VTNET_LRO_FEATURES;
 785
 786         features = VTNET_FEATURES & ~mask;
 787         features |= VIRTIO_F_NOTIFY_ON_EMPTY;
 788         sc->vtnet_features = virtio_negotiate_features(dev, features);
 789 }
 790
 791 static int
 792 vtnet_alloc_virtqueues(struct vtnet_softc *sc)
 793 {
 794         device_t dev;
 795         struct vq_alloc_info vq_info[3];
 796         int nvqs, rxsegs;
 797
 798         dev = sc->vtnet_dev;
 799         nvqs = 2;
 800
 801         /*
 802          * Indirect descriptors are not needed for the Rx
 803          * virtqueue when mergeable buffers are negotiated.
 804          * The header is placed inline with the data, not
 805          * in a separate descriptor, and mbuf clusters are
 806          * always physically contiguous.
 807          */
 808         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
 809                 rxsegs = sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG ?
 810                     VTNET_MAX_RX_SEGS : VTNET_MIN_RX_SEGS;
 811         } else
 812                 rxsegs = 0;
 813
 814         VQ_ALLOC_INFO_INIT(&vq_info[0], rxsegs,
 815             vtnet_rx_vq_intr, sc, &sc->vtnet_rx_vq,
 816             "%s receive", device_get_nameunit(dev));
 817
 818         VQ_ALLOC_INFO_INIT(&vq_info[1], VTNET_MAX_TX_SEGS,
 819             vtnet_tx_vq_intr, sc, &sc->vtnet_tx_vq,
 820             "%s transmit", device_get_nameunit(dev));
 821
 822         if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
 823                 nvqs++;
 824
 825                 VQ_ALLOC_INFO_INIT(&vq_info[2], 0, NULL, NULL,
 826                     &sc->vtnet_ctrl_vq, "%s control",
 827                     device_get_nameunit(dev));
 828         }
 829
 830         return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
 831 }
 832
 833 static void
 834 vtnet_get_hwaddr(struct vtnet_softc *sc)
 835 {
 836         device_t dev;
 837
 838         dev = sc->vtnet_dev;
 839
 840         if (virtio_with_feature(dev, VIRTIO_NET_F_MAC)) {
 841                 virtio_read_device_config(dev,
 842                     offsetof(struct virtio_net_config, mac),
 843                     sc->vtnet_hwaddr, ETHER_ADDR_LEN);
 844         } else {
 845                 /* Generate random locally administered unicast address. */
 846                 sc->vtnet_hwaddr[0] = 0xB2;
 847                 karc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1);
 848
 849                 vtnet_set_hwaddr(sc);
 850         }
 851 }
 852
 853 static void
 854 vtnet_set_hwaddr(struct vtnet_softc *sc)
 855 {
 856         device_t dev;
 857
 858         dev = sc->vtnet_dev;
 859
 860         virtio_write_device_config(dev,
 861             offsetof(struct virtio_net_config, mac),
 862             sc->vtnet_hwaddr, ETHER_ADDR_LEN);
 863 }
 864
 865 static int
 866 vtnet_is_link_up(struct vtnet_softc *sc)
 867 {
 868         device_t dev;
 869         struct ifnet *ifp;
 870         uint16_t status;
 871
 872         dev = sc->vtnet_dev;
 873         ifp = sc->vtnet_ifp;
 874
 875         ASSERT_SERIALIZED(&sc->vtnet_slz);
 876
 877         status = virtio_read_dev_config_2(dev,
 878                         offsetof(struct virtio_net_config, status));
 879
 880         return ((status & VIRTIO_NET_S_LINK_UP) != 0);
 881 }
 882
 883 static void
 884 vtnet_update_link_status(struct vtnet_softc *sc)
 885 {
 886         device_t dev;
 887         struct ifnet *ifp;
 888         struct ifaltq_subque *ifsq;
 889         int link;
 890
 891         dev = sc->vtnet_dev;
 892         ifp = sc->vtnet_ifp;
 893         ifsq = ifq_get_subq_default(&ifp->if_snd);
 894
 895         link = vtnet_is_link_up(sc);
 896
 897         if (link && ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0)) {
 898                 sc->vtnet_flags |= VTNET_FLAG_LINK;
 899                 if (bootverbose)
 900                         device_printf(dev, "Link is up\n");
 901                 ifp->if_link_state = LINK_STATE_UP;
 902                 if_link_state_change(ifp);
 903                 if (!ifsq_is_empty(ifsq))
 904                         vtnet_start_locked(ifp, ifsq);
 905         } else if (!link && (sc->vtnet_flags & VTNET_FLAG_LINK)) {
 906                 sc->vtnet_flags &= ~VTNET_FLAG_LINK;
 907                 if (bootverbose)
 908                         device_printf(dev, "Link is down\n");
 909
 910                 ifp->if_link_state = LINK_STATE_DOWN;
 911                 if_link_state_change(ifp);
 912         }
 913 }
 914
 915 #if 0
 916 static void
 917 vtnet_watchdog(struct vtnet_softc *sc)
 918 {
 919         struct ifnet *ifp;
 920
 921         ifp = sc->vtnet_ifp;
 922
 923 #ifdef VTNET_TX_INTR_MODERATION
 924         vtnet_txeof(sc);
 925 #endif
 926
 927         if (sc->vtnet_watchdog_timer == 0 || --sc->vtnet_watchdog_timer)
 928                 return;
 929
 930         if_printf(ifp, "watchdog timeout -- resetting\n");
 931 #ifdef VTNET_DEBUG
 932         virtqueue_dump(sc->vtnet_tx_vq);
 933 #endif
 934         ifp->if_oerrors++;
 935         ifp->if_flags &= ~IFF_RUNNING;
 936         vtnet_init_locked(sc);
 937 }
 938 #endif
 939
 940 static void
 941 vtnet_config_change_task(void *arg, int pending)
 942 {
 943         struct vtnet_softc *sc;
 944
 945         sc = arg;
 946
 947         lwkt_serialize_enter(&sc->vtnet_slz);
 948         vtnet_update_link_status(sc);
 949         lwkt_serialize_exit(&sc->vtnet_slz);
 950 }
 951
 952 static int
 953 vtnet_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data,struct ucred *cr)
 954 {
 955         struct vtnet_softc *sc;
 956         struct ifreq *ifr;
 957         int reinit, mask, error;
 958
 959         sc = ifp->if_softc;
 960         ifr = (struct ifreq *) data;
 961         reinit = 0;
 962         error = 0;
 963
 964         switch (cmd) {
 965         case SIOCSIFMTU:
 966                 if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > VTNET_MAX_MTU)
 967                         error = EINVAL;
 968                 else if (ifp->if_mtu != ifr->ifr_mtu) {
 969                         lwkt_serialize_enter(&sc->vtnet_slz);
 970                         error = vtnet_change_mtu(sc, ifr->ifr_mtu);
 971                         lwkt_serialize_exit(&sc->vtnet_slz);
 972                 }
 973                 break;
 974
 975         case SIOCSIFFLAGS:
 976                 lwkt_serialize_enter(&sc->vtnet_slz);
 977                 if ((ifp->if_flags & IFF_UP) == 0) {
 978                         if (ifp->if_flags & IFF_RUNNING)
 979                                 vtnet_stop(sc);
 980                 } else if (ifp->if_flags & IFF_RUNNING) {
 981                         if ((ifp->if_flags ^ sc->vtnet_if_flags) &
 982                             (IFF_PROMISC | IFF_ALLMULTI)) {
 983                                 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX)
 984                                         vtnet_rx_filter(sc);
 985                                 else
 986                                         error = ENOTSUP;
 987                         }
 988                 } else
 989                         vtnet_init_locked(sc);
 990
 991                 if (error == 0)
 992                         sc->vtnet_if_flags = ifp->if_flags;
 993                 lwkt_serialize_exit(&sc->vtnet_slz);
 994                 break;
 995
 996         case SIOCADDMULTI:
 997         case SIOCDELMULTI:
 998                 lwkt_serialize_enter(&sc->vtnet_slz);
 999                 if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) &&
1000                     (ifp->if_flags & IFF_RUNNING))
1001                         vtnet_rx_filter_mac(sc);
1002                 lwkt_serialize_exit(&sc->vtnet_slz);
1003                 break;
1004
1005         case SIOCSIFMEDIA:
1006         case SIOCGIFMEDIA:
1007                 error = ifmedia_ioctl(ifp, ifr, &sc->vtnet_media, cmd);
1008                 break;
1009
1010         case SIOCSIFCAP:
1011                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1012
1013                 lwkt_serialize_enter(&sc->vtnet_slz);
1014
1015                 if (mask & IFCAP_TXCSUM) {
1016                         ifp->if_capenable ^= IFCAP_TXCSUM;
1017                         if (ifp->if_capenable & IFCAP_TXCSUM)
1018                                 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;
1019                         else
1020                                 ifp->if_hwassist &= ~VTNET_CSUM_OFFLOAD;
1021                 }
1022
1023                 if (mask & IFCAP_TSO4) {
1024                         ifp->if_capenable ^= IFCAP_TSO4;
1025                         if (ifp->if_capenable & IFCAP_TSO4)
1026                                 ifp->if_hwassist |= CSUM_TSO;
1027                         else
1028                                 ifp->if_hwassist &= ~CSUM_TSO;
1029                 }
1030
1031                 if (mask & IFCAP_RXCSUM) {
1032                         ifp->if_capenable ^= IFCAP_RXCSUM;
1033                         reinit = 1;
1034                 }
1035
1036                 if (mask & IFCAP_LRO) {
1037                         ifp->if_capenable ^= IFCAP_LRO;
1038                         reinit = 1;
1039                 }
1040
1041                 if (mask & IFCAP_VLAN_HWFILTER) {
1042                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1043                         reinit = 1;
1044                 }
1045
1046                 if (mask & IFCAP_VLAN_HWTSO)
1047                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1048
1049                 if (mask & IFCAP_VLAN_HWTAGGING)
1050                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1051
1052                 if (reinit && (ifp->if_flags & IFF_RUNNING)) {
1053                         ifp->if_flags &= ~IFF_RUNNING;
1054                         vtnet_init_locked(sc);
1055                 }
1056                 //VLAN_CAPABILITIES(ifp);
1057
1058                 lwkt_serialize_exit(&sc->vtnet_slz);
1059                 break;
1060
1061         default:
1062                 error = ether_ioctl(ifp, cmd, data);
1063                 break;
1064         }
1065
1066         return (error);
1067 }
1068
1069 static int
1070 vtnet_change_mtu(struct vtnet_softc *sc, int new_mtu)
1071 {
1072         struct ifnet *ifp;
1073         int new_frame_size, clsize;
1074
1075         ifp = sc->vtnet_ifp;
1076
1077         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1078                 new_frame_size = sizeof(struct vtnet_rx_header) +
1079                     sizeof(struct ether_vlan_header) + new_mtu;
1080
1081                 if (new_frame_size > MJUM9BYTES)
1082                         return (EINVAL);
1083
1084                 if (new_frame_size <= MCLBYTES)
1085                         clsize = MCLBYTES;
1086                 else
1087                         clsize = MJUM9BYTES;
1088         } else {
1089                 new_frame_size = sizeof(struct virtio_net_hdr_mrg_rxbuf) +
1090                     sizeof(struct ether_vlan_header) + new_mtu;
1091
1092                 if (new_frame_size <= MCLBYTES)
1093                         clsize = MCLBYTES;
1094                 else
1095                         clsize = MJUMPAGESIZE;
1096         }
1097
1098         sc->vtnet_rx_mbuf_size = clsize;
1099         sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc);
1100         KASSERT(sc->vtnet_rx_mbuf_count < VTNET_MAX_RX_SEGS,
1101             ("too many rx mbufs: %d", sc->vtnet_rx_mbuf_count));
1102
1103         ifp->if_mtu = new_mtu;
1104
1105         if (ifp->if_flags & IFF_RUNNING) {
1106                 ifp->if_flags &= ~IFF_RUNNING;
1107                 vtnet_init_locked(sc);
1108         }
1109
1110         return (0);
1111 }
1112
1113 static int
1114 vtnet_init_rx_vq(struct vtnet_softc *sc)
1115 {
1116         struct virtqueue *vq;
1117         int nbufs, error;
1118
1119         vq = sc->vtnet_rx_vq;
1120         nbufs = 0;
1121         error = ENOSPC;
1122
1123         while (!virtqueue_full(vq)) {
1124                 if ((error = vtnet_newbuf(sc)) != 0)
1125                         break;
1126                 nbufs++;
1127         }
1128
1129         if (nbufs > 0) {
1130                 virtqueue_notify(vq, &sc->vtnet_slz);
1131
1132                 /*
1133                  * EMSGSIZE signifies the virtqueue did not have enough
1134                  * entries available to hold the last mbuf. This is not
1135                  * an error. We should not get ENOSPC since we check if
1136                  * the virtqueue is full before attempting to add a
1137                  * buffer.
1138                  */
1139                 if (error == EMSGSIZE)
1140                         error = 0;
1141         }
1142
1143         return (error);
1144 }
1145
1146 static void
1147 vtnet_free_rx_mbufs(struct vtnet_softc *sc)
1148 {
1149         struct virtqueue *vq;
1150         struct mbuf *m;
1151         int last;
1152
1153         vq = sc->vtnet_rx_vq;
1154         last = 0;
1155
1156         while ((m = virtqueue_drain(vq, &last)) != NULL)
1157                 m_freem(m);
1158
1159         KASSERT(virtqueue_empty(vq), ("mbufs remaining in Rx Vq"));
1160 }
1161
1162 static void
1163 vtnet_free_tx_mbufs(struct vtnet_softc *sc)
1164 {
1165         struct virtqueue *vq;
1166         struct vtnet_tx_header *txhdr;
1167         int last;
1168
1169         vq = sc->vtnet_tx_vq;
1170         last = 0;
1171
1172         while ((txhdr = virtqueue_drain(vq, &last)) != NULL) {
1173                 m_freem(txhdr->vth_mbuf);
1174         }
1175
1176         KASSERT(virtqueue_empty(vq), ("mbufs remaining in Tx Vq"));
1177 }
1178
1179 static void
1180 vtnet_free_ctrl_vq(struct vtnet_softc *sc)
1181 {
1182         /*
1183          * The control virtqueue is only polled, therefore
1184          * it should already be empty.
1185          */
1186         KASSERT(virtqueue_empty(sc->vtnet_ctrl_vq),
1187                 ("Ctrl Vq not empty"));
1188 }
1189
1190 static struct mbuf *
1191 vtnet_alloc_rxbuf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp)
1192 {
1193         struct mbuf *m_head, *m_tail, *m;
1194         int i, clsize;
1195
1196         clsize = sc->vtnet_rx_mbuf_size;
1197
1198         /*use getcl instead of getjcl. see  if_mxge.c comment line 2398*/
1199         //m_head = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, clsize);
1200         m_head = m_getcl(MB_DONTWAIT, MT_DATA, M_PKTHDR );
1201         if (m_head == NULL)
1202                 goto fail;
1203
1204         m_head->m_len = clsize;
1205         m_tail = m_head;
1206
1207         if (nbufs > 1) {
1208                 KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
1209                         ("chained Rx mbuf requested without LRO_NOMRG"));
1210
1211                 for (i = 0; i < nbufs - 1; i++) {
1212                         //m = m_getjcl(M_DONTWAIT, MT_DATA, 0, clsize);
1213                         m = m_getcl(MB_DONTWAIT, MT_DATA, 0);
1214                         if (m == NULL)
1215                                 goto fail;
1216
1217                         m->m_len = clsize;
1218                         m_tail->m_next = m;
1219                         m_tail = m;
1220                 }
1221         }
1222
1223         if (m_tailp != NULL)
1224                 *m_tailp = m_tail;
1225
1226         return (m_head);
1227
1228 fail:
1229         sc->vtnet_stats.mbuf_alloc_failed++;
1230         m_freem(m_head);
1231
1232         return (NULL);
1233 }
1234
1235 static int
1236 vtnet_replace_rxbuf(struct vtnet_softc *sc, struct mbuf *m0, int len0)
1237 {
1238         struct mbuf *m, *m_prev;
1239         struct mbuf *m_new, *m_tail;
1240         int len, clsize, nreplace, error;
1241
1242         m = m0;
1243         m_prev = NULL;
1244         len = len0;
1245
1246         m_tail = NULL;
1247         clsize = sc->vtnet_rx_mbuf_size;
1248         nreplace = 0;
1249
1250         if (m->m_next != NULL)
1251                 KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
1252                     ("chained Rx mbuf without LRO_NOMRG"));
1253
1254         /*
1255          * Since LRO_NOMRG mbuf chains are so large, we want to avoid
1256          * allocating an entire chain for each received frame. When
1257          * the received frame's length is less than that of the chain,
1258          * the unused mbufs are reassigned to the new chain.
1259          */
1260         while (len > 0) {
1261                 /*
1262                  * Something is seriously wrong if we received
1263                  * a frame larger than the mbuf chain. Drop it.
1264                  */
1265                 if (m == NULL) {
1266                         sc->vtnet_stats.rx_frame_too_large++;
1267                         return (EMSGSIZE);
1268                 }
1269
1270                 KASSERT(m->m_len == clsize,
1271                     ("mbuf length not expected cluster size: %d",
1272                     m->m_len));
1273
1274                 m->m_len = MIN(m->m_len, len);
1275                 len -= m->m_len;
1276
1277                 m_prev = m;
1278                 m = m->m_next;
1279                 nreplace++;
1280         }
1281
1282         KASSERT(m_prev != NULL, ("m_prev == NULL"));
1283         KASSERT(nreplace <= sc->vtnet_rx_mbuf_count,
1284                 ("too many replacement mbufs: %d/%d", nreplace,
1285                 sc->vtnet_rx_mbuf_count));
1286
1287         m_new = vtnet_alloc_rxbuf(sc, nreplace, &m_tail);
1288         if (m_new == NULL) {
1289                 m_prev->m_len = clsize;
1290                 return (ENOBUFS);
1291         }
1292
1293         /*
1294          * Move unused mbufs, if any, from the original chain
1295          * onto the end of the new chain.
1296          */
1297         if (m_prev->m_next != NULL) {
1298                 m_tail->m_next = m_prev->m_next;
1299                 m_prev->m_next = NULL;
1300         }
1301
1302         error = vtnet_enqueue_rxbuf(sc, m_new);
1303         if (error) {
1304                 /*
1305                  * BAD! We could not enqueue the replacement mbuf chain. We
1306                  * must restore the m0 chain to the original state if it was
1307                  * modified so we can subsequently discard it.
1308                  *
1309                  * NOTE: The replacement is suppose to be an identical copy
1310                  * to the one just dequeued so this is an unexpected error.
1311                  */
1312                 sc->vtnet_stats.rx_enq_replacement_failed++;
1313
1314                 if (m_tail->m_next != NULL) {
1315                         m_prev->m_next = m_tail->m_next;
1316                         m_tail->m_next = NULL;
1317                 }
1318
1319                 m_prev->m_len = clsize;
1320                 m_freem(m_new);
1321         }
1322
1323         return (error);
1324 }
1325
1326 static int
1327 vtnet_newbuf(struct vtnet_softc *sc)
1328 {
1329         struct mbuf *m;
1330         int error;
1331
1332         m = vtnet_alloc_rxbuf(sc, sc->vtnet_rx_mbuf_count, NULL);
1333         if (m == NULL)
1334                 return (ENOBUFS);
1335
1336         error = vtnet_enqueue_rxbuf(sc, m);
1337         if (error)
1338                 m_freem(m);
1339
1340         return (error);
1341 }
1342
1343 static void
1344 vtnet_discard_merged_rxbuf(struct vtnet_softc *sc, int nbufs)
1345 {
1346         struct virtqueue *vq;
1347         struct mbuf *m;
1348
1349         vq = sc->vtnet_rx_vq;
1350
1351         while (--nbufs > 0) {
1352                 if ((m = virtqueue_dequeue(vq, NULL)) == NULL)
1353                         break;
1354                 vtnet_discard_rxbuf(sc, m);
1355         }
1356 }
1357
1358 static void
1359 vtnet_discard_rxbuf(struct vtnet_softc *sc, struct mbuf *m)
1360 {
1361         int error;
1362
1363         /*
1364          * Requeue the discarded mbuf. This should always be
1365          * successful since it was just dequeued.
1366          */
1367         error = vtnet_enqueue_rxbuf(sc, m);
1368         KASSERT(error == 0, ("cannot requeue discarded mbuf"));
1369 }
1370
1371 static int
1372 vtnet_enqueue_rxbuf(struct vtnet_softc *sc, struct mbuf *m)
1373 {
1374         struct sglist sg;
1375         struct sglist_seg segs[VTNET_MAX_RX_SEGS];
1376         struct vtnet_rx_header *rxhdr;
1377         struct virtio_net_hdr *hdr;
1378         uint8_t *mdata;
1379         int offset, error;
1380
1381         ASSERT_SERIALIZED(&sc->vtnet_slz);
1382         if ((sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) == 0)
1383                 KASSERT(m->m_next == NULL, ("chained Rx mbuf"));
1384
1385         sglist_init(&sg, VTNET_MAX_RX_SEGS, segs);
1386
1387         mdata = mtod(m, uint8_t *);
1388         offset = 0;
1389
1390         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1391                 rxhdr = (struct vtnet_rx_header *) mdata;
1392                 hdr = &rxhdr->vrh_hdr;
1393                 offset += sizeof(struct vtnet_rx_header);
1394
1395                 error = sglist_append(&sg, hdr, sc->vtnet_hdr_size);
1396                 KASSERT(error == 0, ("cannot add header to sglist"));
1397         }
1398
1399         error = sglist_append(&sg, mdata + offset, m->m_len - offset);
1400         if (error)
1401                 return (error);
1402
1403         if (m->m_next != NULL) {
1404                 error = sglist_append_mbuf(&sg, m->m_next);
1405                 if (error)
1406                         return (error);
1407         }
1408
1409         return (virtqueue_enqueue(sc->vtnet_rx_vq, m, &sg, 0, sg.sg_nseg));
1410 }
1411
1412 static void
1413 vtnet_vlan_tag_remove(struct mbuf *m)
1414 {
1415         struct ether_vlan_header *evl;
1416
1417         evl = mtod(m, struct ether_vlan_header *);
1418
1419         m->m_pkthdr.ether_vlantag = ntohs(evl->evl_tag);
1420         m->m_flags |= M_VLANTAG;
1421
1422         /* Strip the 802.1Q header. */
1423         bcopy((char *) evl, (char *) evl + ETHER_VLAN_ENCAP_LEN,
1424             ETHER_HDR_LEN - ETHER_TYPE_LEN);
1425         m_adj(m, ETHER_VLAN_ENCAP_LEN);
1426 }
1427
1428 /*
1429  * Alternative method of doing receive checksum offloading. Rather
1430  * than parsing the received frame down to the IP header, use the
1431  * csum_offset to determine which CSUM_* flags are appropriate. We
1432  * can get by with doing this only because the checksum offsets are
1433  * unique for the things we care about.
1434  */
1435 static int
1436 vtnet_rx_csum(struct vtnet_softc *sc, struct mbuf *m,
1437     struct virtio_net_hdr *hdr)
1438 {
1439         struct ether_header *eh;
1440         struct ether_vlan_header *evh;
1441         struct udphdr *udp;
1442         int csum_len;
1443         uint16_t eth_type;
1444
1445         csum_len = hdr->csum_start + hdr->csum_offset;
1446
1447         if (csum_len < sizeof(struct ether_header) + sizeof(struct ip))
1448                 return (1);
1449         if (m->m_len < csum_len)
1450                 return (1);
1451
1452         eh = mtod(m, struct ether_header *);
1453         eth_type = ntohs(eh->ether_type);
1454         if (eth_type == ETHERTYPE_VLAN) {
1455                 evh = mtod(m, struct ether_vlan_header *);
1456                 eth_type = ntohs(evh->evl_proto);
1457         }
1458
1459         if (eth_type != ETHERTYPE_IP && eth_type != ETHERTYPE_IPV6) {
1460                 sc->vtnet_stats.rx_csum_bad_ethtype++;
1461                 return (1);
1462         }
1463
1464         /* Use the offset to determine the appropriate CSUM_* flags. */
1465         switch (hdr->csum_offset) {
1466         case offsetof(struct udphdr, uh_sum):
1467                 if (m->m_len < hdr->csum_start + sizeof(struct udphdr))
1468                         return (1);
1469                 udp = (struct udphdr *)(mtod(m, uint8_t *) + hdr->csum_start);
1470                 if (udp->uh_sum == 0)
1471                         return (0);
1472
1473                 /* FALLTHROUGH */
1474
1475         case offsetof(struct tcphdr, th_sum):
1476                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1477                 m->m_pkthdr.csum_data = 0xFFFF;
1478                 break;
1479
1480         case offsetof(struct sctphdr, checksum):
1481                 //m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID;
1482                 break;
1483
1484         default:
1485                 sc->vtnet_stats.rx_csum_bad_offset++;
1486                 return (1);
1487         }
1488
1489         sc->vtnet_stats.rx_csum_offloaded++;
1490
1491         return (0);
1492 }
1493
1494 static int
1495 vtnet_rxeof_merged(struct vtnet_softc *sc, struct mbuf *m_head, int nbufs)
1496 {
1497         struct ifnet *ifp;
1498         struct virtqueue *vq;
1499         struct mbuf *m, *m_tail;
1500         int len;
1501
1502         ifp = sc->vtnet_ifp;
1503         vq = sc->vtnet_rx_vq;
1504         m_tail = m_head;
1505
1506         while (--nbufs > 0) {
1507                 m = virtqueue_dequeue(vq, &len);
1508                 if (m == NULL) {
1509                         ifp->if_ierrors++;
1510                         goto fail;
1511                 }
1512
1513                 if (vtnet_newbuf(sc) != 0) {
1514                         ifp->if_iqdrops++;
1515                         vtnet_discard_rxbuf(sc, m);
1516                         if (nbufs > 1)
1517                                 vtnet_discard_merged_rxbuf(sc, nbufs);
1518                         goto fail;
1519                 }
1520
1521                 if (m->m_len < len)
1522                         len = m->m_len;
1523
1524                 m->m_len = len;
1525                 m->m_flags &= ~M_PKTHDR;
1526
1527                 m_head->m_pkthdr.len += len;
1528                 m_tail->m_next = m;
1529                 m_tail = m;
1530         }
1531
1532         return (0);
1533
1534 fail:
1535         sc->vtnet_stats.rx_mergeable_failed++;
1536         m_freem(m_head);
1537
1538         return (1);
1539 }
1540
1541 static int
1542 vtnet_rxeof(struct vtnet_softc *sc, int count, int *rx_npktsp)
1543 {
1544         struct virtio_net_hdr lhdr;
1545         struct ifnet *ifp;
1546         struct virtqueue *vq;
1547         struct mbuf *m;
1548         struct ether_header *eh;
1549         struct virtio_net_hdr *hdr;
1550         struct virtio_net_hdr_mrg_rxbuf *mhdr;
1551         int len, deq, nbufs, adjsz, rx_npkts;
1552
1553         ifp = sc->vtnet_ifp;
1554         vq = sc->vtnet_rx_vq;
1555         hdr = &lhdr;
1556         deq = 0;
1557         rx_npkts = 0;
1558
1559         ASSERT_SERIALIZED(&sc->vtnet_slz);
1560
1561         while (--count >= 0) {
1562                 m = virtqueue_dequeue(vq, &len);
1563                 if (m == NULL)
1564                         break;
1565                 deq++;
1566
1567                 if (len < sc->vtnet_hdr_size + ETHER_HDR_LEN) {
1568                         ifp->if_ierrors++;
1569                         vtnet_discard_rxbuf(sc, m);
1570                         continue;
1571                 }
1572
1573                 if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1574                         nbufs = 1;
1575                         adjsz = sizeof(struct vtnet_rx_header);
1576                         /*
1577                          * Account for our pad between the header and
1578                          * the actual start of the frame.
1579                          */
1580                         len += VTNET_RX_HEADER_PAD;
1581                 } else {
1582                         mhdr = mtod(m, struct virtio_net_hdr_mrg_rxbuf *);
1583                         nbufs = mhdr->num_buffers;
1584                         adjsz = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1585                 }
1586
1587                 if (vtnet_replace_rxbuf(sc, m, len) != 0) {
1588                         ifp->if_iqdrops++;
1589                         vtnet_discard_rxbuf(sc, m);
1590                         if (nbufs > 1)
1591                                 vtnet_discard_merged_rxbuf(sc, nbufs);
1592                         continue;
1593                 }
1594
1595                 m->m_pkthdr.len = len;
1596                 m->m_pkthdr.rcvif = ifp;
1597                 m->m_pkthdr.csum_flags = 0;
1598
1599                 if (nbufs > 1) {
1600                         if (vtnet_rxeof_merged(sc, m, nbufs) != 0)
1601                                 continue;
1602                 }
1603
1604                 ifp->if_ipackets++;
1605
1606                 /*
1607                  * Save copy of header before we strip it. For both mergeable
1608                  * and non-mergeable, the VirtIO header is placed first in the
1609                  * mbuf's data. We no longer need num_buffers, so always use a
1610                  * virtio_net_hdr.
1611                  */
1612                 memcpy(hdr, mtod(m, void *), sizeof(struct virtio_net_hdr));
1613                 m_adj(m, adjsz);
1614
1615                 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1616                         eh = mtod(m, struct ether_header *);
1617                         if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
1618                                 vtnet_vlan_tag_remove(m);
1619
1620                                 /*
1621                                  * With the 802.1Q header removed, update the
1622                                  * checksum starting location accordingly.
1623                                  */
1624                                 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
1625                                         hdr->csum_start -=
1626                                             ETHER_VLAN_ENCAP_LEN;
1627                         }
1628                 }
1629
1630                 if (ifp->if_capenable & IFCAP_RXCSUM &&
1631                     hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
1632                         if (vtnet_rx_csum(sc, m, hdr) != 0)
1633                                 sc->vtnet_stats.rx_csum_failed++;
1634                 }
1635
1636                 lwkt_serialize_exit(&sc->vtnet_slz);
1637                 rx_npkts++;
1638                 (*ifp->if_input)(ifp, m);
1639                 lwkt_serialize_enter(&sc->vtnet_slz);
1640
1641                 /*
1642                  * The interface may have been stopped while we were
1643                  * passing the packet up the network stack.
1644                  */
1645                 if ((ifp->if_flags & IFF_RUNNING) == 0)
1646                         break;
1647         }
1648
1649         virtqueue_notify(vq, &sc->vtnet_slz);
1650
1651         if (rx_npktsp != NULL)
1652                 *rx_npktsp = rx_npkts;
1653
1654         return (count > 0 ? 0 : EAGAIN);
1655 }
1656
1657 static void
1658 vtnet_rx_intr_task(void *arg)
1659 {
1660         struct vtnet_softc *sc;
1661         struct ifnet *ifp;
1662         int more;
1663
1664         sc = arg;
1665         ifp = sc->vtnet_ifp;
1666
1667 //      lwkt_serialize_enter(&sc->vtnet_slz);
1668
1669         if ((ifp->if_flags & IFF_RUNNING) == 0) {
1670                 vtnet_enable_rx_intr(sc);
1671 //              lwkt_serialize_exit(&sc->vtnet_slz);
1672                 return;
1673         }
1674
1675         more = vtnet_rxeof(sc, sc->vtnet_rx_process_limit, NULL);
1676         if (!more && vtnet_enable_rx_intr(sc) != 0) {
1677                 vtnet_disable_rx_intr(sc);
1678                 more = 1;
1679         }
1680
1681 //      lwkt_serialize_exit(&sc->vtnet_slz);
1682
1683         if (more) {
1684                 sc->vtnet_stats.rx_task_rescheduled++;
1685                 /* XXX: Loop?! */
1686                 vtnet_rx_intr_task(sc);
1687         }
1688 }
1689
1690 static int
1691 vtnet_rx_vq_intr(void *xsc)
1692 {
1693         struct vtnet_softc *sc;
1694
1695         sc = xsc;
1696
1697         vtnet_disable_rx_intr(sc);
1698         vtnet_rx_intr_task(sc);
1699
1700         return (1);
1701 }
1702
1703 static void
1704 vtnet_txeof(struct vtnet_softc *sc)
1705 {
1706         struct virtqueue *vq;
1707         struct ifnet *ifp;
1708         struct vtnet_tx_header *txhdr;
1709         int deq;
1710
1711         vq = sc->vtnet_tx_vq;
1712         ifp = sc->vtnet_ifp;
1713         deq = 0;
1714
1715         ASSERT_SERIALIZED(&sc->vtnet_slz);
1716
1717         while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) {
1718                 deq++;
1719                 ifp->if_opackets++;
1720                 m_freem(txhdr->vth_mbuf);
1721         }
1722
1723         if (deq > 0) {
1724                 ifq_clr_oactive(&ifp->if_snd);
1725                 if (virtqueue_empty(vq))
1726                         sc->vtnet_watchdog_timer = 0;
1727         }
1728 }
1729
1730 static struct mbuf *
1731 vtnet_tx_offload(struct vtnet_softc *sc, struct mbuf *m,
1732     struct virtio_net_hdr *hdr)
1733 {
1734         struct ifnet *ifp;
1735         struct ether_header *eh;
1736         struct ether_vlan_header *evh;
1737         struct ip *ip;
1738         struct ip6_hdr *ip6;
1739         struct tcphdr *tcp;
1740         int ip_offset;
1741         uint16_t eth_type, csum_start;
1742         uint8_t ip_proto, gso_type;
1743
1744         ifp = sc->vtnet_ifp;
1745         M_ASSERTPKTHDR(m);
1746
1747         ip_offset = sizeof(struct ether_header);
1748         if (m->m_len < ip_offset) {
1749                 if ((m = m_pullup(m, ip_offset)) == NULL)
1750                         return (NULL);
1751         }
1752
1753         eh = mtod(m, struct ether_header *);
1754         eth_type = ntohs(eh->ether_type);
1755         if (eth_type == ETHERTYPE_VLAN) {
1756                 ip_offset = sizeof(struct ether_vlan_header);
1757                 if (m->m_len < ip_offset) {
1758                         if ((m = m_pullup(m, ip_offset)) == NULL)
1759                                 return (NULL);
1760                 }
1761                 evh = mtod(m, struct ether_vlan_header *);
1762                 eth_type = ntohs(evh->evl_proto);
1763         }
1764
1765         switch (eth_type) {
1766         case ETHERTYPE_IP:
1767                 if (m->m_len < ip_offset + sizeof(struct ip)) {
1768                         m = m_pullup(m, ip_offset + sizeof(struct ip));
1769                         if (m == NULL)
1770                                 return (NULL);
1771                 }
1772
1773                 ip = (struct ip *)(mtod(m, uint8_t *) + ip_offset);
1774                 ip_proto = ip->ip_p;
1775                 csum_start = ip_offset + (ip->ip_hl << 2);
1776                 gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
1777                 break;
1778
1779         case ETHERTYPE_IPV6:
1780                 if (m->m_len < ip_offset + sizeof(struct ip6_hdr)) {
1781                         m = m_pullup(m, ip_offset + sizeof(struct ip6_hdr));
1782                         if (m == NULL)
1783                                 return (NULL);
1784                 }
1785
1786                 ip6 = (struct ip6_hdr *)(mtod(m, uint8_t *) + ip_offset);
1787                 /*
1788                  * XXX Assume no extension headers are present. Presently,
1789                  * this will always be true in the case of TSO, and FreeBSD
1790                  * does not perform checksum offloading of IPv6 yet.
1791                  */
1792                 ip_proto = ip6->ip6_nxt;
1793                 csum_start = ip_offset + sizeof(struct ip6_hdr);
1794                 gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
1795                 break;
1796
1797         default:
1798                 return (m);
1799         }
1800
1801         if (m->m_pkthdr.csum_flags & VTNET_CSUM_OFFLOAD) {
1802                 hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM;
1803                 hdr->csum_start = csum_start;
1804                 hdr->csum_offset = m->m_pkthdr.csum_data;
1805
1806                 sc->vtnet_stats.tx_csum_offloaded++;
1807         }
1808
1809         if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1810                 if (ip_proto != IPPROTO_TCP)
1811                         return (m);
1812
1813                 if (m->m_len < csum_start + sizeof(struct tcphdr)) {
1814                         m = m_pullup(m, csum_start + sizeof(struct tcphdr));
1815                         if (m == NULL)
1816                                 return (NULL);
1817                 }
1818
1819                 tcp = (struct tcphdr *)(mtod(m, uint8_t *) + csum_start);
1820                 hdr->gso_type = gso_type;
1821                 hdr->hdr_len = csum_start + (tcp->th_off << 2);
1822                 hdr->gso_size = m->m_pkthdr.tso_segsz;
1823
1824                 if (tcp->th_flags & TH_CWR) {
1825                         /*
1826                          * Drop if we did not negotiate VIRTIO_NET_F_HOST_ECN.
1827                          * ECN support is only configurable globally with the
1828                          * net.inet.tcp.ecn.enable sysctl knob.
1829                          */
1830                         if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) {
1831                                 if_printf(ifp, "TSO with ECN not supported "
1832                                     "by host\n");
1833                                 m_freem(m);
1834                                 return (NULL);
1835                         }
1836
1837                         hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN;
1838                 }
1839
1840                 sc->vtnet_stats.tx_tso_offloaded++;
1841         }
1842
1843         return (m);
1844 }
1845
1846 static int
1847 vtnet_enqueue_txbuf(struct vtnet_softc *sc, struct mbuf **m_head,
1848     struct vtnet_tx_header *txhdr)
1849 {
1850         struct sglist sg;
1851         struct sglist_seg segs[VTNET_MAX_TX_SEGS];
1852         struct virtqueue *vq;
1853         struct mbuf *m;
1854         int collapsed, error;
1855
1856         vq = sc->vtnet_tx_vq;
1857         m = *m_head;
1858         collapsed = 0;
1859
1860         sglist_init(&sg, VTNET_MAX_TX_SEGS, segs);
1861         error = sglist_append(&sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size);
1862         KASSERT(error == 0 && sg.sg_nseg == 1,
1863             ("cannot add header to sglist"));
1864
1865 again:
1866         error = sglist_append_mbuf(&sg, m);
1867         if (error) {
1868                 if (collapsed)
1869                         goto fail;
1870
1871                 //m = m_collapse(m, MB_DONTWAIT, VTNET_MAX_TX_SEGS - 1);
1872                 m = m_defrag(m, MB_DONTWAIT);
1873                 if (m == NULL)
1874                         goto fail;
1875
1876                 *m_head = m;
1877                 collapsed = 1;
1878                 goto again;
1879         }
1880
1881         txhdr->vth_mbuf = m;
1882
1883         return (virtqueue_enqueue(vq, txhdr, &sg, sg.sg_nseg, 0));
1884
1885 fail:
1886         m_freem(*m_head);
1887         *m_head = NULL;
1888
1889         return (ENOBUFS);
1890 }
1891
1892 static struct mbuf *
1893 vtnet_vlan_tag_insert(struct mbuf *m)
1894 {
1895         struct mbuf *n;
1896         struct ether_vlan_header *evl;
1897
1898         if (M_WRITABLE(m) == 0) {
1899                 n = m_dup(m, MB_DONTWAIT);
1900                 m_freem(m);
1901                 if ((m = n) == NULL)
1902                         return (NULL);
1903         }
1904
1905         M_PREPEND(m, ETHER_VLAN_ENCAP_LEN, MB_DONTWAIT);
1906         if (m == NULL)
1907                 return (NULL);
1908         if (m->m_len < sizeof(struct ether_vlan_header)) {
1909                 m = m_pullup(m, sizeof(struct ether_vlan_header));
1910                 if (m == NULL)
1911                         return (NULL);
1912         }
1913
1914         /* Insert 802.1Q header into the existing Ethernet header. */
1915         evl = mtod(m, struct ether_vlan_header *);
1916         bcopy((char *) evl + ETHER_VLAN_ENCAP_LEN,
1917               (char *) evl, ETHER_HDR_LEN - ETHER_TYPE_LEN);
1918         evl->evl_encap_proto = htons(ETHERTYPE_VLAN);
1919         evl->evl_tag = htons(m->m_pkthdr.ether_vlantag);
1920         m->m_flags &= ~M_VLANTAG;
1921
1922         return (m);
1923 }
1924
1925 static int
1926 vtnet_encap(struct vtnet_softc *sc, struct mbuf **m_head)
1927 {
1928         struct vtnet_tx_header *txhdr;
1929         struct virtio_net_hdr *hdr;
1930         struct mbuf *m;
1931         int error;
1932
1933         txhdr = &sc->vtnet_txhdrarea[sc->vtnet_txhdridx];
1934         memset(txhdr, 0, sizeof(struct vtnet_tx_header));
1935
1936         /*
1937          * Always use the non-mergeable header to simplify things. When
1938          * the mergeable feature is negotiated, the num_buffers field
1939          * must be set to zero. We use vtnet_hdr_size later to enqueue
1940          * the correct header size to the host.
1941          */
1942         hdr = &txhdr->vth_uhdr.hdr;
1943         m = *m_head;
1944
1945         error = ENOBUFS;
1946
1947         if (m->m_flags & M_VLANTAG) {
1948                 //m = ether_vlanencap(m, m->m_pkthdr.ether_vtag);
1949                 m = vtnet_vlan_tag_insert(m);
1950                 if ((*m_head = m) == NULL)
1951                         goto fail;
1952                 m->m_flags &= ~M_VLANTAG;
1953         }
1954
1955         if (m->m_pkthdr.csum_flags != 0) {
1956                 m = vtnet_tx_offload(sc, m, hdr);
1957                 if ((*m_head = m) == NULL)
1958                         goto fail;
1959         }
1960
1961         error = vtnet_enqueue_txbuf(sc, m_head, txhdr);
1962         if (error == 0)
1963                 sc->vtnet_txhdridx =
1964                     (sc->vtnet_txhdridx + 1) % ((sc->vtnet_tx_size / 2) + 1);
1965 fail:
1966         return (error);
1967 }
1968
1969 static void
1970 vtnet_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
1971 {
1972         struct vtnet_softc *sc;
1973
1974         sc = ifp->if_softc;
1975
1976         ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq);
1977         lwkt_serialize_enter(&sc->vtnet_slz);
1978         vtnet_start_locked(ifp, ifsq);
1979         lwkt_serialize_exit(&sc->vtnet_slz);
1980 }
1981
1982 static void
1983 vtnet_start_locked(struct ifnet *ifp, struct ifaltq_subque *ifsq)
1984 {
1985         struct vtnet_softc *sc;
1986         struct virtqueue *vq;
1987         struct mbuf *m0;
1988         int enq;
1989
1990         sc = ifp->if_softc;
1991         vq = sc->vtnet_tx_vq;
1992         enq = 0;
1993
1994         ASSERT_SERIALIZED(&sc->vtnet_slz);
1995
1996         if ((ifp->if_flags & (IFF_RUNNING)) !=
1997             IFF_RUNNING || ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0))
1998                 return;
1999
2000 #ifdef VTNET_TX_INTR_MODERATION
2001         if (virtqueue_nused(vq) >= sc->vtnet_tx_size / 2)
2002                 vtnet_txeof(sc);
2003 #endif
2004
2005         while (!ifsq_is_empty(ifsq)) {
2006                 if (virtqueue_full(vq)) {
2007                         ifq_set_oactive(&ifp->if_snd);
2008                         break;
2009                 }
2010
2011                 m0 = ifq_dequeue(&ifp->if_snd);
2012                 if (m0 == NULL)
2013                         break;
2014
2015                 if (vtnet_encap(sc, &m0) != 0) {
2016                         if (m0 == NULL)
2017                                 break;
2018                         ifq_prepend(&ifp->if_snd, m0);
2019                         ifq_set_oactive(&ifp->if_snd);
2020                         break;
2021                 }
2022
2023                 enq++;
2024                 ETHER_BPF_MTAP(ifp, m0);
2025         }
2026
2027         if (enq > 0) {
2028                 virtqueue_notify(vq, &sc->vtnet_slz);
2029                 sc->vtnet_watchdog_timer = VTNET_WATCHDOG_TIMEOUT;
2030         }
2031 }
2032
2033 static void
2034 vtnet_tick(void *xsc)
2035 {
2036         struct vtnet_softc *sc;
2037
2038         sc = xsc;
2039
2040 #if 0
2041         ASSERT_SERIALIZED(&sc->vtnet_slz);
2042 #ifdef VTNET_DEBUG
2043         virtqueue_dump(sc->vtnet_rx_vq);
2044         virtqueue_dump(sc->vtnet_tx_vq);
2045 #endif
2046
2047         vtnet_watchdog(sc);
2048         callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc);
2049 #endif
2050 }
2051
2052 static void
2053 vtnet_tx_intr_task(void *arg)
2054 {
2055         struct vtnet_softc *sc;
2056         struct ifnet *ifp;
2057         struct ifaltq_subque *ifsq;
2058
2059         sc = arg;
2060         ifp = sc->vtnet_ifp;
2061         ifsq = ifq_get_subq_default(&ifp->if_snd);
2062
2063 //      lwkt_serialize_enter(&sc->vtnet_slz);
2064
2065         if ((ifp->if_flags & IFF_RUNNING) == 0) {
2066                 vtnet_enable_tx_intr(sc);
2067 //              lwkt_serialize_exit(&sc->vtnet_slz);
2068                 return;
2069         }
2070
2071         vtnet_txeof(sc);
2072
2073         if (!ifsq_is_empty(ifsq))
2074                 vtnet_start_locked(ifp, ifsq);
2075
2076         if (vtnet_enable_tx_intr(sc) != 0) {
2077                 vtnet_disable_tx_intr(sc);
2078                 sc->vtnet_stats.tx_task_rescheduled++;
2079 //              lwkt_serialize_exit(&sc->vtnet_slz);
2080                 vtnet_tx_intr_task(sc);
2081                 /* XXX: loop?! */
2082                 return;
2083         }
2084
2085 //      lwkt_serialize_exit(&sc->vtnet_slz);
2086 }
2087
2088 static int
2089 vtnet_tx_vq_intr(void *xsc)
2090 {
2091         struct vtnet_softc *sc;
2092
2093         sc = xsc;
2094
2095         vtnet_disable_tx_intr(sc);
2096         vtnet_tx_intr_task(sc);
2097
2098         return (1);
2099 }
2100
2101 static void
2102 vtnet_stop(struct vtnet_softc *sc)
2103 {
2104         device_t dev;
2105         struct ifnet *ifp;
2106
2107         dev = sc->vtnet_dev;
2108         ifp = sc->vtnet_ifp;
2109
2110         ASSERT_SERIALIZED(&sc->vtnet_slz);
2111
2112         sc->vtnet_watchdog_timer = 0;
2113         callout_stop(&sc->vtnet_tick_ch);
2114         ifq_clr_oactive(&ifp->if_snd);
2115         ifp->if_flags &= ~(IFF_RUNNING);
2116
2117         vtnet_disable_rx_intr(sc);
2118         vtnet_disable_tx_intr(sc);
2119
2120         /*
2121          * Stop the host VirtIO adapter. Note this will reset the host
2122          * adapter's state back to the pre-initialized state, so in
2123          * order to make the device usable again, we must drive it
2124          * through virtio_reinit() and virtio_reinit_complete().
2125          */
2126         virtio_stop(dev);
2127
2128         sc->vtnet_flags &= ~VTNET_FLAG_LINK;
2129
2130         vtnet_free_rx_mbufs(sc);
2131         vtnet_free_tx_mbufs(sc);
2132 }
2133
2134 static int
2135 vtnet_reinit(struct vtnet_softc *sc)
2136 {
2137         struct ifnet *ifp;
2138         uint64_t features;
2139
2140         ifp = sc->vtnet_ifp;
2141         features = sc->vtnet_features;
2142
2143         /*
2144          * Re-negotiate with the host, removing any disabled receive
2145          * features. Transmit features are disabled only on our side
2146          * via if_capenable and if_hwassist.
2147          */
2148
2149         if (ifp->if_capabilities & IFCAP_RXCSUM) {
2150                 if ((ifp->if_capenable & IFCAP_RXCSUM) == 0)
2151                         features &= ~VIRTIO_NET_F_GUEST_CSUM;
2152         }
2153
2154         if (ifp->if_capabilities & IFCAP_LRO) {
2155                 if ((ifp->if_capenable & IFCAP_LRO) == 0)
2156                         features &= ~VTNET_LRO_FEATURES;
2157         }
2158
2159         if (ifp->if_capabilities & IFCAP_VLAN_HWFILTER) {
2160                 if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
2161                         features &= ~VIRTIO_NET_F_CTRL_VLAN;
2162         }
2163
2164         return (virtio_reinit(sc->vtnet_dev, features));
2165 }
2166
2167 static void
2168 vtnet_init_locked(struct vtnet_softc *sc)
2169 {
2170         device_t dev;
2171         struct ifnet *ifp;
2172         int error;
2173
2174         dev = sc->vtnet_dev;
2175         ifp = sc->vtnet_ifp;
2176
2177         ASSERT_SERIALIZED(&sc->vtnet_slz);
2178
2179         if (ifp->if_flags & IFF_RUNNING)
2180                 return;
2181
2182         /* Stop host's adapter, cancel any pending I/O. */
2183         vtnet_stop(sc);
2184
2185         /* Reinitialize the host device. */
2186         error = vtnet_reinit(sc);
2187         if (error) {
2188                 device_printf(dev,
2189                     "reinitialization failed, stopping device...\n");
2190                 vtnet_stop(sc);
2191                 return;
2192         }
2193
2194         /* Update host with assigned MAC address. */
2195         bcopy(IF_LLADDR(ifp), sc->vtnet_hwaddr, ETHER_ADDR_LEN);
2196         vtnet_set_hwaddr(sc);
2197
2198         ifp->if_hwassist = 0;
2199         if (ifp->if_capenable & IFCAP_TXCSUM)
2200                 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;
2201         if (ifp->if_capenable & IFCAP_TSO4)
2202                 ifp->if_hwassist |= CSUM_TSO;
2203
2204         error = vtnet_init_rx_vq(sc);
2205         if (error) {
2206                 device_printf(dev,
2207                     "cannot allocate mbufs for Rx virtqueue\n");
2208                 vtnet_stop(sc);
2209                 return;
2210         }
2211
2212         if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
2213                 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
2214                         /* Restore promiscuous and all-multicast modes. */
2215                         vtnet_rx_filter(sc);
2216
2217                         /* Restore filtered MAC addresses. */
2218                         vtnet_rx_filter_mac(sc);
2219                 }
2220
2221                 /* Restore VLAN filters. */
2222                 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
2223                         vtnet_rx_filter_vlan(sc);
2224         }
2225
2226         {
2227                 vtnet_enable_rx_intr(sc);
2228                 vtnet_enable_tx_intr(sc);
2229         }
2230
2231         ifp->if_flags |= IFF_RUNNING;
2232         ifq_clr_oactive(&ifp->if_snd);
2233
2234         virtio_reinit_complete(dev);
2235
2236         vtnet_update_link_status(sc);
2237         callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc);
2238 }
2239
2240 static void
2241 vtnet_init(void *xsc)
2242 {
2243         struct vtnet_softc *sc;
2244
2245         sc = xsc;
2246
2247         lwkt_serialize_enter(&sc->vtnet_slz);
2248         vtnet_init_locked(sc);
2249         lwkt_serialize_exit(&sc->vtnet_slz);
2250 }
2251
2252 static void
2253 vtnet_exec_ctrl_cmd(struct vtnet_softc *sc, void *cookie,
2254     struct sglist *sg, int readable, int writable)
2255 {
2256         struct virtqueue *vq;
2257         void *c;
2258
2259         vq = sc->vtnet_ctrl_vq;
2260
2261         ASSERT_SERIALIZED(&sc->vtnet_slz);
2262         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ,
2263             ("no control virtqueue"));
2264         KASSERT(virtqueue_empty(vq),
2265             ("control command already enqueued"));
2266
2267         if (virtqueue_enqueue(vq, cookie, sg, readable, writable) != 0)
2268                 return;
2269
2270         virtqueue_notify(vq, &sc->vtnet_slz);
2271
2272         /*
2273          * Poll until the command is complete. Previously, we would
2274          * sleep until the control virtqueue interrupt handler woke
2275          * us up, but dropping the VTNET_MTX leads to serialization
2276          * difficulties.
2277          *
2278          * Furthermore, it appears QEMU/KVM only allocates three MSIX
2279          * vectors. Two of those vectors are needed for the Rx and Tx
2280          * virtqueues. We do not support sharing both a Vq and config
2281          * changed notification on the same MSIX vector.
2282          */
2283         c = virtqueue_poll(vq, NULL);
2284         KASSERT(c == cookie, ("unexpected control command response"));
2285 }
2286
2287 static void
2288 vtnet_rx_filter(struct vtnet_softc *sc)
2289 {
2290         device_t dev;
2291         struct ifnet *ifp;
2292
2293         dev = sc->vtnet_dev;
2294         ifp = sc->vtnet_ifp;
2295
2296         ASSERT_SERIALIZED(&sc->vtnet_slz);
2297         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
2298             ("CTRL_RX feature not negotiated"));
2299
2300         if (vtnet_set_promisc(sc, ifp->if_flags & IFF_PROMISC) != 0)
2301                 device_printf(dev, "cannot %s promiscuous mode\n",
2302                     ifp->if_flags & IFF_PROMISC ? "enable" : "disable");
2303
2304         if (vtnet_set_allmulti(sc, ifp->if_flags & IFF_ALLMULTI) != 0)
2305                 device_printf(dev, "cannot %s all-multicast mode\n",
2306                     ifp->if_flags & IFF_ALLMULTI ? "enable" : "disable");
2307 }
2308
2309 static int
2310 vtnet_ctrl_rx_cmd(struct vtnet_softc *sc, int cmd, int on)
2311 {
2312         struct virtio_net_ctrl_hdr hdr __aligned(2);
2313         struct sglist_seg segs[3];
2314         struct sglist sg;
2315         uint8_t onoff, ack;
2316         int error;
2317
2318         if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0)
2319                 return (ENOTSUP);
2320
2321         error = 0;
2322
2323         hdr.class = VIRTIO_NET_CTRL_RX;
2324         hdr.cmd = cmd;
2325         onoff = !!on;
2326         ack = VIRTIO_NET_ERR;
2327
2328         sglist_init(&sg, 3, segs);
2329         error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
2330         error |= sglist_append(&sg, &onoff, sizeof(uint8_t));
2331         error |= sglist_append(&sg, &ack, sizeof(uint8_t));
2332         KASSERT(error == 0 && sg.sg_nseg == 3,
2333             ("error adding Rx filter message to sglist"));
2334
2335         vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
2336
2337         return (ack == VIRTIO_NET_OK ? 0 : EIO);
2338 }
2339
2340 static int
2341 vtnet_set_promisc(struct vtnet_softc *sc, int on)
2342 {
2343
2344         return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_PROMISC, on));
2345 }
2346
2347 static int
2348 vtnet_set_allmulti(struct vtnet_softc *sc, int on)
2349 {
2350
2351         return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, on));
2352 }
2353
2354 static void
2355 vtnet_rx_filter_mac(struct vtnet_softc *sc)
2356 {
2357         struct virtio_net_ctrl_hdr hdr;
2358         struct vtnet_mac_filter *filter;
2359         struct sglist_seg segs[4];
2360         struct sglist sg;
2361         struct ifnet *ifp;
2362         struct ifaddr *ifa;
2363         struct ifaddr_container *ifac;
2364         struct ifmultiaddr *ifma;
2365         int ucnt, mcnt, promisc, allmulti, error;
2366         uint8_t ack;
2367
2368         ifp = sc->vtnet_ifp;
2369         ucnt = 0;
2370         mcnt = 0;
2371         promisc = 0;
2372         allmulti = 0;
2373         error = 0;
2374
2375         ASSERT_SERIALIZED(&sc->vtnet_slz);
2376         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
2377             ("CTRL_RX feature not negotiated"));
2378
2379         /* Use the MAC filtering table allocated in vtnet_attach. */
2380         filter = sc->vtnet_macfilter;
2381
2382         /* Unicast MAC addresses: */
2383         //if_addr_rlock(ifp);
2384         TAILQ_FOREACH(ifac, &ifp->if_addrheads[mycpuid], ifa_link) {
2385                 ifa = ifac->ifa;
2386                 if (ifa->ifa_addr->sa_family != AF_LINK)
2387                         continue;
2388                 else if (ucnt == VTNET_MAX_MAC_ENTRIES)
2389                         break;
2390
2391                 bcopy(LLADDR((struct sockaddr_dl *)ifa->ifa_addr),
2392                     &filter->vmf_unicast.macs[ucnt], ETHER_ADDR_LEN);
2393                 ucnt++;
2394         }
2395         //if_addr_runlock(ifp);
2396
2397         if (ucnt >= VTNET_MAX_MAC_ENTRIES) {
2398                 promisc = 1;
2399                 filter->vmf_unicast.nentries = 0;
2400
2401                 if_printf(ifp, "more than %d MAC addresses assigned, "
2402                     "falling back to promiscuous mode\n",
2403                     VTNET_MAX_MAC_ENTRIES);
2404         } else
2405                 filter->vmf_unicast.nentries = ucnt;
2406
2407         /* Multicast MAC addresses: */
2408         //if_maddr_rlock(ifp);
2409         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2410                 if (ifma->ifma_addr->sa_family != AF_LINK)
2411                         continue;
2412                 else if (mcnt == VTNET_MAX_MAC_ENTRIES)
2413                         break;
2414
2415                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2416                     &filter->vmf_multicast.macs[mcnt], ETHER_ADDR_LEN);
2417                 mcnt++;
2418         }
2419         //if_maddr_runlock(ifp);
2420
2421         if (mcnt >= VTNET_MAX_MAC_ENTRIES) {
2422                 allmulti = 1;
2423                 filter->vmf_multicast.nentries = 0;
2424
2425                 if_printf(ifp, "more than %d multicast MAC addresses "
2426                     "assigned, falling back to all-multicast mode\n",
2427                     VTNET_MAX_MAC_ENTRIES);
2428         } else
2429                 filter->vmf_multicast.nentries = mcnt;
2430
2431         if (promisc && allmulti)
2432                 goto out;
2433
2434         hdr.class = VIRTIO_NET_CTRL_MAC;
2435         hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
2436         ack = VIRTIO_NET_ERR;
2437
2438         sglist_init(&sg, 4, segs);
2439         error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
2440         error |= sglist_append(&sg, &filter->vmf_unicast,
2441             sizeof(struct vtnet_mac_table));
2442         error |= sglist_append(&sg, &filter->vmf_multicast,
2443             sizeof(struct vtnet_mac_table));
2444         error |= sglist_append(&sg, &ack, sizeof(uint8_t));
2445         KASSERT(error == 0 && sg.sg_nseg == 4,
2446             ("error adding MAC filtering message to sglist"));
2447
2448         vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
2449
2450         if (ack != VIRTIO_NET_OK)
2451                 if_printf(ifp, "error setting host MAC filter table\n");
2452
2453 out:
2454         if (promisc)
2455                 if (vtnet_set_promisc(sc, 1) != 0)
2456                         if_printf(ifp, "cannot enable promiscuous mode\n");
2457         if (allmulti)
2458                 if (vtnet_set_allmulti(sc, 1) != 0)
2459                         if_printf(ifp, "cannot enable all-multicast mode\n");
2460 }
2461
2462 static int
2463 vtnet_exec_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
2464 {
2465         struct virtio_net_ctrl_hdr hdr __aligned(2);
2466         struct sglist_seg segs[3];
2467         struct sglist sg;
2468         uint8_t ack;
2469         int error;
2470
2471         hdr.class = VIRTIO_NET_CTRL_VLAN;
2472         hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
2473         ack = VIRTIO_NET_ERR;
2474         error = 0;
2475
2476         sglist_init(&sg, 3, segs);
2477         error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
2478         error |= sglist_append(&sg, &tag, sizeof(uint16_t));
2479         error |= sglist_append(&sg, &ack, sizeof(uint8_t));
2480         KASSERT(error == 0 && sg.sg_nseg == 3,
2481             ("error adding VLAN control message to sglist"));
2482
2483         vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
2484
2485         return (ack == VIRTIO_NET_OK ? 0 : EIO);
2486 }
2487
2488 static void
2489 vtnet_rx_filter_vlan(struct vtnet_softc *sc)
2490 {
2491         device_t dev;
2492         uint32_t w, mask;
2493         uint16_t tag;
2494         int i, nvlans, error;
2495
2496         ASSERT_SERIALIZED(&sc->vtnet_slz);
2497         KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER,
2498             ("VLAN_FILTER feature not negotiated"));
2499
2500         dev = sc->vtnet_dev;
2501         nvlans = sc->vtnet_nvlans;
2502         error = 0;
2503
2504         /* Enable filtering for each configured VLAN. */
2505         for (i = 0; i < VTNET_VLAN_SHADOW_SIZE && nvlans > 0; i++) {
2506                 w = sc->vtnet_vlan_shadow[i];
2507                 for (mask = 1, tag = i * 32; w != 0; mask <<= 1, tag++) {
2508                         if ((w & mask) != 0) {
2509                                 w &= ~mask;
2510                                 nvlans--;
2511                                 if (vtnet_exec_vlan_filter(sc, 1, tag) != 0)
2512                                         error++;
2513                         }
2514                 }
2515         }
2516
2517         KASSERT(nvlans == 0, ("VLAN count incorrect"));
2518         if (error)
2519                 device_printf(dev, "cannot restore VLAN filter table\n");
2520 }
2521
2522 static void
2523 vtnet_set_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
2524 {
2525         struct ifnet *ifp;
2526         int idx, bit;
2527
2528         KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER,
2529             ("VLAN_FILTER feature not negotiated"));
2530
2531         if ((tag == 0) || (tag > 4095))
2532                 return;
2533
2534         ifp = sc->vtnet_ifp;
2535         idx = (tag >> 5) & 0x7F;
2536         bit = tag & 0x1F;
2537
2538         lwkt_serialize_enter(&sc->vtnet_slz);
2539
2540         /* Update shadow VLAN table. */
2541         if (add) {
2542                 sc->vtnet_nvlans++;
2543                 sc->vtnet_vlan_shadow[idx] |= (1 << bit);
2544         } else {
2545                 sc->vtnet_nvlans--;
2546                 sc->vtnet_vlan_shadow[idx] &= ~(1 << bit);
2547         }
2548
2549         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
2550                 if (vtnet_exec_vlan_filter(sc, add, tag) != 0) {
2551                         device_printf(sc->vtnet_dev,
2552                             "cannot %s VLAN %d %s the host filter table\n",
2553                             add ? "add" : "remove", tag,
2554                             add ? "to" : "from");
2555                 }
2556         }
2557
2558         lwkt_serialize_exit(&sc->vtnet_slz);
2559 }
2560
2561 static void
2562 vtnet_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
2563 {
2564
2565         if (ifp->if_softc != arg)
2566                 return;
2567
2568         vtnet_set_vlan_filter(arg, 1, tag);
2569 }
2570
2571 static void
2572 vtnet_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
2573 {
2574
2575         if (ifp->if_softc != arg)
2576                 return;
2577
2578         vtnet_set_vlan_filter(arg, 0, tag);
2579 }
2580
2581 static int
2582 vtnet_ifmedia_upd(struct ifnet *ifp)
2583 {
2584         struct vtnet_softc *sc;
2585         struct ifmedia *ifm;
2586
2587         sc = ifp->if_softc;
2588         ifm = &sc->vtnet_media;
2589
2590         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
2591                 return (EINVAL);
2592
2593         return (0);
2594 }
2595
2596 static void
2597 vtnet_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
2598 {
2599         struct vtnet_softc *sc;
2600
2601         sc = ifp->if_softc;
2602
2603         ifmr->ifm_status = IFM_AVALID;
2604         ifmr->ifm_active = IFM_ETHER;
2605
2606         lwkt_serialize_enter(&sc->vtnet_slz);
2607         if (vtnet_is_link_up(sc) != 0) {
2608                 ifmr->ifm_status |= IFM_ACTIVE;
2609                 ifmr->ifm_active |= VTNET_MEDIATYPE;
2610         } else
2611                 ifmr->ifm_active |= IFM_NONE;
2612         lwkt_serialize_exit(&sc->vtnet_slz);
2613 }
2614
2615 static void
2616 vtnet_add_statistics(struct vtnet_softc *sc)
2617 {
2618         device_t dev;
2619         struct vtnet_statistics *stats;
2620         //struct sysctl_ctx_list *ctx;
2621         //struct sysctl_oid *tree;
2622         //struct sysctl_oid_list *child;
2623         int error = 0;
2624
2625         dev = sc->vtnet_dev;
2626         stats = &sc->vtnet_stats;
2627         sysctl_ctx_init(&sc->vtnet_sysctl_ctx);
2628         sc->vtnet_sysctl_tree = SYSCTL_ADD_NODE(&sc->vtnet_sysctl_ctx,
2629                                                 SYSCTL_STATIC_CHILDREN(_hw),
2630                                                 OID_AUTO,
2631                                                 device_get_nameunit(dev),
2632                                                 CTLFLAG_RD, 0, "");
2633
2634         if (sc->vtnet_sysctl_tree == NULL) {
2635                 device_printf(dev, "can't add sysctl node\n");
2636                 error = ENXIO;
2637         }
2638
2639
2640         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx,
2641                          SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO,
2642                          "mbuf_alloc_failed", CTLFLAG_RD, &stats->mbuf_alloc_failed,
2643                          "Mbuf cluster allocation failures");
2644         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx,
2645                          SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO,
2646                          "rx_frame_too_large", CTLFLAG_RD, &stats->rx_frame_too_large,
2647                          "Received frame larger than the mbuf chain");
2648         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx,SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_enq_replacement_failed",
2649             CTLFLAG_RD, &stats->rx_enq_replacement_failed,
2650             "Enqueuing the replacement receive mbuf failed");
2651         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_mergeable_failed",
2652             CTLFLAG_RD, &stats->rx_mergeable_failed,
2653             "Mergeable buffers receive failures");
2654         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_bad_ethtype",
2655             CTLFLAG_RD, &stats->rx_csum_bad_ethtype,
2656             "Received checksum offloaded buffer with unsupported "
2657             "Ethernet type");
2658         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_bad_start",
2659             CTLFLAG_RD, &stats->rx_csum_bad_start,
2660             "Received checksum offloaded buffer with incorrect start offset");
2661         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_bad_ipproto",
2662             CTLFLAG_RD, &stats->rx_csum_bad_ipproto,
2663             "Received checksum offloaded buffer with incorrect IP protocol");
2664         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_bad_offset",
2665             CTLFLAG_RD, &stats->rx_csum_bad_offset,
2666             "Received checksum offloaded buffer with incorrect offset");
2667         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_failed",
2668             CTLFLAG_RD, &stats->rx_csum_failed,
2669             "Received buffer checksum offload failed");
2670         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_offloaded",
2671             CTLFLAG_RD, &stats->rx_csum_offloaded,
2672             "Received buffer checksum offload succeeded");
2673         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_task_rescheduled",
2674             CTLFLAG_RD, &stats->rx_task_rescheduled,
2675             "Times the receive interrupt task rescheduled itself");
2676
2677         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_csum_offloaded",
2678             CTLFLAG_RD, &stats->tx_csum_offloaded,
2679             "Offloaded checksum of transmitted buffer");
2680         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_tso_offloaded",
2681             CTLFLAG_RD, &stats->tx_tso_offloaded,
2682             "Segmentation offload of transmitted buffer");
2683         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_csum_bad_ethtype",
2684             CTLFLAG_RD, &stats->tx_csum_bad_ethtype,
2685             "Aborted transmit of checksum offloaded buffer with unknown "
2686             "Ethernet type");
2687         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_tso_bad_ethtype",
2688             CTLFLAG_RD, &stats->tx_tso_bad_ethtype,
2689             "Aborted transmit of TSO buffer with unknown Ethernet type");
2690         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_task_rescheduled",
2691             CTLFLAG_RD, &stats->tx_task_rescheduled,
2692             "Times the transmit interrupt task rescheduled itself");
2693 }
2694
2695 static int
2696 vtnet_enable_rx_intr(struct vtnet_softc *sc)
2697 {
2698
2699         return (virtqueue_enable_intr(sc->vtnet_rx_vq));
2700 }
2701
2702 static void
2703 vtnet_disable_rx_intr(struct vtnet_softc *sc)
2704 {
2705
2706         virtqueue_disable_intr(sc->vtnet_rx_vq);
2707 }
2708
2709 static int
2710 vtnet_enable_tx_intr(struct vtnet_softc *sc)
2711 {
2712
2713 #ifdef VTNET_TX_INTR_MODERATION
2714         return (0);
2715 #else
2716         return (virtqueue_enable_intr(sc->vtnet_tx_vq));
2717 #endif
2718 }
2719
2720 static void
2721 vtnet_disable_tx_intr(struct vtnet_softc *sc)
2722 {
2723
2724         virtqueue_disable_intr(sc->vtnet_tx_vq);
2725 }