sys/dev/virtual/virtio/net/if_vtnet.c

   1 /*-
   2  * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice unmodified, this list of conditions, and the following
  10  *    disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 /* Driver for VirtIO network devices. */
  28
  29 #include <sys/cdefs.h>
  30
  31 #include <sys/param.h>
  32 #include <sys/systm.h>
  33 #include <sys/kernel.h>
  34 #include <sys/sockio.h>
  35 #include <sys/mbuf.h>
  36 #include <sys/malloc.h>
  37 #include <sys/module.h>
  38 #include <sys/socket.h>
  39 #include <sys/sysctl.h>
  40 #include <sys/taskqueue.h>
  41 #include <sys/random.h>
  42 #include <sys/sglist.h>
  43 #include <sys/serialize.h>
  44 #include <sys/bus.h>
  45 #include <sys/rman.h>
  46
  47 #include <net/ethernet.h>
  48 #include <net/if.h>
  49 #include <net/if_arp.h>
  50 #include <net/if_dl.h>
  51 #include <net/if_types.h>
  52 #include <net/if_media.h>
  53 #include <net/vlan/if_vlan_var.h>
  54 #include <net/vlan/if_vlan_ether.h>
  55 #include <net/ifq_var.h>
  56
  57 #include <net/bpf.h>
  58
  59 #include <netinet/in_systm.h>
  60 #include <netinet/in.h>
  61 #include <netinet/ip.h>
  62 #include <netinet/ip6.h>
  63 #include <netinet/udp.h>
  64 #include <netinet/tcp.h>
  65 #include <netinet/sctp.h>
  66
  67 #include <dev/virtual/virtio/virtio/virtio.h>
  68 #include <dev/virtual/virtio/virtio/virtqueue.h>
  69
  70 #include "virtio_net.h"
  71 #include "virtio_if.h"
  72
  73 struct vtnet_statistics {
  74         unsigned long           mbuf_alloc_failed;
  75
  76         unsigned long           rx_frame_too_large;
  77         unsigned long           rx_enq_replacement_failed;
  78         unsigned long           rx_mergeable_failed;
  79         unsigned long           rx_csum_bad_ethtype;
  80         unsigned long           rx_csum_bad_start;
  81         unsigned long           rx_csum_bad_ipproto;
  82         unsigned long           rx_csum_bad_offset;
  83         unsigned long           rx_csum_failed;
  84         unsigned long           rx_csum_offloaded;
  85         unsigned long           rx_task_rescheduled;
  86
  87         unsigned long           tx_csum_offloaded;
  88         unsigned long           tx_tso_offloaded;
  89         unsigned long           tx_csum_bad_ethtype;
  90         unsigned long           tx_tso_bad_ethtype;
  91         unsigned long           tx_task_rescheduled;
  92 };
  93
  94 struct vtnet_softc {
  95         device_t                vtnet_dev;
  96         struct ifnet            *vtnet_ifp;
  97         struct lwkt_serialize   vtnet_slz;
  98
  99         uint32_t                vtnet_flags;
 100 #define VTNET_FLAG_LINK         0x0001
 101 #define VTNET_FLAG_SUSPENDED    0x0002
 102 #define VTNET_FLAG_CTRL_VQ      0x0004
 103 #define VTNET_FLAG_CTRL_RX      0x0008
 104 #define VTNET_FLAG_VLAN_FILTER  0x0010
 105 #define VTNET_FLAG_TSO_ECN      0x0020
 106 #define VTNET_FLAG_MRG_RXBUFS   0x0040
 107 #define VTNET_FLAG_LRO_NOMRG    0x0080
 108
 109         struct virtqueue        *vtnet_rx_vq;
 110         struct virtqueue        *vtnet_tx_vq;
 111         struct virtqueue        *vtnet_ctrl_vq;
 112
 113         struct vtnet_tx_header  *txhdrarea;
 114         uint32_t                txhdridx;
 115
 116         int                     vtnet_hdr_size;
 117         int                     vtnet_tx_size;
 118         int                     vtnet_rx_size;
 119         int                     vtnet_rx_process_limit;
 120         int                     vtnet_rx_mbuf_size;
 121         int                     vtnet_rx_mbuf_count;
 122         int                     vtnet_if_flags;
 123         int                     vtnet_watchdog_timer;
 124         uint64_t                vtnet_features;
 125
 126         struct task             vtnet_cfgchg_task;
 127
 128         struct vtnet_statistics vtnet_stats;
 129
 130         struct sysctl_ctx_list  vtnet_sysctl_ctx;
 131         struct sysctl_oid       *vtnet_sysctl_tree;
 132
 133         struct callout          vtnet_tick_ch;
 134
 135         eventhandler_tag        vtnet_vlan_attach;
 136         eventhandler_tag        vtnet_vlan_detach;
 137
 138         struct ifmedia          vtnet_media;
 139         /*
 140          * Fake media type; the host does not provide us with
 141          * any real media information.
 142          */
 143 #define VTNET_MEDIATYPE         (IFM_ETHER | IFM_1000_T | IFM_FDX)
 144         char                    vtnet_hwaddr[ETHER_ADDR_LEN];
 145
 146         /*
 147          * During reset, the host's VLAN filtering table is lost. The
 148          * array below is used to restore all the VLANs configured on
 149          * this interface after a reset.
 150          */
 151 #define VTNET_VLAN_SHADOW_SIZE  (4096 / 32)
 152         int                     vtnet_nvlans;
 153         uint32_t                vtnet_vlan_shadow[VTNET_VLAN_SHADOW_SIZE];
 154
 155         char                    vtnet_mtx_name[16];
 156 };
 157
 158 /*
 159  * When mergeable buffers are not negotiated, the vtnet_rx_header structure
 160  * below is placed at the beginning of the mbuf data. Use 4 bytes of pad to
 161  * both keep the VirtIO header and the data non-contiguous and to keep the
 162  * frame's payload 4 byte aligned.
 163  *
 164  * When mergeable buffers are negotiated, the host puts the VirtIO header in
 165  * the beginning of the first mbuf's data.
 166  */
 167 #define VTNET_RX_HEADER_PAD     4
 168 struct vtnet_rx_header {
 169         struct virtio_net_hdr   vrh_hdr;
 170         char                    vrh_pad[VTNET_RX_HEADER_PAD];
 171 } __packed;
 172
 173 /*
 174  * For each outgoing frame, the vtnet_tx_header below is allocated from
 175  * the vtnet_tx_header_zone.
 176  */
 177 struct vtnet_tx_header {
 178         union {
 179                 struct virtio_net_hdr           hdr;
 180                 struct virtio_net_hdr_mrg_rxbuf mhdr;
 181         } vth_uhdr;
 182
 183         struct mbuf             *vth_mbuf;
 184 };
 185
 186 MALLOC_DEFINE(M_VTNET, "VTNET_TX", "Outgoing VTNET TX frame header");
 187
 188 /*
 189  * The VirtIO specification does not place a limit on the number of MAC
 190  * addresses the guest driver may request to be filtered. In practice,
 191  * the host is constrained by available resources. To simplify this driver,
 192  * impose a reasonably high limit of MAC addresses we will filter before
 193  * falling back to promiscuous or all-multicast modes.
 194  */
 195 #define VTNET_MAX_MAC_ENTRIES   128
 196
 197 struct vtnet_mac_table {
 198         uint32_t                nentries;
 199         uint8_t                 macs[VTNET_MAX_MAC_ENTRIES][ETHER_ADDR_LEN];
 200 } __packed;
 201
 202 struct vtnet_mac_filter {
 203         struct vtnet_mac_table  vmf_unicast;
 204         uint32_t                vmf_pad; /* Make tables non-contiguous. */
 205         struct vtnet_mac_table  vmf_multicast;
 206 };
 207
 208 /*
 209  * The MAC filter table is malloc(9)'d when needed. Ensure it will
 210  * always fit in one segment.
 211  */
 212 CTASSERT(sizeof(struct vtnet_mac_filter) <= PAGE_SIZE);
 213
 214 #define VTNET_WATCHDOG_TIMEOUT  5
 215 #define VTNET_CSUM_OFFLOAD      (CSUM_TCP | CSUM_UDP)// | CSUM_SCTP)
 216
 217 /* Features desired/implemented by this driver. */
 218 #define VTNET_FEATURES          \
 219     (VIRTIO_NET_F_MAC           | \
 220      VIRTIO_NET_F_STATUS        | \
 221      VIRTIO_NET_F_CTRL_VQ       | \
 222      VIRTIO_NET_F_CTRL_RX       | \
 223      VIRTIO_NET_F_CTRL_VLAN     | \
 224      VIRTIO_NET_F_CSUM          | \
 225      VIRTIO_NET_F_HOST_TSO4     | \
 226      VIRTIO_NET_F_HOST_TSO6     | \
 227      VIRTIO_NET_F_HOST_ECN      | \
 228      VIRTIO_NET_F_GUEST_CSUM    | \
 229      VIRTIO_NET_F_GUEST_TSO4    | \
 230      VIRTIO_NET_F_GUEST_TSO6    | \
 231      VIRTIO_NET_F_GUEST_ECN     | \
 232      VIRTIO_NET_F_MRG_RXBUF)
 233
 234 /*
 235  * The VIRTIO_NET_F_GUEST_TSO[46] features permit the host to send us
 236  * frames larger than 1514 bytes. We do not yet support software LRO
 237  * via tcp_lro_rx().
 238  */
 239 #define VTNET_LRO_FEATURES (VIRTIO_NET_F_GUEST_TSO4 | \
 240                             VIRTIO_NET_F_GUEST_TSO6 | VIRTIO_NET_F_GUEST_ECN)
 241
 242 #define VTNET_MAX_MTU           65536
 243 #define VTNET_MAX_RX_SIZE       65550
 244
 245 /*
 246  * Used to preallocate the Vq indirect descriptors. The first segment
 247  * is reserved for the header.
 248  */
 249 #define VTNET_MIN_RX_SEGS       2
 250 #define VTNET_MAX_RX_SEGS       34
 251 #define VTNET_MAX_TX_SEGS       34
 252
 253 #ifndef CSUM_TSO
 254 #define CSUM_TSO                0
 255 #endif
 256
 257 #define IFCAP_TSO4              0x00100 /* can do TCP Segmentation Offload */
 258 #define IFCAP_TSO6              0x00200 /* can do TCP6 Segmentation Offload */
 259 #define IFCAP_LRO               0x00400 /* can do Large Receive Offload */
 260 #define IFCAP_VLAN_HWFILTER     0x10000 /* interface hw can filter vlan tag */
 261 #define IFCAP_VLAN_HWTSO        0x40000 /* can do IFCAP_TSO on VLANs */
 262
 263
 264 /*
 265  * Assert we can receive and transmit the maximum with regular
 266  * size clusters.
 267  */
 268 CTASSERT(((VTNET_MAX_RX_SEGS - 1) * MCLBYTES) >= VTNET_MAX_RX_SIZE);
 269 CTASSERT(((VTNET_MAX_TX_SEGS - 1) * MCLBYTES) >= VTNET_MAX_MTU);
 270
 271 /*
 272  * Determine how many mbufs are in each receive buffer. For LRO without
 273  * mergeable descriptors, we must allocate an mbuf chain large enough to
 274  * hold both the vtnet_rx_header and the maximum receivable data.
 275  */
 276 #define VTNET_NEEDED_RX_MBUFS(_sc)                                      \
 277         ((_sc)->vtnet_flags & VTNET_FLAG_LRO_NOMRG) == 0 ? 1 :          \
 278         howmany(sizeof(struct vtnet_rx_header) + VTNET_MAX_RX_SIZE,     \
 279         (_sc)->vtnet_rx_mbuf_size)
 280
 281 static int      vtnet_modevent(module_t, int, void *);
 282
 283 static int      vtnet_probe(device_t);
 284 static int      vtnet_attach(device_t);
 285 static int      vtnet_detach(device_t);
 286 static int      vtnet_suspend(device_t);
 287 static int      vtnet_resume(device_t);
 288 static int      vtnet_shutdown(device_t);
 289 static int      vtnet_config_change(device_t);
 290
 291 static void     vtnet_negotiate_features(struct vtnet_softc *);
 292 static int      vtnet_alloc_virtqueues(struct vtnet_softc *);
 293 static void     vtnet_get_hwaddr(struct vtnet_softc *);
 294 static void     vtnet_set_hwaddr(struct vtnet_softc *);
 295 static int      vtnet_is_link_up(struct vtnet_softc *);
 296 static void     vtnet_update_link_status(struct vtnet_softc *);
 297 #if 0
 298 static void     vtnet_watchdog(struct vtnet_softc *);
 299 #endif
 300 static void     vtnet_config_change_task(void *, int);
 301 static int      vtnet_change_mtu(struct vtnet_softc *, int);
 302 static int      vtnet_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 303
 304 static int      vtnet_init_rx_vq(struct vtnet_softc *);
 305 static void     vtnet_free_rx_mbufs(struct vtnet_softc *);
 306 static void     vtnet_free_tx_mbufs(struct vtnet_softc *);
 307 static void     vtnet_free_ctrl_vq(struct vtnet_softc *);
 308
 309 static struct mbuf * vtnet_alloc_rxbuf(struct vtnet_softc *, int,
 310                     struct mbuf **);
 311 static int      vtnet_replace_rxbuf(struct vtnet_softc *,
 312                     struct mbuf *, int);
 313 static int      vtnet_newbuf(struct vtnet_softc *);
 314 static void     vtnet_discard_merged_rxbuf(struct vtnet_softc *, int);
 315 static void     vtnet_discard_rxbuf(struct vtnet_softc *, struct mbuf *);
 316 static int      vtnet_enqueue_rxbuf(struct vtnet_softc *, struct mbuf *);
 317 static void     vtnet_vlan_tag_remove(struct mbuf *);
 318 static int      vtnet_rx_csum(struct vtnet_softc *, struct mbuf *,
 319                     struct virtio_net_hdr *);
 320 static int      vtnet_rxeof_merged(struct vtnet_softc *, struct mbuf *, int);
 321 static int      vtnet_rxeof(struct vtnet_softc *, int, int *);
 322 static void     vtnet_rx_intr_task(void *);
 323 static int      vtnet_rx_vq_intr(void *);
 324
 325 static void     vtnet_txeof(struct vtnet_softc *);
 326 static struct mbuf * vtnet_tx_offload(struct vtnet_softc *, struct mbuf *,
 327                     struct virtio_net_hdr *);
 328 static int      vtnet_enqueue_txbuf(struct vtnet_softc *, struct mbuf **,
 329                     struct vtnet_tx_header *);
 330 static int      vtnet_encap(struct vtnet_softc *, struct mbuf **);
 331 static void     vtnet_start_locked(struct ifnet *, struct ifaltq_subque *);
 332 static void     vtnet_start(struct ifnet *, struct ifaltq_subque *);
 333 static void     vtnet_tick(void *);
 334 static void     vtnet_tx_intr_task(void *);
 335 static int      vtnet_tx_vq_intr(void *);
 336
 337 static void     vtnet_stop(struct vtnet_softc *);
 338 static int      vtnet_reinit(struct vtnet_softc *);
 339 static void     vtnet_init_locked(struct vtnet_softc *);
 340 static void     vtnet_init(void *);
 341
 342 static void     vtnet_exec_ctrl_cmd(struct vtnet_softc *, void *,
 343                     struct sglist *, int, int);
 344
 345 static void     vtnet_rx_filter(struct vtnet_softc *sc);
 346 static int      vtnet_ctrl_rx_cmd(struct vtnet_softc *, int, int);
 347 static int      vtnet_set_promisc(struct vtnet_softc *, int);
 348 static int      vtnet_set_allmulti(struct vtnet_softc *, int);
 349 static void     vtnet_rx_filter_mac(struct vtnet_softc *);
 350
 351 static int      vtnet_exec_vlan_filter(struct vtnet_softc *, int, uint16_t);
 352 static void     vtnet_rx_filter_vlan(struct vtnet_softc *);
 353 static void     vtnet_set_vlan_filter(struct vtnet_softc *, int, uint16_t);
 354 static void     vtnet_register_vlan(void *, struct ifnet *, uint16_t);
 355 static void     vtnet_unregister_vlan(void *, struct ifnet *, uint16_t);
 356
 357 static int      vtnet_ifmedia_upd(struct ifnet *);
 358 static void     vtnet_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 359
 360 static void     vtnet_add_statistics(struct vtnet_softc *);
 361
 362 static int      vtnet_enable_rx_intr(struct vtnet_softc *);
 363 static int      vtnet_enable_tx_intr(struct vtnet_softc *);
 364 static void     vtnet_disable_rx_intr(struct vtnet_softc *);
 365 static void     vtnet_disable_tx_intr(struct vtnet_softc *);
 366
 367 /* Tunables. */
 368 static int vtnet_csum_disable = 0;
 369 TUNABLE_INT("hw.vtnet.csum_disable", &vtnet_csum_disable);
 370 static int vtnet_tso_disable = 1;
 371 TUNABLE_INT("hw.vtnet.tso_disable", &vtnet_tso_disable);
 372 static int vtnet_lro_disable = 1;
 373 TUNABLE_INT("hw.vtnet.lro_disable", &vtnet_lro_disable);
 374
 375 /*
 376  * Reducing the number of transmit completed interrupts can
 377  * improve performance. To do so, the define below keeps the
 378  * Tx vq interrupt disabled and adds calls to vtnet_txeof()
 379  * in the start and watchdog paths. The price to pay for this
 380  * is the m_free'ing of transmitted mbufs may be delayed until
 381  * the watchdog fires.
 382  */
 383 #define VTNET_TX_INTR_MODERATION
 384
 385 static struct virtio_feature_desc vtnet_feature_desc[] = {
 386         { VIRTIO_NET_F_CSUM,            "TxChecksum"    },
 387         { VIRTIO_NET_F_GUEST_CSUM,      "RxChecksum"    },
 388         { VIRTIO_NET_F_MAC,             "MacAddress"    },
 389         { VIRTIO_NET_F_GSO,             "TxAllGSO"      },
 390         { VIRTIO_NET_F_GUEST_TSO4,      "RxTSOv4"       },
 391         { VIRTIO_NET_F_GUEST_TSO6,      "RxTSOv6"       },
 392         { VIRTIO_NET_F_GUEST_ECN,       "RxECN"         },
 393         { VIRTIO_NET_F_GUEST_UFO,       "RxUFO"         },
 394         { VIRTIO_NET_F_HOST_TSO4,       "TxTSOv4"       },
 395         { VIRTIO_NET_F_HOST_TSO6,       "TxTSOv6"       },
 396         { VIRTIO_NET_F_HOST_ECN,        "TxTSOECN"      },
 397         { VIRTIO_NET_F_HOST_UFO,        "TxUFO"         },
 398         { VIRTIO_NET_F_MRG_RXBUF,       "MrgRxBuf"      },
 399         { VIRTIO_NET_F_STATUS,          "Status"        },
 400         { VIRTIO_NET_F_CTRL_VQ,         "ControlVq"     },
 401         { VIRTIO_NET_F_CTRL_RX,         "RxMode"        },
 402         { VIRTIO_NET_F_CTRL_VLAN,       "VLanFilter"    },
 403         { VIRTIO_NET_F_CTRL_RX_EXTRA,   "RxModeExtra"   },
 404         { VIRTIO_NET_F_MQ,              "RFS"           },
 405         { 0, NULL }
 406 };
 407
 408 static device_method_t vtnet_methods[] = {
 409         /* Device methods. */
 410         DEVMETHOD(device_probe,         vtnet_probe),
 411         DEVMETHOD(device_attach,        vtnet_attach),
 412         DEVMETHOD(device_detach,        vtnet_detach),
 413         DEVMETHOD(device_suspend,       vtnet_suspend),
 414         DEVMETHOD(device_resume,        vtnet_resume),
 415         DEVMETHOD(device_shutdown,      vtnet_shutdown),
 416
 417         /* VirtIO methods. */
 418         DEVMETHOD(virtio_config_change, vtnet_config_change),
 419
 420         { 0, 0 }
 421 };
 422
 423 static driver_t vtnet_driver = {
 424         "vtnet",
 425         vtnet_methods,
 426         sizeof(struct vtnet_softc)
 427 };
 428
 429 static devclass_t vtnet_devclass;
 430
 431 DRIVER_MODULE(vtnet, virtio_pci, vtnet_driver, vtnet_devclass,
 432               vtnet_modevent, 0);
 433 MODULE_VERSION(vtnet, 1);
 434 MODULE_DEPEND(vtnet, virtio, 1, 1, 1);
 435
 436 static int
 437 vtnet_modevent(module_t mod, int type, void *unused)
 438 {
 439         int error;
 440
 441         error = 0;
 442
 443         switch (type) {
 444         case MOD_LOAD:
 445                 break;
 446         case MOD_UNLOAD:
 447                 break;
 448         case MOD_SHUTDOWN:
 449                 break;
 450         default:
 451                 error = EOPNOTSUPP;
 452                 break;
 453         }
 454
 455         return (error);
 456 }
 457
 458 static int
 459 vtnet_probe(device_t dev)
 460 {
 461         if (virtio_get_device_type(dev) != VIRTIO_ID_NETWORK)
 462                 return (ENXIO);
 463
 464         device_set_desc(dev, "VirtIO Networking Adapter");
 465
 466         return (BUS_PROBE_DEFAULT);
 467 }
 468
 469 static int
 470 vtnet_attach(device_t dev)
 471 {
 472         struct vtnet_softc *sc;
 473         struct ifnet *ifp;
 474         int tx_size, error;
 475
 476         sc = device_get_softc(dev);
 477         sc->vtnet_dev = dev;
 478
 479         lwkt_serialize_init(&sc->vtnet_slz);
 480         callout_init(&sc->vtnet_tick_ch);
 481
 482         ifmedia_init(&sc->vtnet_media, IFM_IMASK, vtnet_ifmedia_upd,
 483                      vtnet_ifmedia_sts);
 484         ifmedia_add(&sc->vtnet_media, VTNET_MEDIATYPE, 0, NULL);
 485         ifmedia_set(&sc->vtnet_media, VTNET_MEDIATYPE);
 486
 487         vtnet_add_statistics(sc);
 488
 489         virtio_set_feature_desc(dev, vtnet_feature_desc);
 490         vtnet_negotiate_features(sc);
 491
 492         if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) {
 493                 sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS;
 494                 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 495         } else {
 496                 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
 497         }
 498
 499         sc->vtnet_rx_mbuf_size = MCLBYTES;
 500         sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc);
 501
 502         if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VQ)) {
 503                 sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ;
 504
 505                 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX))
 506                         sc->vtnet_flags |= VTNET_FLAG_CTRL_RX;
 507                 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VLAN))
 508                         sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER;
 509         }
 510
 511         vtnet_get_hwaddr(sc);
 512
 513         error = vtnet_alloc_virtqueues(sc);
 514         if (error) {
 515                 device_printf(dev, "cannot allocate virtqueues\n");
 516                 goto fail;
 517         }
 518
 519         ifp = sc->vtnet_ifp = if_alloc(IFT_ETHER);
 520         if (ifp == NULL) {
 521                 device_printf(dev, "cannot allocate ifnet structure\n");
 522                 error = ENOSPC;
 523                 goto fail;
 524         }
 525
 526         ifp->if_softc = sc;
 527         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 528         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 529         ifp->if_init = vtnet_init;
 530         ifp->if_start = vtnet_start;
 531         ifp->if_ioctl = vtnet_ioctl;
 532
 533         sc->vtnet_rx_size = virtqueue_size(sc->vtnet_rx_vq);
 534         sc->vtnet_rx_process_limit = sc->vtnet_rx_size;
 535
 536         tx_size = virtqueue_size(sc->vtnet_tx_vq);
 537         sc->vtnet_tx_size = tx_size;
 538         sc->txhdridx = 0;
 539         sc->txhdrarea = contigmalloc(
 540             ((sc->vtnet_tx_size + 3) / 2) * sizeof(struct vtnet_tx_header),
 541             M_VTNET, M_WAITOK, 0, BUS_SPACE_MAXADDR, 4, 0);
 542         if (sc->txhdrarea == NULL) {
 543                 panic("cannot contigmalloc the tx headers\n");
 544         }
 545         ifq_set_maxlen(&ifp->if_snd, tx_size - 1);
 546         ifq_set_ready(&ifp->if_snd);
 547
 548         ether_ifattach(ifp, sc->vtnet_hwaddr, NULL);
 549
 550         if (virtio_with_feature(dev, VIRTIO_NET_F_STATUS)){
 551                 //ifp->if_capabilities |= IFCAP_LINKSTATE;
 552                  kprintf("add dynamic link state\n");
 553         }
 554
 555         /* Tell the upper layer(s) we support long frames. */
 556         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
 557         ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU;
 558
 559         if (virtio_with_feature(dev, VIRTIO_NET_F_CSUM)) {
 560                 ifp->if_capabilities |= IFCAP_TXCSUM;
 561
 562                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4))
 563                         ifp->if_capabilities |= IFCAP_TSO4;
 564                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
 565                         ifp->if_capabilities |= IFCAP_TSO6;
 566                 if (ifp->if_capabilities & IFCAP_TSO)
 567                         ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
 568
 569                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN))
 570                         sc->vtnet_flags |= VTNET_FLAG_TSO_ECN;
 571         }
 572
 573         if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_CSUM)) {
 574                 ifp->if_capabilities |= IFCAP_RXCSUM;
 575
 576                 if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) ||
 577                     virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6))
 578                         ifp->if_capabilities |= IFCAP_LRO;
 579         }
 580
 581         if (ifp->if_capabilities & IFCAP_HWCSUM) {
 582                 /*
 583                  * VirtIO does not support VLAN tagging, but we can fake
 584                  * it by inserting and removing the 802.1Q header during
 585                  * transmit and receive. We are then able to do checksum
 586                  * offloading of VLAN frames.
 587                  */
 588                 ifp->if_capabilities |=
 589                         IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM;
 590         }
 591
 592         ifp->if_capenable = ifp->if_capabilities;
 593
 594         /*
 595          * Capabilities after here are not enabled by default.
 596          */
 597
 598         if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) {
 599                 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
 600
 601                 sc->vtnet_vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 602                     vtnet_register_vlan, sc, EVENTHANDLER_PRI_FIRST);
 603                 sc->vtnet_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 604                     vtnet_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST);
 605         }
 606
 607         TASK_INIT(&sc->vtnet_cfgchg_task, 0, vtnet_config_change_task, sc);
 608
 609         error = virtio_setup_intr(dev, &sc->vtnet_slz);
 610         if (error) {
 611                 device_printf(dev, "cannot setup virtqueue interrupts\n");
 612                 ether_ifdetach(ifp);
 613                 goto fail;
 614         }
 615
 616         /*
 617          * Device defaults to promiscuous mode for backwards
 618          * compatibility. Turn it off if possible.
 619          */
 620         if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
 621                 lwkt_serialize_enter(&sc->vtnet_slz);
 622                 if (vtnet_set_promisc(sc, 0) != 0) {
 623                         ifp->if_flags |= IFF_PROMISC;
 624                         device_printf(dev,
 625                             "cannot disable promiscuous mode\n");
 626                 }
 627                 lwkt_serialize_exit(&sc->vtnet_slz);
 628         } else
 629                 ifp->if_flags |= IFF_PROMISC;
 630
 631 fail:
 632         if (error)
 633                 vtnet_detach(dev);
 634
 635         return (error);
 636 }
 637
 638 static int
 639 vtnet_detach(device_t dev)
 640 {
 641         struct vtnet_softc *sc;
 642         struct ifnet *ifp;
 643
 644         sc = device_get_softc(dev);
 645         ifp = sc->vtnet_ifp;
 646
 647         if (device_is_attached(dev)) {
 648                 lwkt_serialize_enter(&sc->vtnet_slz);
 649                 vtnet_stop(sc);
 650                 lwkt_serialize_exit(&sc->vtnet_slz);
 651
 652                 callout_stop(&sc->vtnet_tick_ch);
 653                 taskqueue_drain(taskqueue_swi, &sc->vtnet_cfgchg_task);
 654
 655                 ether_ifdetach(ifp);
 656         }
 657
 658         if (sc->vtnet_vlan_attach != NULL) {
 659                 EVENTHANDLER_DEREGISTER(vlan_config, sc->vtnet_vlan_attach);
 660                 sc->vtnet_vlan_attach = NULL;
 661         }
 662         if (sc->vtnet_vlan_detach != NULL) {
 663                 EVENTHANDLER_DEREGISTER(vlan_unconfg, sc->vtnet_vlan_detach);
 664                 sc->vtnet_vlan_detach = NULL;
 665         }
 666
 667         if (ifp) {
 668                 if_free(ifp);
 669                 sc->vtnet_ifp = NULL;
 670         }
 671
 672         if (sc->vtnet_rx_vq != NULL)
 673                 vtnet_free_rx_mbufs(sc);
 674         if (sc->vtnet_tx_vq != NULL)
 675                 vtnet_free_tx_mbufs(sc);
 676         if (sc->vtnet_ctrl_vq != NULL)
 677                 vtnet_free_ctrl_vq(sc);
 678
 679         contigfree(sc->txhdrarea,
 680             ((sc->vtnet_tx_size + 3) / 2) * sizeof(struct vtnet_tx_header),
 681             M_VTNET);
 682
 683         ifmedia_removeall(&sc->vtnet_media);
 684
 685         return (0);
 686 }
 687
 688 static int
 689 vtnet_suspend(device_t dev)
 690 {
 691         struct vtnet_softc *sc;
 692
 693         sc = device_get_softc(dev);
 694
 695         lwkt_serialize_enter(&sc->vtnet_slz);
 696         vtnet_stop(sc);
 697         sc->vtnet_flags |= VTNET_FLAG_SUSPENDED;
 698         lwkt_serialize_exit(&sc->vtnet_slz);
 699
 700         return (0);
 701 }
 702
 703 static int
 704 vtnet_resume(device_t dev)
 705 {
 706         struct vtnet_softc *sc;
 707         struct ifnet *ifp;
 708
 709         sc = device_get_softc(dev);
 710         ifp = sc->vtnet_ifp;
 711
 712         lwkt_serialize_enter(&sc->vtnet_slz);
 713         if (ifp->if_flags & IFF_UP)
 714                 vtnet_init_locked(sc);
 715         sc->vtnet_flags &= ~VTNET_FLAG_SUSPENDED;
 716         lwkt_serialize_exit(&sc->vtnet_slz);
 717
 718         return (0);
 719 }
 720
 721 static int
 722 vtnet_shutdown(device_t dev)
 723 {
 724
 725         /*
 726          * Suspend already does all of what we need to
 727          * do here; we just never expect to be resumed.
 728          */
 729         return (vtnet_suspend(dev));
 730 }
 731
 732 static int
 733 vtnet_config_change(device_t dev)
 734 {
 735         struct vtnet_softc *sc;
 736
 737         sc = device_get_softc(dev);
 738
 739         taskqueue_enqueue(taskqueue_thread[mycpuid], &sc->vtnet_cfgchg_task);
 740
 741         return (1);
 742 }
 743
 744 static void
 745 vtnet_negotiate_features(struct vtnet_softc *sc)
 746 {
 747         device_t dev;
 748         uint64_t mask, features;
 749
 750         dev = sc->vtnet_dev;
 751         mask = 0;
 752
 753         if (vtnet_csum_disable)
 754                 mask |= VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM;
 755
 756         /*
 757          * TSO and LRO are only available when their corresponding
 758          * checksum offload feature is also negotiated.
 759          */
 760
 761         if (vtnet_csum_disable || vtnet_tso_disable)
 762                 mask |= VIRTIO_NET_F_HOST_TSO4 | VIRTIO_NET_F_HOST_TSO6 |
 763                     VIRTIO_NET_F_HOST_ECN;
 764
 765         if (vtnet_csum_disable || vtnet_lro_disable)
 766                 mask |= VTNET_LRO_FEATURES;
 767
 768         features = VTNET_FEATURES & ~mask;
 769         features |= VIRTIO_F_NOTIFY_ON_EMPTY;
 770         sc->vtnet_features = virtio_negotiate_features(dev, features);
 771 }
 772
 773 static int
 774 vtnet_alloc_virtqueues(struct vtnet_softc *sc)
 775 {
 776         device_t dev;
 777         struct vq_alloc_info vq_info[3];
 778         int nvqs, rxsegs;
 779
 780         dev = sc->vtnet_dev;
 781         nvqs = 2;
 782
 783         /*
 784          * Indirect descriptors are not needed for the Rx
 785          * virtqueue when mergeable buffers are negotiated.
 786          * The header is placed inline with the data, not
 787          * in a separate descriptor, and mbuf clusters are
 788          * always physically contiguous.
 789          */
 790         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
 791                 rxsegs = sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG ?
 792                     VTNET_MAX_RX_SEGS : VTNET_MIN_RX_SEGS;
 793         } else
 794                 rxsegs = 0;
 795
 796         VQ_ALLOC_INFO_INIT(&vq_info[0], rxsegs,
 797             vtnet_rx_vq_intr, sc, &sc->vtnet_rx_vq,
 798             "%s receive", device_get_nameunit(dev));
 799
 800         VQ_ALLOC_INFO_INIT(&vq_info[1], VTNET_MAX_TX_SEGS,
 801             vtnet_tx_vq_intr, sc, &sc->vtnet_tx_vq,
 802             "%s transmit", device_get_nameunit(dev));
 803
 804         if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
 805                 nvqs++;
 806
 807                 VQ_ALLOC_INFO_INIT(&vq_info[2], 0, NULL, NULL,
 808                     &sc->vtnet_ctrl_vq, "%s control",
 809                     device_get_nameunit(dev));
 810         }
 811
 812         return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
 813 }
 814
 815 static void
 816 vtnet_get_hwaddr(struct vtnet_softc *sc)
 817 {
 818         device_t dev;
 819
 820         dev = sc->vtnet_dev;
 821
 822         if (virtio_with_feature(dev, VIRTIO_NET_F_MAC)) {
 823                 virtio_read_device_config(dev,
 824                     offsetof(struct virtio_net_config, mac),
 825                     sc->vtnet_hwaddr, ETHER_ADDR_LEN);
 826         } else {
 827                 /* Generate random locally administered unicast address. */
 828                 sc->vtnet_hwaddr[0] = 0xB2;
 829                 karc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1);
 830
 831                 vtnet_set_hwaddr(sc);
 832         }
 833 }
 834
 835 static void
 836 vtnet_set_hwaddr(struct vtnet_softc *sc)
 837 {
 838         device_t dev;
 839
 840         dev = sc->vtnet_dev;
 841
 842         virtio_write_device_config(dev,
 843             offsetof(struct virtio_net_config, mac),
 844             sc->vtnet_hwaddr, ETHER_ADDR_LEN);
 845 }
 846
 847 static int
 848 vtnet_is_link_up(struct vtnet_softc *sc)
 849 {
 850         device_t dev;
 851         struct ifnet *ifp;
 852         uint16_t status;
 853
 854         dev = sc->vtnet_dev;
 855         ifp = sc->vtnet_ifp;
 856
 857         ASSERT_SERIALIZED(&sc->vtnet_slz);
 858
 859         status = virtio_read_dev_config_2(dev,
 860                         offsetof(struct virtio_net_config, status));
 861
 862         return ((status & VIRTIO_NET_S_LINK_UP) != 0);
 863 }
 864
 865 static void
 866 vtnet_update_link_status(struct vtnet_softc *sc)
 867 {
 868         device_t dev;
 869         struct ifnet *ifp;
 870         struct ifaltq_subque *ifsq;
 871         int link;
 872
 873         dev = sc->vtnet_dev;
 874         ifp = sc->vtnet_ifp;
 875         ifsq = ifq_get_subq_default(&ifp->if_snd);
 876
 877         link = vtnet_is_link_up(sc);
 878
 879         if (link && ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0)) {
 880                 sc->vtnet_flags |= VTNET_FLAG_LINK;
 881                 if (bootverbose)
 882                         device_printf(dev, "Link is up\n");
 883                 ifp->if_link_state = LINK_STATE_UP;
 884                 if_link_state_change(ifp);
 885                 if (!ifsq_is_empty(ifsq))
 886                         vtnet_start_locked(ifp, ifsq);
 887         } else if (!link && (sc->vtnet_flags & VTNET_FLAG_LINK)) {
 888                 sc->vtnet_flags &= ~VTNET_FLAG_LINK;
 889                 if (bootverbose)
 890                         device_printf(dev, "Link is down\n");
 891
 892                 ifp->if_link_state = LINK_STATE_DOWN;
 893                 if_link_state_change(ifp);
 894         }
 895 }
 896
 897 #if 0
 898 static void
 899 vtnet_watchdog(struct vtnet_softc *sc)
 900 {
 901         struct ifnet *ifp;
 902
 903         ifp = sc->vtnet_ifp;
 904
 905 #ifdef VTNET_TX_INTR_MODERATION
 906         vtnet_txeof(sc);
 907 #endif
 908
 909         if (sc->vtnet_watchdog_timer == 0 || --sc->vtnet_watchdog_timer)
 910                 return;
 911
 912         if_printf(ifp, "watchdog timeout -- resetting\n");
 913 #ifdef VTNET_DEBUG
 914         virtqueue_dump(sc->vtnet_tx_vq);
 915 #endif
 916         ifp->if_oerrors++;
 917         ifp->if_flags &= ~IFF_RUNNING;
 918         vtnet_init_locked(sc);
 919 }
 920 #endif
 921
 922 static void
 923 vtnet_config_change_task(void *arg, int pending)
 924 {
 925         struct vtnet_softc *sc;
 926
 927         sc = arg;
 928
 929         lwkt_serialize_enter(&sc->vtnet_slz);
 930         vtnet_update_link_status(sc);
 931         lwkt_serialize_exit(&sc->vtnet_slz);
 932 }
 933
 934 static int
 935 vtnet_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data,struct ucred *cr)
 936 {
 937         struct vtnet_softc *sc;
 938         struct ifreq *ifr;
 939         int reinit, mask, error;
 940
 941         sc = ifp->if_softc;
 942         ifr = (struct ifreq *) data;
 943         reinit = 0;
 944         error = 0;
 945
 946         switch (cmd) {
 947         case SIOCSIFMTU:
 948                 if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > VTNET_MAX_MTU)
 949                         error = EINVAL;
 950                 else if (ifp->if_mtu != ifr->ifr_mtu) {
 951                         lwkt_serialize_enter(&sc->vtnet_slz);
 952                         error = vtnet_change_mtu(sc, ifr->ifr_mtu);
 953                         lwkt_serialize_exit(&sc->vtnet_slz);
 954                 }
 955                 break;
 956
 957         case SIOCSIFFLAGS:
 958                 lwkt_serialize_enter(&sc->vtnet_slz);
 959                 if ((ifp->if_flags & IFF_UP) == 0) {
 960                         if (ifp->if_flags & IFF_RUNNING)
 961                                 vtnet_stop(sc);
 962                 } else if (ifp->if_flags & IFF_RUNNING) {
 963                         if ((ifp->if_flags ^ sc->vtnet_if_flags) &
 964                             (IFF_PROMISC | IFF_ALLMULTI)) {
 965                                 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX)
 966                                         vtnet_rx_filter(sc);
 967                                 else
 968                                         error = ENOTSUP;
 969                         }
 970                 } else
 971                         vtnet_init_locked(sc);
 972
 973                 if (error == 0)
 974                         sc->vtnet_if_flags = ifp->if_flags;
 975                 lwkt_serialize_exit(&sc->vtnet_slz);
 976                 break;
 977
 978         case SIOCADDMULTI:
 979         case SIOCDELMULTI:
 980                 lwkt_serialize_enter(&sc->vtnet_slz);
 981                 if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) &&
 982                     (ifp->if_flags & IFF_RUNNING))
 983                         vtnet_rx_filter_mac(sc);
 984                 lwkt_serialize_exit(&sc->vtnet_slz);
 985                 break;
 986
 987         case SIOCSIFMEDIA:
 988         case SIOCGIFMEDIA:
 989                 error = ifmedia_ioctl(ifp, ifr, &sc->vtnet_media, cmd);
 990                 break;
 991
 992         case SIOCSIFCAP:
 993                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 994
 995                 lwkt_serialize_enter(&sc->vtnet_slz);
 996
 997                 if (mask & IFCAP_TXCSUM) {
 998                         ifp->if_capenable ^= IFCAP_TXCSUM;
 999                         if (ifp->if_capenable & IFCAP_TXCSUM)
1000                                 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;
1001                         else
1002                                 ifp->if_hwassist &= ~VTNET_CSUM_OFFLOAD;
1003                 }
1004
1005                 if (mask & IFCAP_TSO4) {
1006                         ifp->if_capenable ^= IFCAP_TSO4;
1007                         if (ifp->if_capenable & IFCAP_TSO4)
1008                                 ifp->if_hwassist |= CSUM_TSO;
1009                         else
1010                                 ifp->if_hwassist &= ~CSUM_TSO;
1011                 }
1012
1013                 if (mask & IFCAP_RXCSUM) {
1014                         ifp->if_capenable ^= IFCAP_RXCSUM;
1015                         reinit = 1;
1016                 }
1017
1018                 if (mask & IFCAP_LRO) {
1019                         ifp->if_capenable ^= IFCAP_LRO;
1020                         reinit = 1;
1021                 }
1022
1023                 if (mask & IFCAP_VLAN_HWFILTER) {
1024                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1025                         reinit = 1;
1026                 }
1027
1028                 if (mask & IFCAP_VLAN_HWTSO)
1029                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1030
1031                 if (mask & IFCAP_VLAN_HWTAGGING)
1032                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1033
1034                 if (reinit && (ifp->if_flags & IFF_RUNNING)) {
1035                         ifp->if_flags &= ~IFF_RUNNING;
1036                         vtnet_init_locked(sc);
1037                 }
1038                 //VLAN_CAPABILITIES(ifp);
1039
1040                 lwkt_serialize_exit(&sc->vtnet_slz);
1041                 break;
1042
1043         default:
1044                 error = ether_ioctl(ifp, cmd, data);
1045                 break;
1046         }
1047
1048         return (error);
1049 }
1050
1051 static int
1052 vtnet_change_mtu(struct vtnet_softc *sc, int new_mtu)
1053 {
1054         struct ifnet *ifp;
1055         int new_frame_size, clsize;
1056
1057         ifp = sc->vtnet_ifp;
1058
1059         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1060                 new_frame_size = sizeof(struct vtnet_rx_header) +
1061                     sizeof(struct ether_vlan_header) + new_mtu;
1062
1063                 if (new_frame_size > MJUM9BYTES)
1064                         return (EINVAL);
1065
1066                 if (new_frame_size <= MCLBYTES)
1067                         clsize = MCLBYTES;
1068                 else
1069                         clsize = MJUM9BYTES;
1070         } else {
1071                 new_frame_size = sizeof(struct virtio_net_hdr_mrg_rxbuf) +
1072                     sizeof(struct ether_vlan_header) + new_mtu;
1073
1074                 if (new_frame_size <= MCLBYTES)
1075                         clsize = MCLBYTES;
1076                 else
1077                         clsize = MJUMPAGESIZE;
1078         }
1079
1080         sc->vtnet_rx_mbuf_size = clsize;
1081         sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc);
1082         KASSERT(sc->vtnet_rx_mbuf_count < VTNET_MAX_RX_SEGS,
1083             ("too many rx mbufs: %d", sc->vtnet_rx_mbuf_count));
1084
1085         ifp->if_mtu = new_mtu;
1086
1087         if (ifp->if_flags & IFF_RUNNING) {
1088                 ifp->if_flags &= ~IFF_RUNNING;
1089                 vtnet_init_locked(sc);
1090         }
1091
1092         return (0);
1093 }
1094
1095 static int
1096 vtnet_init_rx_vq(struct vtnet_softc *sc)
1097 {
1098         struct virtqueue *vq;
1099         int nbufs, error;
1100
1101         vq = sc->vtnet_rx_vq;
1102         nbufs = 0;
1103         error = ENOSPC;
1104
1105         while (!virtqueue_full(vq)) {
1106                 if ((error = vtnet_newbuf(sc)) != 0)
1107                         break;
1108                 nbufs++;
1109         }
1110
1111         if (nbufs > 0) {
1112                 virtqueue_notify(vq, &sc->vtnet_slz);
1113
1114                 /*
1115                  * EMSGSIZE signifies the virtqueue did not have enough
1116                  * entries available to hold the last mbuf. This is not
1117                  * an error. We should not get ENOSPC since we check if
1118                  * the virtqueue is full before attempting to add a
1119                  * buffer.
1120                  */
1121                 if (error == EMSGSIZE)
1122                         error = 0;
1123         }
1124
1125         return (error);
1126 }
1127
1128 static void
1129 vtnet_free_rx_mbufs(struct vtnet_softc *sc)
1130 {
1131         struct virtqueue *vq;
1132         struct mbuf *m;
1133         int last;
1134
1135         vq = sc->vtnet_rx_vq;
1136         last = 0;
1137
1138         while ((m = virtqueue_drain(vq, &last)) != NULL)
1139                 m_freem(m);
1140
1141         KASSERT(virtqueue_empty(vq), ("mbufs remaining in Rx Vq"));
1142 }
1143
1144 static void
1145 vtnet_free_tx_mbufs(struct vtnet_softc *sc)
1146 {
1147         struct virtqueue *vq;
1148         struct vtnet_tx_header *txhdr;
1149         int last;
1150
1151         vq = sc->vtnet_tx_vq;
1152         last = 0;
1153
1154         while ((txhdr = virtqueue_drain(vq, &last)) != NULL) {
1155                 m_freem(txhdr->vth_mbuf);
1156         }
1157
1158         KASSERT(virtqueue_empty(vq), ("mbufs remaining in Tx Vq"));
1159 }
1160
1161 static void
1162 vtnet_free_ctrl_vq(struct vtnet_softc *sc)
1163 {
1164         /*
1165          * The control virtqueue is only polled, therefore
1166          * it should already be empty.
1167          */
1168         KASSERT(virtqueue_empty(sc->vtnet_ctrl_vq),
1169                 ("Ctrl Vq not empty"));
1170 }
1171
1172 static struct mbuf *
1173 vtnet_alloc_rxbuf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp)
1174 {
1175         struct mbuf *m_head, *m_tail, *m;
1176         int i, clsize;
1177
1178         clsize = sc->vtnet_rx_mbuf_size;
1179
1180         /*use getcl instead of getjcl. see  if_mxge.c comment line 2398*/
1181         //m_head = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, clsize);
1182         m_head = m_getcl(MB_DONTWAIT, MT_DATA, M_PKTHDR );
1183         if (m_head == NULL)
1184                 goto fail;
1185
1186         m_head->m_len = clsize;
1187         m_tail = m_head;
1188
1189         if (nbufs > 1) {
1190                 KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
1191                         ("chained Rx mbuf requested without LRO_NOMRG"));
1192
1193                 for (i = 0; i < nbufs - 1; i++) {
1194                         //m = m_getjcl(M_DONTWAIT, MT_DATA, 0, clsize);
1195                         m = m_getcl(MB_DONTWAIT, MT_DATA, 0);
1196                         if (m == NULL)
1197                                 goto fail;
1198
1199                         m->m_len = clsize;
1200                         m_tail->m_next = m;
1201                         m_tail = m;
1202                 }
1203         }
1204
1205         if (m_tailp != NULL)
1206                 *m_tailp = m_tail;
1207
1208         return (m_head);
1209
1210 fail:
1211         sc->vtnet_stats.mbuf_alloc_failed++;
1212         m_freem(m_head);
1213
1214         return (NULL);
1215 }
1216
1217 static int
1218 vtnet_replace_rxbuf(struct vtnet_softc *sc, struct mbuf *m0, int len0)
1219 {
1220         struct mbuf *m, *m_prev;
1221         struct mbuf *m_new, *m_tail;
1222         int len, clsize, nreplace, error;
1223
1224         m = m0;
1225         m_prev = NULL;
1226         len = len0;
1227
1228         m_tail = NULL;
1229         clsize = sc->vtnet_rx_mbuf_size;
1230         nreplace = 0;
1231
1232         if (m->m_next != NULL)
1233                 KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
1234                     ("chained Rx mbuf without LRO_NOMRG"));
1235
1236         /*
1237          * Since LRO_NOMRG mbuf chains are so large, we want to avoid
1238          * allocating an entire chain for each received frame. When
1239          * the received frame's length is less than that of the chain,
1240          * the unused mbufs are reassigned to the new chain.
1241          */
1242         while (len > 0) {
1243                 /*
1244                  * Something is seriously wrong if we received
1245                  * a frame larger than the mbuf chain. Drop it.
1246                  */
1247                 if (m == NULL) {
1248                         sc->vtnet_stats.rx_frame_too_large++;
1249                         return (EMSGSIZE);
1250                 }
1251
1252                 KASSERT(m->m_len == clsize,
1253                     ("mbuf length not expected cluster size: %d",
1254                     m->m_len));
1255
1256                 m->m_len = MIN(m->m_len, len);
1257                 len -= m->m_len;
1258
1259                 m_prev = m;
1260                 m = m->m_next;
1261                 nreplace++;
1262         }
1263
1264         KASSERT(m_prev != NULL, ("m_prev == NULL"));
1265         KASSERT(nreplace <= sc->vtnet_rx_mbuf_count,
1266                 ("too many replacement mbufs: %d/%d", nreplace,
1267                 sc->vtnet_rx_mbuf_count));
1268
1269         m_new = vtnet_alloc_rxbuf(sc, nreplace, &m_tail);
1270         if (m_new == NULL) {
1271                 m_prev->m_len = clsize;
1272                 return (ENOBUFS);
1273         }
1274
1275         /*
1276          * Move unused mbufs, if any, from the original chain
1277          * onto the end of the new chain.
1278          */
1279         if (m_prev->m_next != NULL) {
1280                 m_tail->m_next = m_prev->m_next;
1281                 m_prev->m_next = NULL;
1282         }
1283
1284         error = vtnet_enqueue_rxbuf(sc, m_new);
1285         if (error) {
1286                 /*
1287                  * BAD! We could not enqueue the replacement mbuf chain. We
1288                  * must restore the m0 chain to the original state if it was
1289                  * modified so we can subsequently discard it.
1290                  *
1291                  * NOTE: The replacement is suppose to be an identical copy
1292                  * to the one just dequeued so this is an unexpected error.
1293                  */
1294                 sc->vtnet_stats.rx_enq_replacement_failed++;
1295
1296                 if (m_tail->m_next != NULL) {
1297                         m_prev->m_next = m_tail->m_next;
1298                         m_tail->m_next = NULL;
1299                 }
1300
1301                 m_prev->m_len = clsize;
1302                 m_freem(m_new);
1303         }
1304
1305         return (error);
1306 }
1307
1308 static int
1309 vtnet_newbuf(struct vtnet_softc *sc)
1310 {
1311         struct mbuf *m;
1312         int error;
1313
1314         m = vtnet_alloc_rxbuf(sc, sc->vtnet_rx_mbuf_count, NULL);
1315         if (m == NULL)
1316                 return (ENOBUFS);
1317
1318         error = vtnet_enqueue_rxbuf(sc, m);
1319         if (error)
1320                 m_freem(m);
1321
1322         return (error);
1323 }
1324
1325 static void
1326 vtnet_discard_merged_rxbuf(struct vtnet_softc *sc, int nbufs)
1327 {
1328         struct virtqueue *vq;
1329         struct mbuf *m;
1330
1331         vq = sc->vtnet_rx_vq;
1332
1333         while (--nbufs > 0) {
1334                 if ((m = virtqueue_dequeue(vq, NULL)) == NULL)
1335                         break;
1336                 vtnet_discard_rxbuf(sc, m);
1337         }
1338 }
1339
1340 static void
1341 vtnet_discard_rxbuf(struct vtnet_softc *sc, struct mbuf *m)
1342 {
1343         int error;
1344
1345         /*
1346          * Requeue the discarded mbuf. This should always be
1347          * successful since it was just dequeued.
1348          */
1349         error = vtnet_enqueue_rxbuf(sc, m);
1350         KASSERT(error == 0, ("cannot requeue discarded mbuf"));
1351 }
1352
1353 static int
1354 vtnet_enqueue_rxbuf(struct vtnet_softc *sc, struct mbuf *m)
1355 {
1356         struct sglist sg;
1357         struct sglist_seg segs[VTNET_MAX_RX_SEGS];
1358         struct vtnet_rx_header *rxhdr;
1359         struct virtio_net_hdr *hdr;
1360         uint8_t *mdata;
1361         int offset, error;
1362
1363         ASSERT_SERIALIZED(&sc->vtnet_slz);
1364         if ((sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) == 0)
1365                 KASSERT(m->m_next == NULL, ("chained Rx mbuf"));
1366
1367         sglist_init(&sg, VTNET_MAX_RX_SEGS, segs);
1368
1369         mdata = mtod(m, uint8_t *);
1370         offset = 0;
1371
1372         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1373                 rxhdr = (struct vtnet_rx_header *) mdata;
1374                 hdr = &rxhdr->vrh_hdr;
1375                 offset += sizeof(struct vtnet_rx_header);
1376
1377                 error = sglist_append(&sg, hdr, sc->vtnet_hdr_size);
1378                 KASSERT(error == 0, ("cannot add header to sglist"));
1379         }
1380
1381         error = sglist_append(&sg, mdata + offset, m->m_len - offset);
1382         if (error)
1383                 return (error);
1384
1385         if (m->m_next != NULL) {
1386                 error = sglist_append_mbuf(&sg, m->m_next);
1387                 if (error)
1388                         return (error);
1389         }
1390
1391         return (virtqueue_enqueue(sc->vtnet_rx_vq, m, &sg, 0, sg.sg_nseg));
1392 }
1393
1394 static void
1395 vtnet_vlan_tag_remove(struct mbuf *m)
1396 {
1397         struct ether_vlan_header *evl;
1398
1399         evl = mtod(m, struct ether_vlan_header *);
1400
1401         m->m_pkthdr.ether_vlantag = ntohs(evl->evl_tag);
1402         m->m_flags |= M_VLANTAG;
1403
1404         /* Strip the 802.1Q header. */
1405         bcopy((char *) evl, (char *) evl + ETHER_VLAN_ENCAP_LEN,
1406             ETHER_HDR_LEN - ETHER_TYPE_LEN);
1407         m_adj(m, ETHER_VLAN_ENCAP_LEN);
1408 }
1409
1410 /*
1411  * Alternative method of doing receive checksum offloading. Rather
1412  * than parsing the received frame down to the IP header, use the
1413  * csum_offset to determine which CSUM_* flags are appropriate. We
1414  * can get by with doing this only because the checksum offsets are
1415  * unique for the things we care about.
1416  */
1417 static int
1418 vtnet_rx_csum(struct vtnet_softc *sc, struct mbuf *m,
1419     struct virtio_net_hdr *hdr)
1420 {
1421         struct ether_header *eh;
1422         struct ether_vlan_header *evh;
1423         struct udphdr *udp;
1424         int csum_len;
1425         uint16_t eth_type;
1426
1427         csum_len = hdr->csum_start + hdr->csum_offset;
1428
1429         if (csum_len < sizeof(struct ether_header) + sizeof(struct ip))
1430                 return (1);
1431         if (m->m_len < csum_len)
1432                 return (1);
1433
1434         eh = mtod(m, struct ether_header *);
1435         eth_type = ntohs(eh->ether_type);
1436         if (eth_type == ETHERTYPE_VLAN) {
1437                 evh = mtod(m, struct ether_vlan_header *);
1438                 eth_type = ntohs(evh->evl_proto);
1439         }
1440
1441         if (eth_type != ETHERTYPE_IP && eth_type != ETHERTYPE_IPV6) {
1442                 sc->vtnet_stats.rx_csum_bad_ethtype++;
1443                 return (1);
1444         }
1445
1446         /* Use the offset to determine the appropriate CSUM_* flags. */
1447         switch (hdr->csum_offset) {
1448         case offsetof(struct udphdr, uh_sum):
1449                 if (m->m_len < hdr->csum_start + sizeof(struct udphdr))
1450                         return (1);
1451                 udp = (struct udphdr *)(mtod(m, uint8_t *) + hdr->csum_start);
1452                 if (udp->uh_sum == 0)
1453                         return (0);
1454
1455                 /* FALLTHROUGH */
1456
1457         case offsetof(struct tcphdr, th_sum):
1458                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1459                 m->m_pkthdr.csum_data = 0xFFFF;
1460                 break;
1461
1462         case offsetof(struct sctphdr, checksum):
1463                 //m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID;
1464                 break;
1465
1466         default:
1467                 sc->vtnet_stats.rx_csum_bad_offset++;
1468                 return (1);
1469         }
1470
1471         sc->vtnet_stats.rx_csum_offloaded++;
1472
1473         return (0);
1474 }
1475
1476 static int
1477 vtnet_rxeof_merged(struct vtnet_softc *sc, struct mbuf *m_head, int nbufs)
1478 {
1479         struct ifnet *ifp;
1480         struct virtqueue *vq;
1481         struct mbuf *m, *m_tail;
1482         int len;
1483
1484         ifp = sc->vtnet_ifp;
1485         vq = sc->vtnet_rx_vq;
1486         m_tail = m_head;
1487
1488         while (--nbufs > 0) {
1489                 m = virtqueue_dequeue(vq, &len);
1490                 if (m == NULL) {
1491                         ifp->if_ierrors++;
1492                         goto fail;
1493                 }
1494
1495                 if (vtnet_newbuf(sc) != 0) {
1496                         ifp->if_iqdrops++;
1497                         vtnet_discard_rxbuf(sc, m);
1498                         if (nbufs > 1)
1499                                 vtnet_discard_merged_rxbuf(sc, nbufs);
1500                         goto fail;
1501                 }
1502
1503                 if (m->m_len < len)
1504                         len = m->m_len;
1505
1506                 m->m_len = len;
1507                 m->m_flags &= ~M_PKTHDR;
1508
1509                 m_head->m_pkthdr.len += len;
1510                 m_tail->m_next = m;
1511                 m_tail = m;
1512         }
1513
1514         return (0);
1515
1516 fail:
1517         sc->vtnet_stats.rx_mergeable_failed++;
1518         m_freem(m_head);
1519
1520         return (1);
1521 }
1522
1523 static int
1524 vtnet_rxeof(struct vtnet_softc *sc, int count, int *rx_npktsp)
1525 {
1526         struct virtio_net_hdr lhdr;
1527         struct ifnet *ifp;
1528         struct virtqueue *vq;
1529         struct mbuf *m;
1530         struct ether_header *eh;
1531         struct virtio_net_hdr *hdr;
1532         struct virtio_net_hdr_mrg_rxbuf *mhdr;
1533         int len, deq, nbufs, adjsz, rx_npkts;
1534
1535         ifp = sc->vtnet_ifp;
1536         vq = sc->vtnet_rx_vq;
1537         hdr = &lhdr;
1538         deq = 0;
1539         rx_npkts = 0;
1540
1541         ASSERT_SERIALIZED(&sc->vtnet_slz);
1542
1543         while (--count >= 0) {
1544                 m = virtqueue_dequeue(vq, &len);
1545                 if (m == NULL)
1546                         break;
1547                 deq++;
1548
1549                 if (len < sc->vtnet_hdr_size + ETHER_HDR_LEN) {
1550                         ifp->if_ierrors++;
1551                         vtnet_discard_rxbuf(sc, m);
1552                         continue;
1553                 }
1554
1555                 if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1556                         nbufs = 1;
1557                         adjsz = sizeof(struct vtnet_rx_header);
1558                         /*
1559                          * Account for our pad between the header and
1560                          * the actual start of the frame.
1561                          */
1562                         len += VTNET_RX_HEADER_PAD;
1563                 } else {
1564                         mhdr = mtod(m, struct virtio_net_hdr_mrg_rxbuf *);
1565                         nbufs = mhdr->num_buffers;
1566                         adjsz = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1567                 }
1568
1569                 if (vtnet_replace_rxbuf(sc, m, len) != 0) {
1570                         ifp->if_iqdrops++;
1571                         vtnet_discard_rxbuf(sc, m);
1572                         if (nbufs > 1)
1573                                 vtnet_discard_merged_rxbuf(sc, nbufs);
1574                         continue;
1575                 }
1576
1577                 m->m_pkthdr.len = len;
1578                 m->m_pkthdr.rcvif = ifp;
1579                 m->m_pkthdr.csum_flags = 0;
1580
1581                 if (nbufs > 1) {
1582                         if (vtnet_rxeof_merged(sc, m, nbufs) != 0)
1583                                 continue;
1584                 }
1585
1586                 ifp->if_ipackets++;
1587
1588                 /*
1589                  * Save copy of header before we strip it. For both mergeable
1590                  * and non-mergeable, the VirtIO header is placed first in the
1591                  * mbuf's data. We no longer need num_buffers, so always use a
1592                  * virtio_net_hdr.
1593                  */
1594                 memcpy(hdr, mtod(m, void *), sizeof(struct virtio_net_hdr));
1595                 m_adj(m, adjsz);
1596
1597                 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1598                         eh = mtod(m, struct ether_header *);
1599                         if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
1600                                 vtnet_vlan_tag_remove(m);
1601
1602                                 /*
1603                                  * With the 802.1Q header removed, update the
1604                                  * checksum starting location accordingly.
1605                                  */
1606                                 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
1607                                         hdr->csum_start -=
1608                                             ETHER_VLAN_ENCAP_LEN;
1609                         }
1610                 }
1611
1612                 if (ifp->if_capenable & IFCAP_RXCSUM &&
1613                     hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
1614                         if (vtnet_rx_csum(sc, m, hdr) != 0)
1615                                 sc->vtnet_stats.rx_csum_failed++;
1616                 }
1617
1618                 lwkt_serialize_exit(&sc->vtnet_slz);
1619                 rx_npkts++;
1620                 (*ifp->if_input)(ifp, m);
1621                 lwkt_serialize_enter(&sc->vtnet_slz);
1622
1623                 /*
1624                  * The interface may have been stopped while we were
1625                  * passing the packet up the network stack.
1626                  */
1627                 if ((ifp->if_flags & IFF_RUNNING) == 0)
1628                         break;
1629         }
1630
1631         virtqueue_notify(vq, &sc->vtnet_slz);
1632
1633         if (rx_npktsp != NULL)
1634                 *rx_npktsp = rx_npkts;
1635
1636         return (count > 0 ? 0 : EAGAIN);
1637 }
1638
1639 static void
1640 vtnet_rx_intr_task(void *arg)
1641 {
1642         struct vtnet_softc *sc;
1643         struct ifnet *ifp;
1644         int more;
1645
1646         sc = arg;
1647         ifp = sc->vtnet_ifp;
1648
1649 //      lwkt_serialize_enter(&sc->vtnet_slz);
1650
1651         if ((ifp->if_flags & IFF_RUNNING) == 0) {
1652                 vtnet_enable_rx_intr(sc);
1653 //              lwkt_serialize_exit(&sc->vtnet_slz);
1654                 return;
1655         }
1656
1657         more = vtnet_rxeof(sc, sc->vtnet_rx_process_limit, NULL);
1658         if (!more && vtnet_enable_rx_intr(sc) != 0) {
1659                 vtnet_disable_rx_intr(sc);
1660                 more = 1;
1661         }
1662
1663 //      lwkt_serialize_exit(&sc->vtnet_slz);
1664
1665         if (more) {
1666                 sc->vtnet_stats.rx_task_rescheduled++;
1667                 /* XXX: Loop?! */
1668                 vtnet_rx_intr_task(sc);
1669         }
1670 }
1671
1672 static int
1673 vtnet_rx_vq_intr(void *xsc)
1674 {
1675         struct vtnet_softc *sc;
1676
1677         sc = xsc;
1678
1679         vtnet_disable_rx_intr(sc);
1680         vtnet_rx_intr_task(sc);
1681
1682         return (1);
1683 }
1684
1685 static void
1686 vtnet_txeof(struct vtnet_softc *sc)
1687 {
1688         struct virtqueue *vq;
1689         struct ifnet *ifp;
1690         struct vtnet_tx_header *txhdr;
1691         int deq;
1692
1693         vq = sc->vtnet_tx_vq;
1694         ifp = sc->vtnet_ifp;
1695         deq = 0;
1696
1697         ASSERT_SERIALIZED(&sc->vtnet_slz);
1698
1699         while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) {
1700                 deq++;
1701                 ifp->if_opackets++;
1702                 m_freem(txhdr->vth_mbuf);
1703         }
1704
1705         if (deq > 0) {
1706                 ifq_clr_oactive(&ifp->if_snd);
1707                 if (virtqueue_empty(vq))
1708                         sc->vtnet_watchdog_timer = 0;
1709         }
1710 }
1711
1712 static struct mbuf *
1713 vtnet_tx_offload(struct vtnet_softc *sc, struct mbuf *m,
1714     struct virtio_net_hdr *hdr)
1715 {
1716         struct ifnet *ifp;
1717         struct ether_header *eh;
1718         struct ether_vlan_header *evh;
1719         struct ip *ip;
1720         struct ip6_hdr *ip6;
1721         struct tcphdr *tcp;
1722         int ip_offset;
1723         uint16_t eth_type, csum_start;
1724         uint8_t ip_proto, gso_type;
1725
1726         ifp = sc->vtnet_ifp;
1727         M_ASSERTPKTHDR(m);
1728
1729         ip_offset = sizeof(struct ether_header);
1730         if (m->m_len < ip_offset) {
1731                 if ((m = m_pullup(m, ip_offset)) == NULL)
1732                         return (NULL);
1733         }
1734
1735         eh = mtod(m, struct ether_header *);
1736         eth_type = ntohs(eh->ether_type);
1737         if (eth_type == ETHERTYPE_VLAN) {
1738                 ip_offset = sizeof(struct ether_vlan_header);
1739                 if (m->m_len < ip_offset) {
1740                         if ((m = m_pullup(m, ip_offset)) == NULL)
1741                                 return (NULL);
1742                 }
1743                 evh = mtod(m, struct ether_vlan_header *);
1744                 eth_type = ntohs(evh->evl_proto);
1745         }
1746
1747         switch (eth_type) {
1748         case ETHERTYPE_IP:
1749                 if (m->m_len < ip_offset + sizeof(struct ip)) {
1750                         m = m_pullup(m, ip_offset + sizeof(struct ip));
1751                         if (m == NULL)
1752                                 return (NULL);
1753                 }
1754
1755                 ip = (struct ip *)(mtod(m, uint8_t *) + ip_offset);
1756                 ip_proto = ip->ip_p;
1757                 csum_start = ip_offset + (ip->ip_hl << 2);
1758                 gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
1759                 break;
1760
1761         case ETHERTYPE_IPV6:
1762                 if (m->m_len < ip_offset + sizeof(struct ip6_hdr)) {
1763                         m = m_pullup(m, ip_offset + sizeof(struct ip6_hdr));
1764                         if (m == NULL)
1765                                 return (NULL);
1766                 }
1767
1768                 ip6 = (struct ip6_hdr *)(mtod(m, uint8_t *) + ip_offset);
1769                 /*
1770                  * XXX Assume no extension headers are present. Presently,
1771                  * this will always be true in the case of TSO, and FreeBSD
1772                  * does not perform checksum offloading of IPv6 yet.
1773                  */
1774                 ip_proto = ip6->ip6_nxt;
1775                 csum_start = ip_offset + sizeof(struct ip6_hdr);
1776                 gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
1777                 break;
1778
1779         default:
1780                 return (m);
1781         }
1782
1783         if (m->m_pkthdr.csum_flags & VTNET_CSUM_OFFLOAD) {
1784                 hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM;
1785                 hdr->csum_start = csum_start;
1786                 hdr->csum_offset = m->m_pkthdr.csum_data;
1787
1788                 sc->vtnet_stats.tx_csum_offloaded++;
1789         }
1790
1791         if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1792                 if (ip_proto != IPPROTO_TCP)
1793                         return (m);
1794
1795                 if (m->m_len < csum_start + sizeof(struct tcphdr)) {
1796                         m = m_pullup(m, csum_start + sizeof(struct tcphdr));
1797                         if (m == NULL)
1798                                 return (NULL);
1799                 }
1800
1801                 tcp = (struct tcphdr *)(mtod(m, uint8_t *) + csum_start);
1802                 hdr->gso_type = gso_type;
1803                 hdr->hdr_len = csum_start + (tcp->th_off << 2);
1804                 hdr->gso_size = m->m_pkthdr.tso_segsz;
1805
1806                 if (tcp->th_flags & TH_CWR) {
1807                         /*
1808                          * Drop if we did not negotiate VIRTIO_NET_F_HOST_ECN.
1809                          * ECN support is only configurable globally with the
1810                          * net.inet.tcp.ecn.enable sysctl knob.
1811                          */
1812                         if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) {
1813                                 if_printf(ifp, "TSO with ECN not supported "
1814                                     "by host\n");
1815                                 m_freem(m);
1816                                 return (NULL);
1817                         }
1818
1819                         hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN;
1820                 }
1821
1822                 sc->vtnet_stats.tx_tso_offloaded++;
1823         }
1824
1825         return (m);
1826 }
1827
1828 static int
1829 vtnet_enqueue_txbuf(struct vtnet_softc *sc, struct mbuf **m_head,
1830     struct vtnet_tx_header *txhdr)
1831 {
1832         struct sglist sg;
1833         struct sglist_seg segs[VTNET_MAX_TX_SEGS];
1834         struct virtqueue *vq;
1835         struct mbuf *m;
1836         int collapsed, error;
1837
1838         vq = sc->vtnet_tx_vq;
1839         m = *m_head;
1840         collapsed = 0;
1841
1842         sglist_init(&sg, VTNET_MAX_TX_SEGS, segs);
1843         error = sglist_append(&sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size);
1844         KASSERT(error == 0 && sg.sg_nseg == 1,
1845             ("cannot add header to sglist"));
1846
1847 again:
1848         error = sglist_append_mbuf(&sg, m);
1849         if (error) {
1850                 if (collapsed)
1851                         goto fail;
1852
1853                 //m = m_collapse(m, MB_DONTWAIT, VTNET_MAX_TX_SEGS - 1);
1854                 m = m_defrag(m, MB_DONTWAIT);
1855                 if (m == NULL)
1856                         goto fail;
1857
1858                 *m_head = m;
1859                 collapsed = 1;
1860                 goto again;
1861         }
1862
1863         txhdr->vth_mbuf = m;
1864
1865         return (virtqueue_enqueue(vq, txhdr, &sg, sg.sg_nseg, 0));
1866
1867 fail:
1868         m_freem(*m_head);
1869         *m_head = NULL;
1870
1871         return (ENOBUFS);
1872 }
1873
1874 static struct mbuf *
1875 vtnet_vlan_tag_insert(struct mbuf *m)
1876 {
1877         struct mbuf *n;
1878         struct ether_vlan_header *evl;
1879
1880         if (M_WRITABLE(m) == 0) {
1881                 n = m_dup(m, MB_DONTWAIT);
1882                 m_freem(m);
1883                 if ((m = n) == NULL)
1884                         return (NULL);
1885         }
1886
1887         M_PREPEND(m, ETHER_VLAN_ENCAP_LEN, MB_DONTWAIT);
1888         if (m == NULL)
1889                 return (NULL);
1890         if (m->m_len < sizeof(struct ether_vlan_header)) {
1891                 m = m_pullup(m, sizeof(struct ether_vlan_header));
1892                 if (m == NULL)
1893                         return (NULL);
1894         }
1895
1896         /* Insert 802.1Q header into the existing Ethernet header. */
1897         evl = mtod(m, struct ether_vlan_header *);
1898         bcopy((char *) evl + ETHER_VLAN_ENCAP_LEN,
1899               (char *) evl, ETHER_HDR_LEN - ETHER_TYPE_LEN);
1900         evl->evl_encap_proto = htons(ETHERTYPE_VLAN);
1901         evl->evl_tag = htons(m->m_pkthdr.ether_vlantag);
1902         m->m_flags &= ~M_VLANTAG;
1903
1904         return (m);
1905 }
1906
1907 static int
1908 vtnet_encap(struct vtnet_softc *sc, struct mbuf **m_head)
1909 {
1910         struct vtnet_tx_header *txhdr;
1911         struct virtio_net_hdr *hdr;
1912         struct mbuf *m;
1913         int error;
1914
1915         txhdr = &sc->txhdrarea[sc->txhdridx];
1916         memset(txhdr, 0, sizeof(struct vtnet_tx_header));
1917
1918         /*
1919          * Always use the non-mergeable header to simplify things. When
1920          * the mergeable feature is negotiated, the num_buffers field
1921          * must be set to zero. We use vtnet_hdr_size later to enqueue
1922          * the correct header size to the host.
1923          */
1924         hdr = &txhdr->vth_uhdr.hdr;
1925         m = *m_head;
1926
1927         error = ENOBUFS;
1928
1929         if (m->m_flags & M_VLANTAG) {
1930                 //m = ether_vlanencap(m, m->m_pkthdr.ether_vtag);
1931                 m = vtnet_vlan_tag_insert(m);
1932                 if ((*m_head = m) == NULL)
1933                         goto fail;
1934                 m->m_flags &= ~M_VLANTAG;
1935         }
1936
1937         if (m->m_pkthdr.csum_flags != 0) {
1938                 m = vtnet_tx_offload(sc, m, hdr);
1939                 if ((*m_head = m) == NULL)
1940                         goto fail;
1941         }
1942
1943         error = vtnet_enqueue_txbuf(sc, m_head, txhdr);
1944         if (error == 0)
1945                 sc->txhdridx =
1946                     (sc->txhdridx + 1) % ((sc->vtnet_tx_size + 3) / 2);
1947 fail:
1948         return (error);
1949 }
1950
1951 static void
1952 vtnet_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
1953 {
1954         struct vtnet_softc *sc;
1955
1956         sc = ifp->if_softc;
1957
1958         ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq);
1959         lwkt_serialize_enter(&sc->vtnet_slz);
1960         vtnet_start_locked(ifp, ifsq);
1961         lwkt_serialize_exit(&sc->vtnet_slz);
1962 }
1963
1964 static void
1965 vtnet_start_locked(struct ifnet *ifp, struct ifaltq_subque *ifsq)
1966 {
1967         struct vtnet_softc *sc;
1968         struct virtqueue *vq;
1969         struct mbuf *m0;
1970         int enq;
1971
1972         sc = ifp->if_softc;
1973         vq = sc->vtnet_tx_vq;
1974         enq = 0;
1975
1976         ASSERT_SERIALIZED(&sc->vtnet_slz);
1977
1978         if ((ifp->if_flags & (IFF_RUNNING)) !=
1979             IFF_RUNNING || ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0))
1980                 return;
1981
1982 #ifdef VTNET_TX_INTR_MODERATION
1983         if (virtqueue_nused(vq) >= sc->vtnet_tx_size / 2)
1984                 vtnet_txeof(sc);
1985 #endif
1986
1987         while (!ifsq_is_empty(ifsq)) {
1988                 if (virtqueue_full(vq)) {
1989                         ifq_set_oactive(&ifp->if_snd);
1990                         break;
1991                 }
1992
1993                 m0 = ifq_dequeue(&ifp->if_snd);
1994                 if (m0 == NULL)
1995                         break;
1996
1997                 if (vtnet_encap(sc, &m0) != 0) {
1998                         if (m0 == NULL)
1999                                 break;
2000                         ifq_prepend(&ifp->if_snd, m0);
2001                         ifq_set_oactive(&ifp->if_snd);
2002                         break;
2003                 }
2004
2005                 enq++;
2006                 ETHER_BPF_MTAP(ifp, m0);
2007         }
2008
2009         if (enq > 0) {
2010                 virtqueue_notify(vq, &sc->vtnet_slz);
2011                 sc->vtnet_watchdog_timer = VTNET_WATCHDOG_TIMEOUT;
2012         }
2013 }
2014
2015 static void
2016 vtnet_tick(void *xsc)
2017 {
2018         struct vtnet_softc *sc;
2019
2020         sc = xsc;
2021
2022 #if 0
2023         ASSERT_SERIALIZED(&sc->vtnet_slz);
2024 #ifdef VTNET_DEBUG
2025         virtqueue_dump(sc->vtnet_rx_vq);
2026         virtqueue_dump(sc->vtnet_tx_vq);
2027 #endif
2028
2029         vtnet_watchdog(sc);
2030         callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc);
2031 #endif
2032 }
2033
2034 static void
2035 vtnet_tx_intr_task(void *arg)
2036 {
2037         struct vtnet_softc *sc;
2038         struct ifnet *ifp;
2039         struct ifaltq_subque *ifsq;
2040
2041         sc = arg;
2042         ifp = sc->vtnet_ifp;
2043         ifsq = ifq_get_subq_default(&ifp->if_snd);
2044
2045 //      lwkt_serialize_enter(&sc->vtnet_slz);
2046
2047         if ((ifp->if_flags & IFF_RUNNING) == 0) {
2048                 vtnet_enable_tx_intr(sc);
2049 //              lwkt_serialize_exit(&sc->vtnet_slz);
2050                 return;
2051         }
2052
2053         vtnet_txeof(sc);
2054
2055         if (!ifsq_is_empty(ifsq))
2056                 vtnet_start_locked(ifp, ifsq);
2057
2058         if (vtnet_enable_tx_intr(sc) != 0) {
2059                 vtnet_disable_tx_intr(sc);
2060                 sc->vtnet_stats.tx_task_rescheduled++;
2061 //              lwkt_serialize_exit(&sc->vtnet_slz);
2062                 vtnet_tx_intr_task(sc);
2063                 /* XXX: loop?! */
2064                 return;
2065         }
2066
2067 //      lwkt_serialize_exit(&sc->vtnet_slz);
2068 }
2069
2070 static int
2071 vtnet_tx_vq_intr(void *xsc)
2072 {
2073         struct vtnet_softc *sc;
2074
2075         sc = xsc;
2076
2077         vtnet_disable_tx_intr(sc);
2078         vtnet_tx_intr_task(sc);
2079
2080         return (1);
2081 }
2082
2083 static void
2084 vtnet_stop(struct vtnet_softc *sc)
2085 {
2086         device_t dev;
2087         struct ifnet *ifp;
2088
2089         dev = sc->vtnet_dev;
2090         ifp = sc->vtnet_ifp;
2091
2092         ASSERT_SERIALIZED(&sc->vtnet_slz);
2093
2094         sc->vtnet_watchdog_timer = 0;
2095         callout_stop(&sc->vtnet_tick_ch);
2096         ifq_clr_oactive(&ifp->if_snd);
2097         ifp->if_flags &= ~(IFF_RUNNING);
2098
2099         vtnet_disable_rx_intr(sc);
2100         vtnet_disable_tx_intr(sc);
2101
2102         /*
2103          * Stop the host VirtIO adapter. Note this will reset the host
2104          * adapter's state back to the pre-initialized state, so in
2105          * order to make the device usable again, we must drive it
2106          * through virtio_reinit() and virtio_reinit_complete().
2107          */
2108         virtio_stop(dev);
2109
2110         sc->vtnet_flags &= ~VTNET_FLAG_LINK;
2111
2112         vtnet_free_rx_mbufs(sc);
2113         vtnet_free_tx_mbufs(sc);
2114 }
2115
2116 static int
2117 vtnet_reinit(struct vtnet_softc *sc)
2118 {
2119         struct ifnet *ifp;
2120         uint64_t features;
2121
2122         ifp = sc->vtnet_ifp;
2123         features = sc->vtnet_features;
2124
2125         /*
2126          * Re-negotiate with the host, removing any disabled receive
2127          * features. Transmit features are disabled only on our side
2128          * via if_capenable and if_hwassist.
2129          */
2130
2131         if (ifp->if_capabilities & IFCAP_RXCSUM) {
2132                 if ((ifp->if_capenable & IFCAP_RXCSUM) == 0)
2133                         features &= ~VIRTIO_NET_F_GUEST_CSUM;
2134         }
2135
2136         if (ifp->if_capabilities & IFCAP_LRO) {
2137                 if ((ifp->if_capenable & IFCAP_LRO) == 0)
2138                         features &= ~VTNET_LRO_FEATURES;
2139         }
2140
2141         if (ifp->if_capabilities & IFCAP_VLAN_HWFILTER) {
2142                 if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
2143                         features &= ~VIRTIO_NET_F_CTRL_VLAN;
2144         }
2145
2146         return (virtio_reinit(sc->vtnet_dev, features));
2147 }
2148
2149 static void
2150 vtnet_init_locked(struct vtnet_softc *sc)
2151 {
2152         device_t dev;
2153         struct ifnet *ifp;
2154         int error;
2155
2156         dev = sc->vtnet_dev;
2157         ifp = sc->vtnet_ifp;
2158
2159         ASSERT_SERIALIZED(&sc->vtnet_slz);
2160
2161         if (ifp->if_flags & IFF_RUNNING)
2162                 return;
2163
2164         /* Stop host's adapter, cancel any pending I/O. */
2165         vtnet_stop(sc);
2166
2167         /* Reinitialize the host device. */
2168         error = vtnet_reinit(sc);
2169         if (error) {
2170                 device_printf(dev,
2171                     "reinitialization failed, stopping device...\n");
2172                 vtnet_stop(sc);
2173                 return;
2174         }
2175
2176         /* Update host with assigned MAC address. */
2177         bcopy(IF_LLADDR(ifp), sc->vtnet_hwaddr, ETHER_ADDR_LEN);
2178         vtnet_set_hwaddr(sc);
2179
2180         ifp->if_hwassist = 0;
2181         if (ifp->if_capenable & IFCAP_TXCSUM)
2182                 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;
2183         if (ifp->if_capenable & IFCAP_TSO4)
2184                 ifp->if_hwassist |= CSUM_TSO;
2185
2186         error = vtnet_init_rx_vq(sc);
2187         if (error) {
2188                 device_printf(dev,
2189                     "cannot allocate mbufs for Rx virtqueue\n");
2190                 vtnet_stop(sc);
2191                 return;
2192         }
2193
2194         if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
2195                 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
2196                         /* Restore promiscuous and all-multicast modes. */
2197                         vtnet_rx_filter(sc);
2198
2199                         /* Restore filtered MAC addresses. */
2200                         vtnet_rx_filter_mac(sc);
2201                 }
2202
2203                 /* Restore VLAN filters. */
2204                 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
2205                         vtnet_rx_filter_vlan(sc);
2206         }
2207
2208         {
2209                 vtnet_enable_rx_intr(sc);
2210                 vtnet_enable_tx_intr(sc);
2211         }
2212
2213         ifp->if_flags |= IFF_RUNNING;
2214         ifq_clr_oactive(&ifp->if_snd);
2215
2216         virtio_reinit_complete(dev);
2217
2218         vtnet_update_link_status(sc);
2219         callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc);
2220 }
2221
2222 static void
2223 vtnet_init(void *xsc)
2224 {
2225         struct vtnet_softc *sc;
2226
2227         sc = xsc;
2228
2229         lwkt_serialize_enter(&sc->vtnet_slz);
2230         vtnet_init_locked(sc);
2231         lwkt_serialize_exit(&sc->vtnet_slz);
2232 }
2233
2234 static void
2235 vtnet_exec_ctrl_cmd(struct vtnet_softc *sc, void *cookie,
2236     struct sglist *sg, int readable, int writable)
2237 {
2238         struct virtqueue *vq;
2239         void *c;
2240
2241         vq = sc->vtnet_ctrl_vq;
2242
2243         ASSERT_SERIALIZED(&sc->vtnet_slz);
2244         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ,
2245             ("no control virtqueue"));
2246         KASSERT(virtqueue_empty(vq),
2247             ("control command already enqueued"));
2248
2249         if (virtqueue_enqueue(vq, cookie, sg, readable, writable) != 0)
2250                 return;
2251
2252         virtqueue_notify(vq, &sc->vtnet_slz);
2253
2254         /*
2255          * Poll until the command is complete. Previously, we would
2256          * sleep until the control virtqueue interrupt handler woke
2257          * us up, but dropping the VTNET_MTX leads to serialization
2258          * difficulties.
2259          *
2260          * Furthermore, it appears QEMU/KVM only allocates three MSIX
2261          * vectors. Two of those vectors are needed for the Rx and Tx
2262          * virtqueues. We do not support sharing both a Vq and config
2263          * changed notification on the same MSIX vector.
2264          */
2265         c = virtqueue_poll(vq, NULL);
2266         KASSERT(c == cookie, ("unexpected control command response"));
2267 }
2268
2269 static void
2270 vtnet_rx_filter(struct vtnet_softc *sc)
2271 {
2272         device_t dev;
2273         struct ifnet *ifp;
2274
2275         dev = sc->vtnet_dev;
2276         ifp = sc->vtnet_ifp;
2277
2278         ASSERT_SERIALIZED(&sc->vtnet_slz);
2279         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
2280             ("CTRL_RX feature not negotiated"));
2281
2282         if (vtnet_set_promisc(sc, ifp->if_flags & IFF_PROMISC) != 0)
2283                 device_printf(dev, "cannot %s promiscuous mode\n",
2284                     ifp->if_flags & IFF_PROMISC ? "enable" : "disable");
2285
2286         if (vtnet_set_allmulti(sc, ifp->if_flags & IFF_ALLMULTI) != 0)
2287                 device_printf(dev, "cannot %s all-multicast mode\n",
2288                     ifp->if_flags & IFF_ALLMULTI ? "enable" : "disable");
2289 }
2290
2291 static int
2292 vtnet_ctrl_rx_cmd(struct vtnet_softc *sc, int cmd, int on)
2293 {
2294         struct virtio_net_ctrl_hdr hdr;
2295         struct sglist_seg segs[3];
2296         struct sglist sg;
2297         uint8_t onoff, ack;
2298         int error;
2299
2300         if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0)
2301                 return (ENOTSUP);
2302
2303         error = 0;
2304
2305         hdr.class = VIRTIO_NET_CTRL_RX;
2306         hdr.cmd = cmd;
2307         onoff = !!on;
2308         ack = VIRTIO_NET_ERR;
2309
2310         sglist_init(&sg, 3, segs);
2311         error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
2312         error |= sglist_append(&sg, &onoff, sizeof(uint8_t));
2313         error |= sglist_append(&sg, &ack, sizeof(uint8_t));
2314         KASSERT(error == 0 && sg.sg_nseg == 3,
2315             ("error adding Rx filter message to sglist"));
2316
2317         vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
2318
2319         return (ack == VIRTIO_NET_OK ? 0 : EIO);
2320 }
2321
2322 static int
2323 vtnet_set_promisc(struct vtnet_softc *sc, int on)
2324 {
2325
2326         return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_PROMISC, on));
2327 }
2328
2329 static int
2330 vtnet_set_allmulti(struct vtnet_softc *sc, int on)
2331 {
2332
2333         return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, on));
2334 }
2335
2336 static void
2337 vtnet_rx_filter_mac(struct vtnet_softc *sc)
2338 {
2339         struct virtio_net_ctrl_hdr hdr;
2340         struct vtnet_mac_filter *filter;
2341         struct sglist_seg segs[4];
2342         struct sglist sg;
2343         struct ifnet *ifp;
2344         struct ifaddr *ifa;
2345         struct ifaddr_container *ifac;
2346         struct ifmultiaddr *ifma;
2347         int ucnt, mcnt, promisc, allmulti, error;
2348         uint8_t ack;
2349
2350         ifp = sc->vtnet_ifp;
2351         ucnt = 0;
2352         mcnt = 0;
2353         promisc = 0;
2354         allmulti = 0;
2355         error = 0;
2356
2357         ASSERT_SERIALIZED(&sc->vtnet_slz);
2358         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
2359             ("CTRL_RX feature not negotiated"));
2360
2361         /*
2362          * Allocate the MAC filtering table. Note we could do this
2363          * at attach time, but it is probably not worth keeping it
2364          * around for an infrequent occurrence.
2365          */
2366         filter = kmalloc(sizeof(struct vtnet_mac_filter), M_DEVBUF,
2367             M_INTWAIT | M_ZERO);
2368
2369         /* Unicast MAC addresses: */
2370         //if_addr_rlock(ifp);
2371         TAILQ_FOREACH(ifac, &ifp->if_addrheads[mycpuid], ifa_link) {
2372                 ifa = ifac->ifa;
2373                 if (ifa->ifa_addr->sa_family != AF_LINK)
2374                         continue;
2375                 else if (ucnt == VTNET_MAX_MAC_ENTRIES)
2376                         break;
2377
2378                 bcopy(LLADDR((struct sockaddr_dl *)ifa->ifa_addr),
2379                     &filter->vmf_unicast.macs[ucnt], ETHER_ADDR_LEN);
2380                 ucnt++;
2381         }
2382         //if_addr_runlock(ifp);
2383
2384         if (ucnt >= VTNET_MAX_MAC_ENTRIES) {
2385                 promisc = 1;
2386                 filter->vmf_unicast.nentries = 0;
2387
2388                 if_printf(ifp, "more than %d MAC addresses assigned, "
2389                     "falling back to promiscuous mode\n",
2390                     VTNET_MAX_MAC_ENTRIES);
2391         } else
2392                 filter->vmf_unicast.nentries = ucnt;
2393
2394         /* Multicast MAC addresses: */
2395         //if_maddr_rlock(ifp);
2396         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2397                 if (ifma->ifma_addr->sa_family != AF_LINK)
2398                         continue;
2399                 else if (mcnt == VTNET_MAX_MAC_ENTRIES)
2400                         break;
2401
2402                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2403                     &filter->vmf_multicast.macs[mcnt], ETHER_ADDR_LEN);
2404                 mcnt++;
2405         }
2406         //if_maddr_runlock(ifp);
2407
2408         if (mcnt >= VTNET_MAX_MAC_ENTRIES) {
2409                 allmulti = 1;
2410                 filter->vmf_multicast.nentries = 0;
2411
2412                 if_printf(ifp, "more than %d multicast MAC addresses "
2413                     "assigned, falling back to all-multicast mode\n",
2414                     VTNET_MAX_MAC_ENTRIES);
2415         } else
2416                 filter->vmf_multicast.nentries = mcnt;
2417
2418         if (promisc && allmulti)
2419                 goto out;
2420
2421         hdr.class = VIRTIO_NET_CTRL_MAC;
2422         hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
2423         ack = VIRTIO_NET_ERR;
2424
2425         sglist_init(&sg, 4, segs);
2426         error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
2427         error |= sglist_append(&sg, &filter->vmf_unicast,
2428             sizeof(struct vtnet_mac_table));
2429         error |= sglist_append(&sg, &filter->vmf_multicast,
2430             sizeof(struct vtnet_mac_table));
2431         error |= sglist_append(&sg, &ack, sizeof(uint8_t));
2432         KASSERT(error == 0 && sg.sg_nseg == 4,
2433             ("error adding MAC filtering message to sglist"));
2434
2435         vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
2436
2437         if (ack != VIRTIO_NET_OK)
2438                 if_printf(ifp, "error setting host MAC filter table\n");
2439
2440 out:
2441         kfree(filter, M_DEVBUF);
2442
2443         if (promisc)
2444                 if (vtnet_set_promisc(sc, 1) != 0)
2445                         if_printf(ifp, "cannot enable promiscuous mode\n");
2446         if (allmulti)
2447                 if (vtnet_set_allmulti(sc, 1) != 0)
2448                         if_printf(ifp, "cannot enable all-multicast mode\n");
2449 }
2450
2451 static int
2452 vtnet_exec_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
2453 {
2454         struct virtio_net_ctrl_hdr hdr;
2455         struct sglist_seg segs[3];
2456         struct sglist sg;
2457         uint8_t ack;
2458         int error;
2459
2460         hdr.class = VIRTIO_NET_CTRL_VLAN;
2461         hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
2462         ack = VIRTIO_NET_ERR;
2463         error = 0;
2464
2465         sglist_init(&sg, 3, segs);
2466         error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
2467         error |= sglist_append(&sg, &tag, sizeof(uint16_t));
2468         error |= sglist_append(&sg, &ack, sizeof(uint8_t));
2469         KASSERT(error == 0 && sg.sg_nseg == 3,
2470             ("error adding VLAN control message to sglist"));
2471
2472         vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
2473
2474         return (ack == VIRTIO_NET_OK ? 0 : EIO);
2475 }
2476
2477 static void
2478 vtnet_rx_filter_vlan(struct vtnet_softc *sc)
2479 {
2480         device_t dev;
2481         uint32_t w, mask;
2482         uint16_t tag;
2483         int i, nvlans, error;
2484
2485         ASSERT_SERIALIZED(&sc->vtnet_slz);
2486         KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER,
2487             ("VLAN_FILTER feature not negotiated"));
2488
2489         dev = sc->vtnet_dev;
2490         nvlans = sc->vtnet_nvlans;
2491         error = 0;
2492
2493         /* Enable filtering for each configured VLAN. */
2494         for (i = 0; i < VTNET_VLAN_SHADOW_SIZE && nvlans > 0; i++) {
2495                 w = sc->vtnet_vlan_shadow[i];
2496                 for (mask = 1, tag = i * 32; w != 0; mask <<= 1, tag++) {
2497                         if ((w & mask) != 0) {
2498                                 w &= ~mask;
2499                                 nvlans--;
2500                                 if (vtnet_exec_vlan_filter(sc, 1, tag) != 0)
2501                                         error++;
2502                         }
2503                 }
2504         }
2505
2506         KASSERT(nvlans == 0, ("VLAN count incorrect"));
2507         if (error)
2508                 device_printf(dev, "cannot restore VLAN filter table\n");
2509 }
2510
2511 static void
2512 vtnet_set_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
2513 {
2514         struct ifnet *ifp;
2515         int idx, bit;
2516
2517         KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER,
2518             ("VLAN_FILTER feature not negotiated"));
2519
2520         if ((tag == 0) || (tag > 4095))
2521                 return;
2522
2523         ifp = sc->vtnet_ifp;
2524         idx = (tag >> 5) & 0x7F;
2525         bit = tag & 0x1F;
2526
2527         lwkt_serialize_enter(&sc->vtnet_slz);
2528
2529         /* Update shadow VLAN table. */
2530         if (add) {
2531                 sc->vtnet_nvlans++;
2532                 sc->vtnet_vlan_shadow[idx] |= (1 << bit);
2533         } else {
2534                 sc->vtnet_nvlans--;
2535                 sc->vtnet_vlan_shadow[idx] &= ~(1 << bit);
2536         }
2537
2538         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
2539                 if (vtnet_exec_vlan_filter(sc, add, tag) != 0) {
2540                         device_printf(sc->vtnet_dev,
2541                             "cannot %s VLAN %d %s the host filter table\n",
2542                             add ? "add" : "remove", tag,
2543                             add ? "to" : "from");
2544                 }
2545         }
2546
2547         lwkt_serialize_exit(&sc->vtnet_slz);
2548 }
2549
2550 static void
2551 vtnet_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
2552 {
2553
2554         if (ifp->if_softc != arg)
2555                 return;
2556
2557         vtnet_set_vlan_filter(arg, 1, tag);
2558 }
2559
2560 static void
2561 vtnet_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
2562 {
2563
2564         if (ifp->if_softc != arg)
2565                 return;
2566
2567         vtnet_set_vlan_filter(arg, 0, tag);
2568 }
2569
2570 static int
2571 vtnet_ifmedia_upd(struct ifnet *ifp)
2572 {
2573         struct vtnet_softc *sc;
2574         struct ifmedia *ifm;
2575
2576         sc = ifp->if_softc;
2577         ifm = &sc->vtnet_media;
2578
2579         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
2580                 return (EINVAL);
2581
2582         return (0);
2583 }
2584
2585 static void
2586 vtnet_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
2587 {
2588         struct vtnet_softc *sc;
2589
2590         sc = ifp->if_softc;
2591
2592         ifmr->ifm_status = IFM_AVALID;
2593         ifmr->ifm_active = IFM_ETHER;
2594
2595         lwkt_serialize_enter(&sc->vtnet_slz);
2596         if (vtnet_is_link_up(sc) != 0) {
2597                 ifmr->ifm_status |= IFM_ACTIVE;
2598                 ifmr->ifm_active |= VTNET_MEDIATYPE;
2599         } else
2600                 ifmr->ifm_active |= IFM_NONE;
2601         lwkt_serialize_exit(&sc->vtnet_slz);
2602 }
2603
2604 static void
2605 vtnet_add_statistics(struct vtnet_softc *sc)
2606 {
2607         device_t dev;
2608         struct vtnet_statistics *stats;
2609         //struct sysctl_ctx_list *ctx;
2610         //struct sysctl_oid *tree;
2611         //struct sysctl_oid_list *child;
2612         int error = 0;
2613
2614         dev = sc->vtnet_dev;
2615         stats = &sc->vtnet_stats;
2616         sysctl_ctx_init(&sc->vtnet_sysctl_ctx);
2617         sc->vtnet_sysctl_tree = SYSCTL_ADD_NODE(&sc->vtnet_sysctl_ctx,
2618                                                 SYSCTL_STATIC_CHILDREN(_hw),
2619                                                 OID_AUTO,
2620                                                 device_get_nameunit(dev),
2621                                                 CTLFLAG_RD, 0, "");
2622
2623         if (sc->vtnet_sysctl_tree == NULL) {
2624                 device_printf(dev, "can't add sysctl node\n");
2625                 error = ENXIO;
2626         }
2627
2628
2629         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx,
2630                          SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO,
2631                          "mbuf_alloc_failed", CTLFLAG_RD, &stats->mbuf_alloc_failed,
2632                          "Mbuf cluster allocation failures");
2633         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx,
2634                          SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO,
2635                          "rx_frame_too_large", CTLFLAG_RD, &stats->rx_frame_too_large,
2636                          "Received frame larger than the mbuf chain");
2637         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx,SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_enq_replacement_failed",
2638             CTLFLAG_RD, &stats->rx_enq_replacement_failed,
2639             "Enqueuing the replacement receive mbuf failed");
2640         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_mergeable_failed",
2641             CTLFLAG_RD, &stats->rx_mergeable_failed,
2642             "Mergeable buffers receive failures");
2643         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_bad_ethtype",
2644             CTLFLAG_RD, &stats->rx_csum_bad_ethtype,
2645             "Received checksum offloaded buffer with unsupported "
2646             "Ethernet type");
2647         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_bad_start",
2648             CTLFLAG_RD, &stats->rx_csum_bad_start,
2649             "Received checksum offloaded buffer with incorrect start offset");
2650         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_bad_ipproto",
2651             CTLFLAG_RD, &stats->rx_csum_bad_ipproto,
2652             "Received checksum offloaded buffer with incorrect IP protocol");
2653         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_bad_offset",
2654             CTLFLAG_RD, &stats->rx_csum_bad_offset,
2655             "Received checksum offloaded buffer with incorrect offset");
2656         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_failed",
2657             CTLFLAG_RD, &stats->rx_csum_failed,
2658             "Received buffer checksum offload failed");
2659         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_offloaded",
2660             CTLFLAG_RD, &stats->rx_csum_offloaded,
2661             "Received buffer checksum offload succeeded");
2662         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_task_rescheduled",
2663             CTLFLAG_RD, &stats->rx_task_rescheduled,
2664             "Times the receive interrupt task rescheduled itself");
2665
2666         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_csum_offloaded",
2667             CTLFLAG_RD, &stats->tx_csum_offloaded,
2668             "Offloaded checksum of transmitted buffer");
2669         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_tso_offloaded",
2670             CTLFLAG_RD, &stats->tx_tso_offloaded,
2671             "Segmentation offload of transmitted buffer");
2672         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_csum_bad_ethtype",
2673             CTLFLAG_RD, &stats->tx_csum_bad_ethtype,
2674             "Aborted transmit of checksum offloaded buffer with unknown "
2675             "Ethernet type");
2676         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_tso_bad_ethtype",
2677             CTLFLAG_RD, &stats->tx_tso_bad_ethtype,
2678             "Aborted transmit of TSO buffer with unknown Ethernet type");
2679         SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_task_rescheduled",
2680             CTLFLAG_RD, &stats->tx_task_rescheduled,
2681             "Times the transmit interrupt task rescheduled itself");
2682 }
2683
2684 static int
2685 vtnet_enable_rx_intr(struct vtnet_softc *sc)
2686 {
2687
2688         return (virtqueue_enable_intr(sc->vtnet_rx_vq));
2689 }
2690
2691 static void
2692 vtnet_disable_rx_intr(struct vtnet_softc *sc)
2693 {
2694
2695         virtqueue_disable_intr(sc->vtnet_rx_vq);
2696 }
2697
2698 static int
2699 vtnet_enable_tx_intr(struct vtnet_softc *sc)
2700 {
2701
2702 #ifdef VTNET_TX_INTR_MODERATION
2703         return (0);
2704 #else
2705         return (virtqueue_enable_intr(sc->vtnet_tx_vq));
2706 #endif
2707 }
2708
2709 static void
2710 vtnet_disable_tx_intr(struct vtnet_softc *sc)
2711 {
2712
2713         virtqueue_disable_intr(sc->vtnet_tx_vq);
2714 }