sys/dev/virtual/virtio/net/if_vtnet.c

   1 /*-
   2  * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice unmodified, this list of conditions, and the following
  10  *    disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 /* Driver for VirtIO network devices. */
  28
  29 #include "opt_ifpoll.h"
  30
  31 #include <sys/cdefs.h>
  32
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35 #include <sys/kernel.h>
  36 #include <sys/sockio.h>
  37 #include <sys/mbuf.h>
  38 #include <sys/malloc.h>
  39 #include <sys/module.h>
  40 #include <sys/socket.h>
  41 #include <sys/sysctl.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/random.h>
  44 #include <sys/sglist.h>
  45 #include <sys/serialize.h>
  46 #include <sys/bus.h>
  47 #include <sys/rman.h>
  48
  49 #include <machine/limits.h>
  50
  51 #include <net/ethernet.h>
  52 #include <net/if.h>
  53 #include <net/if_arp.h>
  54 #include <net/if_dl.h>
  55 #include <net/if_types.h>
  56 #include <net/if_media.h>
  57 #include <net/vlan/if_vlan_var.h>
  58 #include <net/vlan/if_vlan_ether.h>
  59 #include <net/if_poll.h>
  60 #include <net/ifq_var.h>
  61
  62 #include <net/bpf.h>
  63
  64 #include <netinet/in_systm.h>
  65 #include <netinet/in.h>
  66 #include <netinet/ip.h>
  67 #include <netinet/ip6.h>
  68 #include <netinet/udp.h>
  69 #include <netinet/tcp.h>
  70
  71 #include <dev/virtual/virtio/virtio/virtio.h>
  72 #include <dev/virtual/virtio/virtio/virtqueue.h>
  73 #include <dev/virtual/virtio/net/virtio_net.h>
  74 #include <dev/virtual/virtio/net/if_vtnetvar.h>
  75
  76 MALLOC_DEFINE(M_VTNET, "VTNET_TX", "Outgoing VTNET TX frame header");
  77
  78 static int      vtnet_probe(device_t);
  79 static int      vtnet_attach(device_t);
  80 static int      vtnet_detach(device_t);
  81 static int      vtnet_suspend(device_t);
  82 static int      vtnet_resume(device_t);
  83 static int      vtnet_shutdown(device_t);
  84
  85 static void     vtnet_negotiate_features(struct vtnet_softc *);
  86 #ifdef IFPOLL_ENABLE
  87 static void     vtnet_npoll(struct ifnet *, struct ifpoll_info *);
  88 static void     vtnet_npoll_status(struct ifnet *);
  89 static void     vtnet_npoll_rx(struct ifnet *, void *, int);
  90 static void     vtnet_npoll_tx(struct ifnet *, void *, int);
  91 #endif
  92 static void     vtnet_serialize(struct ifnet *, enum ifnet_serialize);
  93 static void     vtnet_deserialize(struct ifnet *, enum ifnet_serialize);
  94 static int      vtnet_tryserialize(struct ifnet *, enum ifnet_serialize);
  95 #ifdef INVARIANTS
  96 static void     vtnet_serialize_assert(struct ifnet *, enum ifnet_serialize,
  97                     boolean_t);
  98 #endif  /* INVARIANTS */
  99 static int      vtnet_alloc_intrs(struct vtnet_softc *);
 100 static int      vtnet_alloc_virtqueues(struct vtnet_softc *);
 101 static int      vtnet_bind_intrs(struct vtnet_softc *);
 102 static void     vtnet_get_hwaddr(struct vtnet_softc *);
 103 static void     vtnet_set_hwaddr(struct vtnet_softc *);
 104 static int      vtnet_is_link_up(struct vtnet_softc *);
 105 static void     vtnet_update_link_status(struct vtnet_softc *);
 106 static void     vtnet_watchdog(struct ifaltq_subque *);
 107 static int      vtnet_setup_interface(struct vtnet_softc *);
 108 static int      vtnet_change_mtu(struct vtnet_softc *, int);
 109 static int      vtnet_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 110
 111 static int      vtnet_init_rx_vq(struct vtnet_softc *);
 112 static void     vtnet_free_rx_mbufs(struct vtnet_softc *);
 113 static void     vtnet_free_tx_mbufs(struct vtnet_softc *);
 114 static void     vtnet_free_ctrl_vq(struct vtnet_softc *);
 115
 116 static struct mbuf * vtnet_alloc_rxbuf(struct vtnet_softc *, int,
 117                     struct mbuf **);
 118 static int      vtnet_replace_rxbuf(struct vtnet_softc *,
 119                     struct mbuf *, int);
 120 static int      vtnet_newbuf(struct vtnet_softc *);
 121 static void     vtnet_discard_merged_rxbuf(struct vtnet_softc *, int);
 122 static void     vtnet_discard_rxbuf(struct vtnet_softc *, struct mbuf *);
 123 static int      vtnet_enqueue_rxbuf(struct vtnet_softc *, struct mbuf *);
 124 static void     vtnet_vlan_tag_remove(struct mbuf *);
 125 static int      vtnet_rx_csum(struct vtnet_softc *, struct mbuf *,
 126                     struct virtio_net_hdr *);
 127 static int      vtnet_rxeof_merged(struct vtnet_softc *, struct mbuf *, int);
 128 static int      vtnet_rxeof(struct vtnet_softc *, int, int *);
 129 static void     vtnet_rx_msix_intr(void *);
 130 static void     vtnet_rx_vq_intr(void *);
 131
 132 static void     vtnet_enqueue_txhdr(struct vtnet_softc *,
 133                     struct vtnet_tx_header *);
 134 static void     vtnet_txeof(struct vtnet_softc *);
 135 static struct mbuf * vtnet_tx_offload(struct vtnet_softc *, struct mbuf *,
 136                     struct virtio_net_hdr *);
 137 static int      vtnet_enqueue_txbuf(struct vtnet_softc *, struct mbuf **,
 138                     struct vtnet_tx_header *);
 139 static int      vtnet_encap(struct vtnet_softc *, struct mbuf **);
 140 static void     vtnet_start(struct ifnet *, struct ifaltq_subque *);
 141
 142 static void     vtnet_config_intr(void *);
 143 static void     vtnet_tx_msix_intr(void *);
 144 static void     vtnet_tx_vq_intr(void *);
 145
 146 static void     vtnet_stop(struct vtnet_softc *);
 147 static int      vtnet_virtio_reinit(struct vtnet_softc *);
 148 static void     vtnet_init(void *);
 149
 150 static void     vtnet_exec_ctrl_cmd(struct vtnet_softc *, void *,
 151                     struct sglist *, int, int);
 152
 153 static int      vtnet_ctrl_mac_cmd(struct vtnet_softc *, uint8_t *);
 154 static int      vtnet_ctrl_rx_cmd(struct vtnet_softc *, int, int);
 155 static int      vtnet_set_promisc(struct vtnet_softc *, int);
 156 static int      vtnet_set_allmulti(struct vtnet_softc *, int);
 157 static void     vtnet_rx_filter(struct vtnet_softc *sc);
 158 static void     vtnet_rx_filter_mac(struct vtnet_softc *);
 159
 160 static int      vtnet_exec_vlan_filter(struct vtnet_softc *, int, uint16_t);
 161 static void     vtnet_rx_filter_vlan(struct vtnet_softc *);
 162 static void     vtnet_update_vlan_filter(struct vtnet_softc *, int, uint16_t);
 163 static void     vtnet_register_vlan(void *, struct ifnet *, uint16_t);
 164 static void     vtnet_unregister_vlan(void *, struct ifnet *, uint16_t);
 165
 166 static int      vtnet_ifmedia_upd(struct ifnet *);
 167 static void     vtnet_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 168
 169 static void     vtnet_add_statistics(struct vtnet_softc *);
 170
 171 static int      vtnet_enable_rx_intr(struct vtnet_softc *);
 172 static int      vtnet_enable_tx_intr(struct vtnet_softc *);
 173 static void     vtnet_disable_rx_intr(struct vtnet_softc *);
 174 static void     vtnet_disable_tx_intr(struct vtnet_softc *);
 175
 176 /* Tunables. */
 177 static int vtnet_csum_disable = 0;
 178 TUNABLE_INT("hw.vtnet.csum_disable", &vtnet_csum_disable);
 179 static int vtnet_tso_disable = 1;
 180 TUNABLE_INT("hw.vtnet.tso_disable", &vtnet_tso_disable);
 181 static int vtnet_lro_disable = 0;
 182 TUNABLE_INT("hw.vtnet.lro_disable", &vtnet_lro_disable);
 183
 184 /*
 185  * Reducing the number of transmit completed interrupts can
 186  * improve performance. To do so, the define below keeps the
 187  * Tx vq interrupt disabled and adds calls to vtnet_txeof()
 188  * in the start path. The price to pay for this is the m_free'ing
 189  * of transmitted mbufs may be delayed.
 190  */
 191 #define VTNET_TX_INTR_MODERATION
 192
 193 static struct virtio_feature_desc vtnet_feature_desc[] = {
 194         { VIRTIO_NET_F_CSUM,            "TxChecksum"    },
 195         { VIRTIO_NET_F_GUEST_CSUM,      "RxChecksum"    },
 196         { VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, "DynOffload"        },
 197         { VIRTIO_NET_F_MAC,             "MacAddress"    },
 198         { VIRTIO_NET_F_GSO,             "TxAllGSO"      },
 199         { VIRTIO_NET_F_GUEST_TSO4,      "RxTSOv4"       },
 200         { VIRTIO_NET_F_GUEST_TSO6,      "RxTSOv6"       },
 201         { VIRTIO_NET_F_GUEST_ECN,       "RxECN"         },
 202         { VIRTIO_NET_F_GUEST_UFO,       "RxUFO"         },
 203         { VIRTIO_NET_F_HOST_TSO4,       "TxTSOv4"       },
 204         { VIRTIO_NET_F_HOST_TSO6,       "TxTSOv6"       },
 205         { VIRTIO_NET_F_HOST_ECN,        "TxTSOECN"      },
 206         { VIRTIO_NET_F_HOST_UFO,        "TxUFO"         },
 207         { VIRTIO_NET_F_MRG_RXBUF,       "MrgRxBuf"      },
 208         { VIRTIO_NET_F_STATUS,          "Status"        },
 209         { VIRTIO_NET_F_CTRL_VQ,         "ControlVq"     },
 210         { VIRTIO_NET_F_CTRL_RX,         "RxMode"        },
 211         { VIRTIO_NET_F_CTRL_VLAN,       "VLanFilter"    },
 212         { VIRTIO_NET_F_CTRL_RX_EXTRA,   "RxModeExtra"   },
 213         { VIRTIO_NET_F_GUEST_ANNOUNCE,  "GuestAnnounce" },
 214         { VIRTIO_NET_F_MQ,              "Multiqueue"    },
 215         { VIRTIO_NET_F_CTRL_MAC_ADDR,   "SetMacAddress" },
 216         { 0, NULL }
 217 };
 218
 219 static device_method_t vtnet_methods[] = {
 220         /* Device methods. */
 221         DEVMETHOD(device_probe,         vtnet_probe),
 222         DEVMETHOD(device_attach,        vtnet_attach),
 223         DEVMETHOD(device_detach,        vtnet_detach),
 224         DEVMETHOD(device_suspend,       vtnet_suspend),
 225         DEVMETHOD(device_resume,        vtnet_resume),
 226         DEVMETHOD(device_shutdown,      vtnet_shutdown),
 227
 228         DEVMETHOD_END
 229 };
 230
 231 static driver_t vtnet_driver = {
 232         "vtnet",
 233         vtnet_methods,
 234         sizeof(struct vtnet_softc)
 235 };
 236
 237 static devclass_t vtnet_devclass;
 238
 239 DRIVER_MODULE(vtnet, virtio_pci, vtnet_driver, vtnet_devclass, NULL, NULL);
 240 MODULE_VERSION(vtnet, 1);
 241 MODULE_DEPEND(vtnet, virtio, 1, 1, 1);
 242
 243 static int
 244 vtnet_probe(device_t dev)
 245 {
 246         if (virtio_get_device_type(dev) != VIRTIO_ID_NETWORK)
 247                 return (ENXIO);
 248
 249         device_set_desc(dev, "VirtIO Networking Adapter");
 250
 251         return (BUS_PROBE_DEFAULT);
 252 }
 253
 254 static int
 255 vtnet_attach(device_t dev)
 256 {
 257         struct vtnet_softc *sc;
 258         int i, error;
 259
 260         sc = device_get_softc(dev);
 261         sc->vtnet_dev = dev;
 262
 263         lwkt_serialize_init(&sc->vtnet_slz);
 264         lwkt_serialize_init(&sc->vtnet_rx_slz);
 265         lwkt_serialize_init(&sc->vtnet_tx_slz);
 266         sc->serializes[0] = &sc->vtnet_slz;
 267         sc->serializes[1] = &sc->vtnet_rx_slz;
 268         sc->serializes[2] = &sc->vtnet_tx_slz;
 269
 270         ifmedia_init(&sc->vtnet_media, IFM_IMASK, vtnet_ifmedia_upd,
 271                      vtnet_ifmedia_sts);
 272         ifmedia_add(&sc->vtnet_media, VTNET_MEDIATYPE, 0, NULL);
 273         ifmedia_set(&sc->vtnet_media, VTNET_MEDIATYPE);
 274
 275         vtnet_add_statistics(sc);
 276         SLIST_INIT(&sc->vtnet_txhdr_free);
 277
 278         /* Register our feature descriptions. */
 279         virtio_set_feature_desc(dev, vtnet_feature_desc);
 280         vtnet_negotiate_features(sc);
 281
 282         if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
 283                 sc->vtnet_flags |= VTNET_FLAG_INDIRECT;
 284
 285         if (virtio_with_feature(dev, VIRTIO_NET_F_MAC)) {
 286                 /* This feature should always be negotiated. */
 287                 sc->vtnet_flags |= VTNET_FLAG_MAC;
 288         }
 289
 290         if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) {
 291                 sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS;
 292                 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 293         } else {
 294                 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
 295         }
 296
 297         sc->vtnet_rx_mbuf_size = MCLBYTES;
 298         sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc);
 299
 300         if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VQ)) {
 301                 sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ;
 302
 303                 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX))
 304                         sc->vtnet_flags |= VTNET_FLAG_CTRL_RX;
 305                 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VLAN))
 306                         sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER;
 307                 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_MAC_ADDR) &&
 308                     virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX))
 309                         sc->vtnet_flags |= VTNET_FLAG_CTRL_MAC;
 310         }
 311
 312         error = vtnet_alloc_intrs(sc);
 313         if (error) {
 314                 device_printf(dev, "cannot allocate interrupts\n");
 315                 goto fail;
 316         }
 317
 318         error = vtnet_alloc_virtqueues(sc);
 319         if (error) {
 320                 device_printf(dev, "cannot allocate virtqueues\n");
 321                 goto fail;
 322         }
 323
 324         error = vtnet_bind_intrs(sc);
 325         if (error) {
 326                 device_printf(dev, "cannot bind virtqueues to interrupts\n");
 327                 goto fail;
 328         }
 329
 330         /* Read (or generate) the MAC address for the adapter. */
 331         vtnet_get_hwaddr(sc);
 332
 333         error = vtnet_setup_interface(sc);
 334         if (error) {
 335                 device_printf(dev, "cannot setup interface\n");
 336                 goto fail;
 337         }
 338
 339         for (i = 0; i < sc->vtnet_nintr; i++) {
 340                 error = virtio_setup_intr(dev, i, sc->vtnet_intr_slz[i]);
 341                 if (error) {
 342                         device_printf(dev, "cannot setup virtqueue "
 343                             "interrupts\n");
 344                         ether_ifdetach(sc->vtnet_ifp);
 345                         goto fail;
 346                 }
 347         }
 348
 349         if ((sc->vtnet_flags & VTNET_FLAG_MAC) == 0) {
 350                 ifnet_serialize_all(sc->vtnet_ifp);
 351                 vtnet_set_hwaddr(sc);
 352                 ifnet_deserialize_all(sc->vtnet_ifp);
 353         }
 354
 355         /*
 356          * Device defaults to promiscuous mode for backwards
 357          * compatibility. Turn it off if possible.
 358          */
 359         if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
 360                 ifnet_serialize_all(sc->vtnet_ifp);
 361                 if (vtnet_set_promisc(sc, 0) != 0) {
 362                         sc->vtnet_ifp->if_flags |= IFF_PROMISC;
 363                         device_printf(dev,
 364                             "cannot disable promiscuous mode\n");
 365                 }
 366                 ifnet_deserialize_all(sc->vtnet_ifp);
 367         } else
 368                 sc->vtnet_ifp->if_flags |= IFF_PROMISC;
 369
 370 fail:
 371         if (error)
 372                 vtnet_detach(dev);
 373
 374         return (error);
 375 }
 376
 377 static int
 378 vtnet_detach(device_t dev)
 379 {
 380         struct vtnet_softc *sc;
 381         struct ifnet *ifp;
 382         int i;
 383
 384         sc = device_get_softc(dev);
 385         ifp = sc->vtnet_ifp;
 386
 387         for (i = 0; i < sc->vtnet_nintr; i++)
 388                 virtio_teardown_intr(dev, i);
 389
 390         if (device_is_attached(dev)) {
 391                 ifnet_serialize_all(ifp);
 392                 vtnet_stop(sc);
 393                 lwkt_serialize_handler_disable(&sc->vtnet_slz);
 394                 lwkt_serialize_handler_disable(&sc->vtnet_rx_slz);
 395                 lwkt_serialize_handler_disable(&sc->vtnet_tx_slz);
 396                 ifnet_deserialize_all(ifp);
 397
 398                 ether_ifdetach(ifp);
 399         }
 400
 401         if (sc->vtnet_vlan_attach != NULL) {
 402                 EVENTHANDLER_DEREGISTER(vlan_config, sc->vtnet_vlan_attach);
 403                 sc->vtnet_vlan_attach = NULL;
 404         }
 405         if (sc->vtnet_vlan_detach != NULL) {
 406                 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vtnet_vlan_detach);
 407                 sc->vtnet_vlan_detach = NULL;
 408         }
 409
 410         if (ifp) {
 411                 if_free(ifp);
 412                 sc->vtnet_ifp = NULL;
 413         }
 414
 415         if (sc->vtnet_rx_vq != NULL)
 416                 vtnet_free_rx_mbufs(sc);
 417         if (sc->vtnet_tx_vq != NULL)
 418                 vtnet_free_tx_mbufs(sc);
 419         if (sc->vtnet_ctrl_vq != NULL)
 420                 vtnet_free_ctrl_vq(sc);
 421
 422         if (sc->vtnet_txhdrarea != NULL) {
 423                 contigfree(sc->vtnet_txhdrarea,
 424                     sc->vtnet_txhdrcount * sizeof(struct vtnet_tx_header),
 425                     M_VTNET);
 426                 sc->vtnet_txhdrarea = NULL;
 427         }
 428         SLIST_INIT(&sc->vtnet_txhdr_free);
 429         if (sc->vtnet_macfilter != NULL) {
 430                 contigfree(sc->vtnet_macfilter,
 431                     sizeof(struct vtnet_mac_filter), M_DEVBUF);
 432                 sc->vtnet_macfilter = NULL;
 433         }
 434
 435         ifmedia_removeall(&sc->vtnet_media);
 436
 437         return (0);
 438 }
 439
 440 static int
 441 vtnet_suspend(device_t dev)
 442 {
 443         struct vtnet_softc *sc;
 444
 445         sc = device_get_softc(dev);
 446
 447         ifnet_serialize_all(sc->vtnet_ifp);
 448         vtnet_stop(sc);
 449         sc->vtnet_flags |= VTNET_FLAG_SUSPENDED;
 450         ifnet_deserialize_all(sc->vtnet_ifp);
 451
 452         return (0);
 453 }
 454
 455 static int
 456 vtnet_resume(device_t dev)
 457 {
 458         struct vtnet_softc *sc;
 459         struct ifnet *ifp;
 460
 461         sc = device_get_softc(dev);
 462         ifp = sc->vtnet_ifp;
 463
 464         ifnet_serialize_all(ifp);
 465         if (ifp->if_flags & IFF_UP)
 466                 vtnet_init(sc);
 467         sc->vtnet_flags &= ~VTNET_FLAG_SUSPENDED;
 468         ifnet_deserialize_all(ifp);
 469
 470         return (0);
 471 }
 472
 473 static int
 474 vtnet_shutdown(device_t dev)
 475 {
 476
 477         /*
 478          * Suspend already does all of what we need to
 479          * do here; we just never expect to be resumed.
 480          */
 481         return (vtnet_suspend(dev));
 482 }
 483
 484 static void
 485 vtnet_negotiate_features(struct vtnet_softc *sc)
 486 {
 487         device_t dev;
 488         uint64_t mask, features;
 489
 490         dev = sc->vtnet_dev;
 491         mask = 0;
 492
 493         if (vtnet_csum_disable)
 494                 mask |= VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM;
 495
 496         /*
 497          * XXX DragonFly doesn't support receive checksum offload for ipv6 yet,
 498          *     hence always disable the virtio feature for now.
 499          * XXX We need to support the DynOffload feature, in order to
 500          *     dynamically enable/disable this feature.
 501          */
 502         mask |= VIRTIO_NET_F_GUEST_CSUM;
 503
 504         /*
 505          * TSO is only available when the tx checksum offload feature is also
 506          * negotiated.
 507          */
 508         if (vtnet_csum_disable || vtnet_tso_disable)
 509                 mask |= VIRTIO_NET_F_HOST_TSO4 | VIRTIO_NET_F_HOST_TSO6 |
 510                     VIRTIO_NET_F_HOST_ECN;
 511
 512         if (vtnet_lro_disable)
 513                 mask |= VTNET_LRO_FEATURES;
 514
 515         features = VTNET_FEATURES & ~mask;
 516         features |= VIRTIO_F_NOTIFY_ON_EMPTY;
 517         features |= VIRTIO_F_ANY_LAYOUT;
 518         sc->vtnet_features = virtio_negotiate_features(dev, features);
 519
 520         if (virtio_with_feature(dev, VTNET_LRO_FEATURES) &&
 521             virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF) == 0) {
 522                 /*
 523                  * LRO without mergeable buffers requires special care. This
 524                  * is not ideal because every receive buffer must be large
 525                  * enough to hold the maximum TCP packet, the Ethernet header,
 526                  * and the header. This requires up to 34 descriptors with
 527                  * MCLBYTES clusters. If we do not have indirect descriptors,
 528                  * LRO is disabled since the virtqueue will not contain very
 529                  * many receive buffers.
 530                  */
 531                 if (!virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) {
 532                         device_printf(dev,
 533                             "LRO disabled due to both mergeable buffers and "
 534                             "indirect descriptors not negotiated\n");
 535
 536                         features &= ~VTNET_LRO_FEATURES;
 537                         sc->vtnet_features =
 538                             virtio_negotiate_features(dev, features);
 539                 } else
 540                         sc->vtnet_flags |= VTNET_FLAG_LRO_NOMRG;
 541         }
 542 }
 543
 544 static void
 545 vtnet_serialize(struct ifnet *ifp, enum ifnet_serialize slz)
 546 {
 547         struct vtnet_softc *sc = ifp->if_softc;
 548
 549         ifnet_serialize_array_enter(sc->serializes, 3, slz);
 550 }
 551
 552 static void
 553 vtnet_deserialize(struct ifnet *ifp, enum ifnet_serialize slz)
 554 {
 555         struct vtnet_softc *sc = ifp->if_softc;
 556
 557         ifnet_serialize_array_exit(sc->serializes, 3, slz);
 558 }
 559
 560 static int
 561 vtnet_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz)
 562 {
 563         struct vtnet_softc *sc = ifp->if_softc;
 564
 565         return ifnet_serialize_array_try(sc->serializes, 3, slz);
 566 }
 567
 568 #ifdef INVARIANTS
 569
 570 static void
 571 vtnet_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz,
 572     boolean_t serialized)
 573 {
 574         struct vtnet_softc *sc = ifp->if_softc;
 575
 576         ifnet_serialize_array_assert(sc->serializes, 3, slz, serialized);
 577 }
 578
 579 #endif  /* INVARIANTS */
 580
 581 static int
 582 vtnet_alloc_intrs(struct vtnet_softc *sc)
 583 {
 584         int cnt, error;
 585         int intrcount = virtio_intr_count(sc->vtnet_dev);
 586         int i;
 587         int use_config;
 588
 589         if (virtio_with_feature(sc->vtnet_dev, VIRTIO_NET_F_STATUS)) {
 590                 use_config = 1;
 591                 /* We can use a maximum of 3 interrupt vectors. */
 592                 intrcount = imin(intrcount, 3);
 593         } else {
 594                 /* We can use a maximum of 2 interrupt vectors. */
 595                 intrcount = imin(intrcount, 2);
 596         }
 597
 598         if (intrcount < 1)
 599                 return (ENXIO);
 600
 601         for (i = 0; i < intrcount; i++)
 602                 sc->vtnet_cpus[i] = -1;
 603
 604         cnt = intrcount;
 605         error = virtio_intr_alloc(sc->vtnet_dev, &cnt, use_config,
 606             sc->vtnet_cpus);
 607         if (error != 0) {
 608                 virtio_intr_release(sc->vtnet_dev);
 609                 return (error);
 610         }
 611         sc->vtnet_nintr = cnt;
 612
 613         return (0);
 614 }
 615
 616 static int
 617 vtnet_alloc_virtqueues(struct vtnet_softc *sc)
 618 {
 619         device_t dev;
 620         struct vq_alloc_info vq_info[3];
 621         int nvqs;
 622
 623         dev = sc->vtnet_dev;
 624         nvqs = 2;
 625
 626         /*
 627          * Indirect descriptors are not needed for the Rx
 628          * virtqueue when mergeable buffers are negotiated.
 629          * The header is placed inline with the data, not
 630          * in a separate descriptor, and mbuf clusters are
 631          * always physically contiguous.
 632          */
 633         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
 634                 sc->vtnet_rx_nsegs = (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) ?
 635                     VTNET_MAX_RX_SEGS : VTNET_MIN_RX_SEGS;
 636         } else
 637                 sc->vtnet_rx_nsegs = VTNET_MRG_RX_SEGS;
 638
 639         if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4) ||
 640             virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
 641                 sc->vtnet_tx_nsegs = VTNET_MAX_TX_SEGS;
 642         else
 643                 sc->vtnet_tx_nsegs = VTNET_MIN_TX_SEGS;
 644
 645         VQ_ALLOC_INFO_INIT(&vq_info[0], sc->vtnet_rx_nsegs, &sc->vtnet_rx_vq,
 646             "%s receive", device_get_nameunit(dev));
 647
 648         VQ_ALLOC_INFO_INIT(&vq_info[1], sc->vtnet_tx_nsegs, &sc->vtnet_tx_vq,
 649             "%s transmit", device_get_nameunit(dev));
 650
 651         if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
 652                 nvqs++;
 653
 654                 VQ_ALLOC_INFO_INIT(&vq_info[2], 0, &sc->vtnet_ctrl_vq,
 655                     "%s control", device_get_nameunit(dev));
 656         }
 657
 658         return (virtio_alloc_virtqueues(dev, nvqs, vq_info));
 659 }
 660
 661 static int
 662 vtnet_bind_intrs(struct vtnet_softc *sc)
 663 {
 664         int error = 0;
 665         int i;
 666
 667         for (i = 0; i < 3; i++)
 668                 sc->vtnet_intr_slz[i] = &sc->vtnet_slz;
 669
 670         /* Possible "Virtqueue <-> IRQ" configurations */
 671         switch (sc->vtnet_nintr) {
 672         case 1:
 673                 sc->vtnet_irqmap[0] = (struct irqmap){0, vtnet_rx_vq_intr};
 674                 sc->vtnet_irqmap[1] = (struct irqmap){0, vtnet_tx_vq_intr};
 675                 break;
 676         case 2:
 677                 if (virtio_with_feature(sc->vtnet_dev, VIRTIO_NET_F_STATUS)) {
 678                         sc->vtnet_irqmap[0] =
 679                             (struct irqmap){1, vtnet_rx_vq_intr};
 680                         sc->vtnet_irqmap[1] =
 681                             (struct irqmap){1, vtnet_tx_vq_intr};
 682                 } else {
 683                         sc->vtnet_irqmap[0] =
 684                             (struct irqmap){0, vtnet_rx_msix_intr};
 685                         sc->vtnet_irqmap[1] =
 686                             (struct irqmap){1, vtnet_tx_msix_intr};
 687                         sc->vtnet_intr_slz[0] = &sc->vtnet_rx_slz;
 688                         sc->vtnet_intr_slz[1] = &sc->vtnet_tx_slz;
 689                 }
 690                 break;
 691         case 3:
 692                 sc->vtnet_irqmap[0] = (struct irqmap){1, vtnet_rx_msix_intr};
 693                 sc->vtnet_irqmap[1] = (struct irqmap){2, vtnet_tx_msix_intr};
 694                 sc->vtnet_intr_slz[1] = &sc->vtnet_rx_slz;
 695                 sc->vtnet_intr_slz[2] = &sc->vtnet_tx_slz;
 696                 break;
 697         default:
 698                 device_printf(sc->vtnet_dev,
 699                     "Invalid interrupt vector count: %d\n", sc->vtnet_nintr);
 700                 error = EINVAL;
 701                 goto fail;
 702         }
 703
 704         for (i = 0; i < 2; i++) {
 705                 error = virtio_bind_intr(sc->vtnet_dev,
 706                     sc->vtnet_irqmap[i].irq, i, sc->vtnet_irqmap[i].handler,
 707                     sc);
 708                 if (error) {
 709                         device_printf(sc->vtnet_dev,
 710                             "cannot bind virtqueue IRQs\n");
 711                         goto fail;
 712                 }
 713         }
 714         if (virtio_with_feature(sc->vtnet_dev, VIRTIO_NET_F_STATUS)) {
 715                 error = virtio_bind_intr(sc->vtnet_dev, 0, -1,
 716                     vtnet_config_intr, sc);
 717                 if (error) {
 718                         device_printf(sc->vtnet_dev,
 719                             "cannot bind config_change IRQ\n");
 720                         goto fail;
 721                 }
 722         }
 723
 724 fail:
 725         return (error);
 726 }
 727
 728 static int
 729 vtnet_setup_interface(struct vtnet_softc *sc)
 730 {
 731         device_t dev;
 732         struct ifnet *ifp;
 733         int i;
 734
 735         dev = sc->vtnet_dev;
 736
 737         ifp = sc->vtnet_ifp = if_alloc(IFT_ETHER);
 738         if (ifp == NULL) {
 739                 device_printf(dev, "cannot allocate ifnet structure\n");
 740                 return (ENOSPC);
 741         }
 742
 743         ifp->if_softc = sc;
 744         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 745         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 746         ifp->if_init = vtnet_init;
 747         ifp->if_start = vtnet_start;
 748 #ifdef IFPOLL_ENABLE
 749         ifp->if_npoll = vtnet_npoll;
 750 #endif
 751         ifp->if_serialize = vtnet_serialize;
 752         ifp->if_deserialize = vtnet_deserialize;
 753         ifp->if_tryserialize = vtnet_tryserialize;
 754 #ifdef INVARIANTS
 755         ifp->if_serialize_assert = vtnet_serialize_assert;
 756 #endif
 757         ifp->if_ioctl = vtnet_ioctl;
 758
 759         sc->vtnet_rx_process_limit = virtqueue_size(sc->vtnet_rx_vq);
 760         sc->vtnet_tx_size = virtqueue_size(sc->vtnet_tx_vq);
 761         if (sc->vtnet_flags & VTNET_FLAG_INDIRECT)
 762                 sc->vtnet_txhdrcount = sc->vtnet_tx_size;
 763         else
 764                 sc->vtnet_txhdrcount = (sc->vtnet_tx_size / 2) + 1;
 765         sc->vtnet_txhdrarea = contigmalloc(
 766             sc->vtnet_txhdrcount * sizeof(struct vtnet_tx_header),
 767             M_VTNET, M_WAITOK, 0, BUS_SPACE_MAXADDR, 4, 0);
 768         if (sc->vtnet_txhdrarea == NULL) {
 769                 device_printf(dev, "cannot contigmalloc the tx headers\n");
 770                 return (ENOMEM);
 771         }
 772         for (i = 0; i < sc->vtnet_txhdrcount; i++)
 773                 vtnet_enqueue_txhdr(sc, &sc->vtnet_txhdrarea[i]);
 774         sc->vtnet_macfilter = contigmalloc(
 775             sizeof(struct vtnet_mac_filter),
 776             M_DEVBUF, M_WAITOK, 0, BUS_SPACE_MAXADDR, 4, 0);
 777         if (sc->vtnet_macfilter == NULL) {
 778                 device_printf(dev,
 779                     "cannot contigmalloc the mac filter table\n");
 780                 return (ENOMEM);
 781         }
 782         ifq_set_maxlen(&ifp->if_snd, sc->vtnet_tx_size - 1);
 783         ifq_set_ready(&ifp->if_snd);
 784
 785         ether_ifattach(ifp, sc->vtnet_hwaddr, NULL);
 786
 787         /* The Tx IRQ is currently always the last allocated interrupt. */
 788         ifq_set_cpuid(&ifp->if_snd, sc->vtnet_cpus[sc->vtnet_nintr - 1]);
 789         ifsq_watchdog_init(&sc->vtnet_tx_watchdog,
 790                            ifq_get_subq_default(&ifp->if_snd),
 791                            vtnet_watchdog,
 792                            IF_WDOG_LASTTICK);
 793         ifq_set_hw_serialize(&ifp->if_snd, &sc->vtnet_tx_slz);
 794
 795         /* Tell the upper layer(s) we support long frames. */
 796         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
 797         ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU;
 798
 799         if (virtio_with_feature(dev, VIRTIO_NET_F_CSUM)) {
 800                 ifp->if_capabilities |= IFCAP_TXCSUM;
 801
 802                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4))
 803                         ifp->if_capabilities |= IFCAP_TSO4;
 804                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
 805                         ifp->if_capabilities |= IFCAP_TSO6;
 806                 if (ifp->if_capabilities & IFCAP_TSO)
 807                         ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
 808
 809                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN))
 810                         sc->vtnet_flags |= VTNET_FLAG_TSO_ECN;
 811         }
 812
 813         if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_CSUM))
 814                 ifp->if_capabilities |= IFCAP_RXCSUM;
 815
 816 #if 0   /* IFCAP_LRO doesn't exist in DragonFly. */
 817         if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) ||
 818             virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6))
 819                 ifp->if_capabilities |= IFCAP_LRO;
 820 #endif
 821
 822         if ((ifp->if_capabilities & IFCAP_HWCSUM) == IFCAP_HWCSUM) {
 823                 /*
 824                  * VirtIO does not support VLAN tagging, but we can fake
 825                  * it by inserting and removing the 802.1Q header during
 826                  * transmit and receive. We are then able to do checksum
 827                  * offloading of VLAN frames.
 828                  */
 829                 ifp->if_capabilities |=
 830                         IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM;
 831         }
 832
 833         ifp->if_capenable = ifp->if_capabilities;
 834
 835         /*
 836          * Capabilities after here are not enabled by default.
 837          */
 838
 839         if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) {
 840                 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
 841
 842                 sc->vtnet_vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 843                     vtnet_register_vlan, sc, EVENTHANDLER_PRI_FIRST);
 844                 sc->vtnet_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 845                     vtnet_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST);
 846         }
 847
 848         return (0);
 849 }
 850
 851 static void
 852 vtnet_set_hwaddr(struct vtnet_softc *sc)
 853 {
 854         device_t dev;
 855
 856         dev = sc->vtnet_dev;
 857
 858         if ((sc->vtnet_flags & VTNET_FLAG_CTRL_MAC) &&
 859             (sc->vtnet_flags & VTNET_FLAG_CTRL_RX)) {
 860                 if (vtnet_ctrl_mac_cmd(sc, sc->vtnet_hwaddr) != 0)
 861                         device_printf(dev, "unable to set MAC address\n");
 862         } else if (sc->vtnet_flags & VTNET_FLAG_MAC) {
 863                 virtio_write_device_config(dev,
 864                     offsetof(struct virtio_net_config, mac),
 865                     sc->vtnet_hwaddr, ETHER_ADDR_LEN);
 866         }
 867 }
 868
 869 static void
 870 vtnet_get_hwaddr(struct vtnet_softc *sc)
 871 {
 872         device_t dev;
 873
 874         dev = sc->vtnet_dev;
 875
 876         if ((sc->vtnet_flags & VTNET_FLAG_MAC) == 0) {
 877                 /*
 878                  * Generate a random locally administered unicast address.
 879                  *
 880                  * It would be nice to generate the same MAC address across
 881                  * reboots, but it seems all the hosts currently available
 882                  * support the MAC feature, so this isn't too important.
 883                  */
 884                 sc->vtnet_hwaddr[0] = 0xB2;
 885                 karc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1);
 886                 return;
 887         }
 888
 889         virtio_read_device_config(dev,
 890             offsetof(struct virtio_net_config, mac),
 891             sc->vtnet_hwaddr, ETHER_ADDR_LEN);
 892 }
 893
 894 static int
 895 vtnet_is_link_up(struct vtnet_softc *sc)
 896 {
 897         device_t dev;
 898         struct ifnet *ifp;
 899         uint16_t status;
 900
 901         dev = sc->vtnet_dev;
 902         ifp = sc->vtnet_ifp;
 903
 904         ASSERT_SERIALIZED(&sc->vtnet_slz);
 905
 906         if (virtio_with_feature(dev, VIRTIO_NET_F_STATUS)) {
 907                 status = virtio_read_dev_config_2(dev,
 908                                 offsetof(struct virtio_net_config, status));
 909         } else {
 910                 status = VIRTIO_NET_S_LINK_UP;
 911         }
 912
 913         return ((status & VIRTIO_NET_S_LINK_UP) != 0);
 914 }
 915
 916 static void
 917 vtnet_update_link_status(struct vtnet_softc *sc)
 918 {
 919         device_t dev;
 920         struct ifnet *ifp;
 921         struct ifaltq_subque *ifsq;
 922         int link;
 923
 924         dev = sc->vtnet_dev;
 925         ifp = sc->vtnet_ifp;
 926         ifsq = ifq_get_subq_default(&ifp->if_snd);
 927
 928         link = vtnet_is_link_up(sc);
 929
 930         if (link && ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0)) {
 931                 sc->vtnet_flags |= VTNET_FLAG_LINK;
 932                 if (bootverbose)
 933                         device_printf(dev, "Link is up\n");
 934                 ifp->if_link_state = LINK_STATE_UP;
 935                 if_link_state_change(ifp);
 936                 if (!ifsq_is_empty(ifsq))
 937                         ifsq_devstart_sched(ifsq);
 938         } else if (!link && (sc->vtnet_flags & VTNET_FLAG_LINK)) {
 939                 sc->vtnet_flags &= ~VTNET_FLAG_LINK;
 940                 if (bootverbose)
 941                         device_printf(dev, "Link is down\n");
 942
 943                 ifp->if_link_state = LINK_STATE_DOWN;
 944                 if_link_state_change(ifp);
 945         }
 946 }
 947
 948 static void
 949 vtnet_watchdog(struct ifaltq_subque *ifsq)
 950 {
 951         struct ifnet *ifp;
 952         struct vtnet_softc *sc;
 953
 954         ifp = ifsq_get_ifp(ifsq);
 955         sc = ifp->if_softc;
 956         ASSERT_IFNET_SERIALIZED_ALL(ifp);
 957
 958         /*
 959          * Clean out expended tx buffers prior to terminal count.
 960          *
 961          * NOTE: vtnet_txeof() will set wd_timer to 0 if the virtqueue
 962          *       becomes empty, preventing further watchdog callbacks.
 963          */
 964         if (sc->vtnet_tx_watchdog.wd_timer != 0) {
 965                 vtnet_txeof(sc);
 966                 if (!ifq_is_empty(&ifp->if_snd))
 967                         if_devstart(ifp);
 968                 return;
 969         }
 970
 971         /*
 972          * Check to see if there are any unexpended transmit descriptors.
 973          */
 974         if (virtqueue_empty(sc->vtnet_tx_vq)) {
 975                 if_printf(ifp, "Spurious TX watchdog timeout -- ignoring\n");
 976                 ifsq_watchdog_set_count(&sc->vtnet_tx_watchdog, 0);
 977                 return;
 978         }
 979
 980         if_printf(ifp, "TX watchdog timeout -- resetting\n");
 981 #ifdef VTNET_DEBUG
 982         virtqueue_dump(sc->vtnet_tx_vq);
 983 #endif
 984         ifp->if_oerrors++;
 985         ifp->if_flags &= ~IFF_RUNNING;
 986         vtnet_init(sc);
 987         ifsq_devstart_sched(ifsq);
 988 }
 989
 990 static int
 991 vtnet_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data,struct ucred *cr)
 992 {
 993         struct vtnet_softc *sc;
 994         struct ifreq *ifr;
 995         int reinit, mask, error;
 996
 997         sc = ifp->if_softc;
 998         ifr = (struct ifreq *) data;
 999         reinit = 0;
1000         error = 0;
1001
1002         switch (cmd) {
1003         case SIOCSIFMTU:
1004                 if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > VTNET_MAX_MTU)
1005                         error = EINVAL;
1006                 else if (ifp->if_mtu != ifr->ifr_mtu)
1007                         error = vtnet_change_mtu(sc, ifr->ifr_mtu);
1008                 break;
1009
1010         case SIOCSIFFLAGS:
1011                 if ((ifp->if_flags & IFF_UP) == 0) {
1012                         if (ifp->if_flags & IFF_RUNNING)
1013                                 vtnet_stop(sc);
1014                 } else if (ifp->if_flags & IFF_RUNNING) {
1015                         if ((ifp->if_flags ^ sc->vtnet_if_flags) &
1016                             (IFF_PROMISC | IFF_ALLMULTI)) {
1017                                 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX)
1018                                         vtnet_rx_filter(sc);
1019                                 else
1020                                         error = ENOTSUP;
1021                         }
1022                 } else {
1023                         vtnet_init(sc);
1024                 }
1025
1026                 if (error == 0)
1027                         sc->vtnet_if_flags = ifp->if_flags;
1028                 break;
1029
1030         case SIOCADDMULTI:
1031         case SIOCDELMULTI:
1032                 if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) &&
1033                     (ifp->if_flags & IFF_RUNNING))
1034                         vtnet_rx_filter_mac(sc);
1035                 break;
1036
1037         case SIOCSIFMEDIA:
1038         case SIOCGIFMEDIA:
1039                 error = ifmedia_ioctl(ifp, ifr, &sc->vtnet_media, cmd);
1040                 break;
1041
1042         case SIOCSIFCAP:
1043                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1044
1045
1046                 if (mask & IFCAP_TXCSUM) {
1047                         ifp->if_capenable ^= IFCAP_TXCSUM;
1048                         if (ifp->if_capenable & IFCAP_TXCSUM)
1049                                 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;
1050                         else
1051                                 ifp->if_hwassist &= ~VTNET_CSUM_OFFLOAD;
1052                 }
1053
1054                 if (mask & IFCAP_TSO4) {
1055                         ifp->if_capenable ^= IFCAP_TSO4;
1056                         if (ifp->if_capenable & IFCAP_TSO4)
1057                                 ifp->if_hwassist |= CSUM_TSO;
1058                         else
1059                                 ifp->if_hwassist &= ~CSUM_TSO;
1060                 }
1061
1062                 if (mask & IFCAP_RXCSUM) {
1063                         ifp->if_capenable ^= IFCAP_RXCSUM;
1064                         reinit = 1;
1065                 }
1066
1067 #if 0   /* IFCAP_LRO doesn't exist in DragonFly. */
1068                 if (mask & IFCAP_LRO) {
1069                         ifp->if_capenable ^= IFCAP_LRO;
1070                         reinit = 1;
1071                 }
1072 #endif
1073
1074                 if (mask & IFCAP_VLAN_HWFILTER) {
1075                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1076                         reinit = 1;
1077                 }
1078
1079                 if (mask & IFCAP_VLAN_HWTSO)
1080                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1081
1082                 if (mask & IFCAP_VLAN_HWTAGGING)
1083                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1084
1085                 if (reinit && (ifp->if_flags & IFF_RUNNING)) {
1086                         ifp->if_flags &= ~IFF_RUNNING;
1087                         vtnet_init(sc);
1088                 }
1089                 //VLAN_CAPABILITIES(ifp);
1090
1091                 break;
1092
1093         default:
1094                 error = ether_ioctl(ifp, cmd, data);
1095                 break;
1096         }
1097
1098         return (error);
1099 }
1100
1101 static int
1102 vtnet_change_mtu(struct vtnet_softc *sc, int new_mtu)
1103 {
1104         struct ifnet *ifp;
1105         int new_frame_size, clsize;
1106
1107         ifp = sc->vtnet_ifp;
1108
1109         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1110                 new_frame_size = sizeof(struct vtnet_rx_header) +
1111                     sizeof(struct ether_vlan_header) + new_mtu;
1112
1113                 if (new_frame_size > MJUM9BYTES)
1114                         return (EINVAL);
1115
1116                 if (new_frame_size <= MCLBYTES)
1117                         clsize = MCLBYTES;
1118                 else
1119                         clsize = MJUM9BYTES;
1120         } else {
1121                 new_frame_size = sizeof(struct virtio_net_hdr_mrg_rxbuf) +
1122                     sizeof(struct ether_vlan_header) + new_mtu;
1123
1124                 if (new_frame_size <= MCLBYTES)
1125                         clsize = MCLBYTES;
1126                 else
1127                         clsize = MJUMPAGESIZE;
1128         }
1129
1130         sc->vtnet_rx_mbuf_size = clsize;
1131         sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc);
1132         KASSERT(sc->vtnet_rx_mbuf_count < VTNET_MAX_RX_SEGS,
1133             ("too many rx mbufs: %d", sc->vtnet_rx_mbuf_count));
1134
1135         ifp->if_mtu = new_mtu;
1136
1137         if (ifp->if_flags & IFF_RUNNING) {
1138                 ifp->if_flags &= ~IFF_RUNNING;
1139                 vtnet_init(sc);
1140         }
1141
1142         return (0);
1143 }
1144
1145 static int
1146 vtnet_init_rx_vq(struct vtnet_softc *sc)
1147 {
1148         struct virtqueue *vq;
1149         int nbufs, error;
1150
1151         vq = sc->vtnet_rx_vq;
1152         nbufs = 0;
1153         error = ENOSPC;
1154
1155         while (!virtqueue_full(vq)) {
1156                 if ((error = vtnet_newbuf(sc)) != 0)
1157                         break;
1158                 nbufs++;
1159         }
1160
1161         if (nbufs > 0) {
1162                 virtqueue_notify(vq, NULL);
1163
1164                 /*
1165                  * EMSGSIZE signifies the virtqueue did not have enough
1166                  * entries available to hold the last mbuf. This is not
1167                  * an error. We should not get ENOSPC since we check if
1168                  * the virtqueue is full before attempting to add a
1169                  * buffer.
1170                  */
1171                 if (error == EMSGSIZE)
1172                         error = 0;
1173         }
1174
1175         return (error);
1176 }
1177
1178 static void
1179 vtnet_free_rx_mbufs(struct vtnet_softc *sc)
1180 {
1181         struct virtqueue *vq;
1182         struct mbuf *m;
1183         int last;
1184
1185         vq = sc->vtnet_rx_vq;
1186         last = 0;
1187
1188         while ((m = virtqueue_drain(vq, &last)) != NULL)
1189                 m_freem(m);
1190
1191         KASSERT(virtqueue_empty(vq), ("mbufs remaining in Rx Vq"));
1192 }
1193
1194 static void
1195 vtnet_free_tx_mbufs(struct vtnet_softc *sc)
1196 {
1197         struct virtqueue *vq;
1198         struct vtnet_tx_header *txhdr;
1199         int last;
1200
1201         vq = sc->vtnet_tx_vq;
1202         last = 0;
1203
1204         while ((txhdr = virtqueue_drain(vq, &last)) != NULL) {
1205                 m_freem(txhdr->vth_mbuf);
1206                 vtnet_enqueue_txhdr(sc, txhdr);
1207         }
1208
1209         KASSERT(virtqueue_empty(vq), ("mbufs remaining in Tx Vq"));
1210 }
1211
1212 static void
1213 vtnet_free_ctrl_vq(struct vtnet_softc *sc)
1214 {
1215         /*
1216          * The control virtqueue is only polled, therefore
1217          * it should already be empty.
1218          */
1219         KASSERT(virtqueue_empty(sc->vtnet_ctrl_vq),
1220                 ("Ctrl Vq not empty"));
1221 }
1222
1223 static struct mbuf *
1224 vtnet_alloc_rxbuf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp)
1225 {
1226         struct mbuf *m_head, *m_tail, *m;
1227         int i, clsize;
1228
1229         clsize = sc->vtnet_rx_mbuf_size;
1230
1231         /*use getcl instead of getjcl. see  if_mxge.c comment line 2398*/
1232         //m_head = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, clsize);
1233         m_head = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR );
1234         if (m_head == NULL)
1235                 goto fail;
1236
1237         m_head->m_len = clsize;
1238         m_tail = m_head;
1239
1240         if (nbufs > 1) {
1241                 KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
1242                         ("chained Rx mbuf requested without LRO_NOMRG"));
1243
1244                 for (i = 0; i < nbufs - 1; i++) {
1245                         //m = m_getjcl(M_DONTWAIT, MT_DATA, 0, clsize);
1246                         m = m_getcl(M_NOWAIT, MT_DATA, 0);
1247                         if (m == NULL)
1248                                 goto fail;
1249
1250                         m->m_len = clsize;
1251                         m_tail->m_next = m;
1252                         m_tail = m;
1253                 }
1254         }
1255
1256         if (m_tailp != NULL)
1257                 *m_tailp = m_tail;
1258
1259         return (m_head);
1260
1261 fail:
1262         sc->vtnet_stats.mbuf_alloc_failed++;
1263         m_freem(m_head);
1264
1265         return (NULL);
1266 }
1267
1268 static int
1269 vtnet_replace_rxbuf(struct vtnet_softc *sc, struct mbuf *m0, int len0)
1270 {
1271         struct mbuf *m, *m_prev;
1272         struct mbuf *m_new, *m_tail;
1273         int len, clsize, nreplace, error;
1274
1275         m = m0;
1276         m_prev = NULL;
1277         len = len0;
1278
1279         m_tail = NULL;
1280         clsize = sc->vtnet_rx_mbuf_size;
1281         nreplace = 0;
1282
1283         if (m->m_next != NULL)
1284                 KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
1285                     ("chained Rx mbuf without LRO_NOMRG"));
1286
1287         /*
1288          * Since LRO_NOMRG mbuf chains are so large, we want to avoid
1289          * allocating an entire chain for each received frame. When
1290          * the received frame's length is less than that of the chain,
1291          * the unused mbufs are reassigned to the new chain.
1292          */
1293         while (len > 0) {
1294                 /*
1295                  * Something is seriously wrong if we received
1296                  * a frame larger than the mbuf chain. Drop it.
1297                  */
1298                 if (m == NULL) {
1299                         sc->vtnet_stats.rx_frame_too_large++;
1300                         return (EMSGSIZE);
1301                 }
1302
1303                 KASSERT(m->m_len == clsize,
1304                     ("mbuf length not expected cluster size: %d",
1305                     m->m_len));
1306
1307                 m->m_len = MIN(m->m_len, len);
1308                 len -= m->m_len;
1309
1310                 m_prev = m;
1311                 m = m->m_next;
1312                 nreplace++;
1313         }
1314
1315         KASSERT(m_prev != NULL, ("m_prev == NULL"));
1316         KASSERT(nreplace <= sc->vtnet_rx_mbuf_count,
1317                 ("too many replacement mbufs: %d/%d", nreplace,
1318                 sc->vtnet_rx_mbuf_count));
1319
1320         m_new = vtnet_alloc_rxbuf(sc, nreplace, &m_tail);
1321         if (m_new == NULL) {
1322                 m_prev->m_len = clsize;
1323                 return (ENOBUFS);
1324         }
1325
1326         /*
1327          * Move unused mbufs, if any, from the original chain
1328          * onto the end of the new chain.
1329          */
1330         if (m_prev->m_next != NULL) {
1331                 m_tail->m_next = m_prev->m_next;
1332                 m_prev->m_next = NULL;
1333         }
1334
1335         error = vtnet_enqueue_rxbuf(sc, m_new);
1336         if (error) {
1337                 /*
1338                  * BAD! We could not enqueue the replacement mbuf chain. We
1339                  * must restore the m0 chain to the original state if it was
1340                  * modified so we can subsequently discard it.
1341                  *
1342                  * NOTE: The replacement is suppose to be an identical copy
1343                  * to the one just dequeued so this is an unexpected error.
1344                  */
1345                 sc->vtnet_stats.rx_enq_replacement_failed++;
1346
1347                 if (m_tail->m_next != NULL) {
1348                         m_prev->m_next = m_tail->m_next;
1349                         m_tail->m_next = NULL;
1350                 }
1351
1352                 m_prev->m_len = clsize;
1353                 m_freem(m_new);
1354         }
1355
1356         return (error);
1357 }
1358
1359 static int
1360 vtnet_newbuf(struct vtnet_softc *sc)
1361 {
1362         struct mbuf *m;
1363         int error;
1364
1365         m = vtnet_alloc_rxbuf(sc, sc->vtnet_rx_mbuf_count, NULL);
1366         if (m == NULL)
1367                 return (ENOBUFS);
1368
1369         error = vtnet_enqueue_rxbuf(sc, m);
1370         if (error)
1371                 m_freem(m);
1372
1373         return (error);
1374 }
1375
1376 static void
1377 vtnet_discard_merged_rxbuf(struct vtnet_softc *sc, int nbufs)
1378 {
1379         struct virtqueue *vq;
1380         struct mbuf *m;
1381
1382         vq = sc->vtnet_rx_vq;
1383
1384         while (--nbufs > 0) {
1385                 if ((m = virtqueue_dequeue(vq, NULL)) == NULL)
1386                         break;
1387                 vtnet_discard_rxbuf(sc, m);
1388         }
1389 }
1390
1391 static void
1392 vtnet_discard_rxbuf(struct vtnet_softc *sc, struct mbuf *m)
1393 {
1394         int error;
1395
1396         /*
1397          * Requeue the discarded mbuf. This should always be
1398          * successful since it was just dequeued.
1399          */
1400         error = vtnet_enqueue_rxbuf(sc, m);
1401         KASSERT(error == 0, ("cannot requeue discarded mbuf"));
1402 }
1403
1404 static int
1405 vtnet_enqueue_rxbuf(struct vtnet_softc *sc, struct mbuf *m)
1406 {
1407         struct sglist sg;
1408         struct sglist_seg segs[VTNET_MAX_RX_SEGS];
1409         struct vtnet_rx_header *rxhdr;
1410         struct virtio_net_hdr *hdr;
1411         uint8_t *mdata;
1412         int offset, error;
1413
1414         ASSERT_SERIALIZED(&sc->vtnet_rx_slz);
1415         if ((sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) == 0)
1416                 KASSERT(m->m_next == NULL, ("chained Rx mbuf"));
1417
1418         sglist_init(&sg, sc->vtnet_rx_nsegs, segs);
1419
1420         mdata = mtod(m, uint8_t *);
1421         offset = 0;
1422
1423         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1424                 rxhdr = (struct vtnet_rx_header *) mdata;
1425                 hdr = &rxhdr->vrh_hdr;
1426                 offset += sizeof(struct vtnet_rx_header);
1427
1428                 error = sglist_append(&sg, hdr, sc->vtnet_hdr_size);
1429                 KASSERT(error == 0, ("cannot add header to sglist"));
1430         }
1431
1432         error = sglist_append(&sg, mdata + offset, m->m_len - offset);
1433         if (error)
1434                 return (error);
1435
1436         if (m->m_next != NULL) {
1437                 error = sglist_append_mbuf(&sg, m->m_next);
1438                 if (error)
1439                         return (error);
1440         }
1441
1442         return (virtqueue_enqueue(sc->vtnet_rx_vq, m, &sg, 0, sg.sg_nseg));
1443 }
1444
1445 #ifdef IFPOLL_ENABLE
1446
1447 static void
1448 vtnet_npoll_status(struct ifnet *ifp)
1449 {
1450         struct vtnet_softc *sc = ifp->if_softc;
1451
1452         ASSERT_SERIALIZED(&sc->vtnet_slz);
1453
1454         vtnet_update_link_status(sc);
1455 }
1456
1457 static void
1458 vtnet_npoll_rx(struct ifnet *ifp, void *arg __unused, int cycle)
1459 {
1460         struct vtnet_softc *sc = ifp->if_softc;
1461
1462         vtnet_rxeof(sc, cycle, NULL);
1463 }
1464
1465 static void
1466 vtnet_npoll_tx(struct ifnet *ifp, void *arg __unused, int cycle __unused)
1467 {
1468         struct vtnet_softc *sc = ifp->if_softc;
1469
1470         ASSERT_SERIALIZED(&sc->vtnet_tx_slz);
1471
1472         vtnet_txeof(sc);
1473         if (!ifq_is_empty(&ifp->if_snd))
1474                 if_devstart(ifp);
1475 }
1476
1477 static void
1478 vtnet_npoll(struct ifnet *ifp, struct ifpoll_info *info)
1479 {
1480         struct vtnet_softc *sc = ifp->if_softc;
1481         int i;
1482
1483         ASSERT_IFNET_SERIALIZED_ALL(ifp);
1484
1485         if (info) {
1486                 int cpu;
1487
1488                 info->ifpi_status.status_func = vtnet_npoll_status;
1489                 info->ifpi_status.serializer = &sc->vtnet_slz;
1490
1491                 /* Use the same cpu for rx and tx. */
1492                 cpu = device_get_unit(device_get_parent(sc->vtnet_dev));
1493                 /* Shuffle a bit. */
1494                 cpu = (cpu * 61) % netisr_ncpus;
1495                 KKASSERT(cpu < netisr_ncpus);
1496                 info->ifpi_tx[cpu].poll_func = vtnet_npoll_tx;
1497                 info->ifpi_tx[cpu].arg = NULL;
1498                 info->ifpi_tx[cpu].serializer = &sc->vtnet_tx_slz;
1499                 ifq_set_cpuid(&ifp->if_snd, cpu);
1500
1501                 info->ifpi_rx[cpu].poll_func = vtnet_npoll_rx;
1502                 info->ifpi_rx[cpu].arg = NULL;
1503                 info->ifpi_rx[cpu].serializer = &sc->vtnet_rx_slz;
1504
1505                 for (i = 0; i < 3; i++)
1506                         lwkt_serialize_handler_disable(sc->serializes[i]);
1507                 vtnet_disable_rx_intr(sc);
1508                 vtnet_disable_tx_intr(sc);
1509                 for (i = 0; i < sc->vtnet_nintr; i++)
1510                         virtio_teardown_intr(sc->vtnet_dev, i);
1511                 if (virtio_with_feature(sc->vtnet_dev, VIRTIO_NET_F_STATUS))
1512                         virtio_unbind_intr(sc->vtnet_dev, -1);
1513                 for (i = 0; i < 2; i++)
1514                         virtio_unbind_intr(sc->vtnet_dev, i);
1515         } else {
1516                 int error;
1517
1518                 ifq_set_cpuid(&ifp->if_snd,
1519                     sc->vtnet_cpus[sc->vtnet_nintr - 1]);
1520                 for (i = 0; i < 3; i++)
1521                         lwkt_serialize_handler_enable(sc->serializes[i]);
1522                 for (i = 0; i < 2; i++) {
1523                         error = virtio_bind_intr(sc->vtnet_dev,
1524                             sc->vtnet_irqmap[i].irq, i,
1525                             sc->vtnet_irqmap[i].handler, sc);
1526                         if (error) {
1527                                 device_printf(sc->vtnet_dev,
1528                                     "cannot re-bind virtqueue IRQs\n");
1529                         }
1530                 }
1531                 if (virtio_with_feature(sc->vtnet_dev, VIRTIO_NET_F_STATUS)) {
1532                         error = virtio_bind_intr(sc->vtnet_dev, 0, -1,
1533                             vtnet_config_intr, sc);
1534                         if (error) {
1535                                 device_printf(sc->vtnet_dev,
1536                                     "cannot re-bind config_change IRQ\n");
1537                         }
1538                 }
1539                 for (i = 0; i < sc->vtnet_nintr; i++) {
1540                         error = virtio_setup_intr(sc->vtnet_dev, i,
1541                             sc->vtnet_intr_slz[i]);
1542                         if (error) {
1543                                 device_printf(sc->vtnet_dev,
1544                                     "cannot setup virtqueue interrupts\n");
1545                         }
1546                 }
1547                 vtnet_enable_rx_intr(sc);
1548                 vtnet_enable_tx_intr(sc);
1549         }
1550 }
1551
1552 #endif  /* IFPOLL_ENABLE */
1553
1554 static void
1555 vtnet_vlan_tag_remove(struct mbuf *m)
1556 {
1557         struct ether_vlan_header *evl;
1558
1559         evl = mtod(m, struct ether_vlan_header *);
1560
1561         m->m_pkthdr.ether_vlantag = ntohs(evl->evl_tag);
1562         m->m_flags |= M_VLANTAG;
1563
1564         /* Strip the 802.1Q header. */
1565         bcopy((char *) evl, (char *) evl + ETHER_VLAN_ENCAP_LEN,
1566             ETHER_HDR_LEN - ETHER_TYPE_LEN);
1567         m_adj(m, ETHER_VLAN_ENCAP_LEN);
1568 }
1569
1570 /*
1571  * Alternative method of doing receive checksum offloading. Rather
1572  * than parsing the received frame down to the IP header, use the
1573  * csum_offset to determine which CSUM_* flags are appropriate. We
1574  * can get by with doing this only because the checksum offsets are
1575  * unique for the things we care about.
1576  */
1577 static int
1578 vtnet_rx_csum(struct vtnet_softc *sc, struct mbuf *m,
1579     struct virtio_net_hdr *hdr)
1580 {
1581         struct ether_header *eh;
1582         struct ether_vlan_header *evh;
1583         struct udphdr *udp;
1584         int csum_len;
1585         uint16_t eth_type;
1586
1587         csum_len = hdr->csum_start + hdr->csum_offset;
1588
1589         if (csum_len < sizeof(struct ether_header) + sizeof(struct ip))
1590                 return (1);
1591         if (m->m_len < csum_len)
1592                 return (1);
1593
1594         eh = mtod(m, struct ether_header *);
1595         eth_type = ntohs(eh->ether_type);
1596         if (eth_type == ETHERTYPE_VLAN) {
1597                 evh = mtod(m, struct ether_vlan_header *);
1598                 eth_type = ntohs(evh->evl_proto);
1599         }
1600
1601         if (eth_type != ETHERTYPE_IP && eth_type != ETHERTYPE_IPV6) {
1602                 sc->vtnet_stats.rx_csum_bad_ethtype++;
1603                 return (1);
1604         }
1605
1606         /* Use the offset to determine the appropriate CSUM_* flags. */
1607         switch (hdr->csum_offset) {
1608         case offsetof(struct udphdr, uh_sum):
1609                 if (m->m_len < hdr->csum_start + sizeof(struct udphdr))
1610                         return (1);
1611                 udp = (struct udphdr *)(mtod(m, uint8_t *) + hdr->csum_start);
1612                 if (udp->uh_sum == 0)
1613                         return (0);
1614
1615                 /* FALLTHROUGH */
1616
1617         case offsetof(struct tcphdr, th_sum):
1618                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1619                 m->m_pkthdr.csum_data = 0xFFFF;
1620                 break;
1621
1622         default:
1623                 sc->vtnet_stats.rx_csum_bad_offset++;
1624                 return (1);
1625         }
1626
1627         sc->vtnet_stats.rx_csum_offloaded++;
1628
1629         return (0);
1630 }
1631
1632 static int
1633 vtnet_rxeof_merged(struct vtnet_softc *sc, struct mbuf *m_head, int nbufs)
1634 {
1635         struct ifnet *ifp;
1636         struct virtqueue *vq;
1637         struct mbuf *m, *m_tail;
1638         int len;
1639
1640         ifp = sc->vtnet_ifp;
1641         vq = sc->vtnet_rx_vq;
1642         m_tail = m_head;
1643
1644         while (--nbufs > 0) {
1645                 m = virtqueue_dequeue(vq, &len);
1646                 if (m == NULL) {
1647                         ifp->if_ierrors++;
1648                         goto fail;
1649                 }
1650
1651                 if (vtnet_newbuf(sc) != 0) {
1652                         ifp->if_iqdrops++;
1653                         vtnet_discard_rxbuf(sc, m);
1654                         if (nbufs > 1)
1655                                 vtnet_discard_merged_rxbuf(sc, nbufs);
1656                         goto fail;
1657                 }
1658
1659                 if (m->m_len < len)
1660                         len = m->m_len;
1661
1662                 m->m_len = len;
1663                 m->m_flags &= ~M_PKTHDR;
1664
1665                 m_head->m_pkthdr.len += len;
1666                 m_tail->m_next = m;
1667                 m_tail = m;
1668         }
1669
1670         return (0);
1671
1672 fail:
1673         sc->vtnet_stats.rx_mergeable_failed++;
1674         m_freem(m_head);
1675
1676         return (1);
1677 }
1678
1679 static int
1680 vtnet_rxeof(struct vtnet_softc *sc, int count, int *rx_npktsp)
1681 {
1682         struct virtio_net_hdr lhdr;
1683         struct ifnet *ifp;
1684         struct virtqueue *vq;
1685         struct mbuf *m;
1686         struct ether_header *eh;
1687         struct virtio_net_hdr *hdr;
1688         struct virtio_net_hdr_mrg_rxbuf *mhdr;
1689         int len, deq, nbufs, adjsz, rx_npkts;
1690
1691         ifp = sc->vtnet_ifp;
1692         vq = sc->vtnet_rx_vq;
1693         hdr = &lhdr;
1694         deq = 0;
1695         rx_npkts = 0;
1696
1697         while (--count >= 0) {
1698                 m = virtqueue_dequeue(vq, &len);
1699                 if (m == NULL)
1700                         break;
1701                 deq++;
1702
1703                 if (len < sc->vtnet_hdr_size + ETHER_HDR_LEN) {
1704                         ifp->if_ierrors++;
1705                         vtnet_discard_rxbuf(sc, m);
1706                         continue;
1707                 }
1708
1709                 if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1710                         nbufs = 1;
1711                         adjsz = sizeof(struct vtnet_rx_header);
1712                         /*
1713                          * Account for our pad between the header and
1714                          * the actual start of the frame.
1715                          */
1716                         len += VTNET_RX_HEADER_PAD;
1717                 } else {
1718                         mhdr = mtod(m, struct virtio_net_hdr_mrg_rxbuf *);
1719                         nbufs = mhdr->num_buffers;
1720                         adjsz = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1721                 }
1722
1723                 if (vtnet_replace_rxbuf(sc, m, len) != 0) {
1724                         ifp->if_iqdrops++;
1725                         vtnet_discard_rxbuf(sc, m);
1726                         if (nbufs > 1)
1727                                 vtnet_discard_merged_rxbuf(sc, nbufs);
1728                         continue;
1729                 }
1730
1731                 m->m_pkthdr.len = len;
1732                 m->m_pkthdr.rcvif = ifp;
1733                 m->m_pkthdr.csum_flags = 0;
1734
1735                 if (nbufs > 1) {
1736                         if (vtnet_rxeof_merged(sc, m, nbufs) != 0)
1737                                 continue;
1738                 }
1739
1740                 ifp->if_ipackets++;
1741
1742                 /*
1743                  * Save copy of header before we strip it. For both mergeable
1744                  * and non-mergeable, the VirtIO header is placed first in the
1745                  * mbuf's data. We no longer need num_buffers, so always use a
1746                  * virtio_net_hdr.
1747                  */
1748                 memcpy(hdr, mtod(m, void *), sizeof(struct virtio_net_hdr));
1749                 m_adj(m, adjsz);
1750
1751                 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1752                         eh = mtod(m, struct ether_header *);
1753                         if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
1754                                 vtnet_vlan_tag_remove(m);
1755
1756                                 /*
1757                                  * With the 802.1Q header removed, update the
1758                                  * checksum starting location accordingly.
1759                                  */
1760                                 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
1761                                         hdr->csum_start -=
1762                                             ETHER_VLAN_ENCAP_LEN;
1763                         }
1764                 }
1765
1766                 if (ifp->if_capenable & IFCAP_RXCSUM &&
1767                     hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
1768                         if (vtnet_rx_csum(sc, m, hdr) != 0)
1769                                 sc->vtnet_stats.rx_csum_failed++;
1770                 }
1771
1772                 rx_npkts++;
1773                 ifp->if_input(ifp, m, NULL, mycpuid);
1774
1775                 /*
1776                  * The interface may have been stopped while we were
1777                  * passing the packet up the network stack.
1778                  */
1779                 if ((ifp->if_flags & IFF_RUNNING) == 0)
1780                         break;
1781         }
1782
1783         if (deq > 0)
1784                 virtqueue_notify(vq, NULL);
1785
1786         if (rx_npktsp != NULL)
1787                 *rx_npktsp = rx_npkts;
1788
1789         return (count > 0 ? 0 : EAGAIN);
1790 }
1791
1792 static void
1793 vtnet_rx_msix_intr(void *xsc)
1794 {
1795         struct vtnet_softc *sc;
1796         struct ifnet *ifp;
1797         int more;
1798
1799         sc = xsc;
1800         ifp = sc->vtnet_ifp;
1801
1802         if (!virtqueue_pending(sc->vtnet_rx_vq))
1803                 return;
1804
1805         vtnet_disable_rx_intr(sc);
1806 next:
1807         if ((ifp->if_flags & IFF_RUNNING) == 0) {
1808                 vtnet_enable_rx_intr(sc);
1809                 return;
1810         }
1811
1812         more = vtnet_rxeof(sc, sc->vtnet_rx_process_limit, NULL);
1813         if (!more && vtnet_enable_rx_intr(sc) != 0) {
1814                 vtnet_disable_rx_intr(sc);
1815                 more = 1;
1816         }
1817
1818         if (more) {
1819                 sc->vtnet_stats.rx_task_rescheduled++;
1820                 goto next;
1821         }
1822 }
1823
1824 static void
1825 vtnet_rx_vq_intr(void *xsc)
1826 {
1827         struct vtnet_softc *sc = xsc;
1828
1829         lwkt_serialize_enter(&sc->vtnet_rx_slz);
1830         vtnet_rx_msix_intr(xsc);
1831         lwkt_serialize_exit(&sc->vtnet_rx_slz);
1832 }
1833
1834 static void
1835 vtnet_enqueue_txhdr(struct vtnet_softc *sc, struct vtnet_tx_header *txhdr)
1836 {
1837         bzero(txhdr, sizeof(*txhdr));
1838         SLIST_INSERT_HEAD(&sc->vtnet_txhdr_free, txhdr, link);
1839 }
1840
1841 static void
1842 vtnet_txeof(struct vtnet_softc *sc)
1843 {
1844         struct virtqueue *vq;
1845         struct ifnet *ifp;
1846         struct vtnet_tx_header *txhdr;
1847         int deq;
1848
1849         vq = sc->vtnet_tx_vq;
1850         ifp = sc->vtnet_ifp;
1851         deq = 0;
1852
1853         while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) {
1854                 deq++;
1855                 ifp->if_opackets++;
1856                 m_freem(txhdr->vth_mbuf);
1857                 vtnet_enqueue_txhdr(sc, txhdr);
1858         }
1859
1860         if (deq > 0) {
1861                 ifq_clr_oactive(&ifp->if_snd);
1862                 if (virtqueue_empty(vq))
1863                         ifsq_watchdog_set_count(&sc->vtnet_tx_watchdog, 0);
1864                 else
1865                         ifsq_watchdog_set_count(&sc->vtnet_tx_watchdog,
1866                                                 VTNET_WATCHDOG_TIMEOUT);
1867         }
1868 }
1869
1870 static struct mbuf *
1871 vtnet_tx_offload(struct vtnet_softc *sc, struct mbuf *m,
1872     struct virtio_net_hdr *hdr)
1873 {
1874         struct ifnet *ifp;
1875         struct ether_header *eh;
1876         struct ether_vlan_header *evh;
1877         struct ip *ip;
1878         struct ip6_hdr *ip6;
1879         struct tcphdr *tcp;
1880         int ip_offset;
1881         uint16_t eth_type, csum_start;
1882         uint8_t ip_proto, gso_type;
1883
1884         ifp = sc->vtnet_ifp;
1885         M_ASSERTPKTHDR(m);
1886
1887         ip_offset = sizeof(struct ether_header);
1888         if (m->m_len < ip_offset) {
1889                 if ((m = m_pullup(m, ip_offset)) == NULL)
1890                         return (NULL);
1891         }
1892
1893         eh = mtod(m, struct ether_header *);
1894         eth_type = ntohs(eh->ether_type);
1895         if (eth_type == ETHERTYPE_VLAN) {
1896                 ip_offset = sizeof(struct ether_vlan_header);
1897                 if (m->m_len < ip_offset) {
1898                         if ((m = m_pullup(m, ip_offset)) == NULL)
1899                                 return (NULL);
1900                 }
1901                 evh = mtod(m, struct ether_vlan_header *);
1902                 eth_type = ntohs(evh->evl_proto);
1903         }
1904
1905         switch (eth_type) {
1906         case ETHERTYPE_IP:
1907                 if (m->m_len < ip_offset + sizeof(struct ip)) {
1908                         m = m_pullup(m, ip_offset + sizeof(struct ip));
1909                         if (m == NULL)
1910                                 return (NULL);
1911                 }
1912
1913                 ip = (struct ip *)(mtod(m, uint8_t *) + ip_offset);
1914                 ip_proto = ip->ip_p;
1915                 csum_start = ip_offset + (ip->ip_hl << 2);
1916                 gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
1917                 break;
1918
1919         case ETHERTYPE_IPV6:
1920                 if (m->m_len < ip_offset + sizeof(struct ip6_hdr)) {
1921                         m = m_pullup(m, ip_offset + sizeof(struct ip6_hdr));
1922                         if (m == NULL)
1923                                 return (NULL);
1924                 }
1925
1926                 ip6 = (struct ip6_hdr *)(mtod(m, uint8_t *) + ip_offset);
1927                 /*
1928                  * XXX Assume no extension headers are present. Presently,
1929                  * this will always be true in the case of TSO, and FreeBSD
1930                  * does not perform checksum offloading of IPv6 yet.
1931                  */
1932                 ip_proto = ip6->ip6_nxt;
1933                 csum_start = ip_offset + sizeof(struct ip6_hdr);
1934                 gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
1935                 break;
1936
1937         default:
1938                 return (m);
1939         }
1940
1941         if (m->m_pkthdr.csum_flags & VTNET_CSUM_OFFLOAD) {
1942                 hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM;
1943                 hdr->csum_start = csum_start;
1944                 hdr->csum_offset = m->m_pkthdr.csum_data;
1945
1946                 sc->vtnet_stats.tx_csum_offloaded++;
1947         }
1948
1949         if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1950                 if (ip_proto != IPPROTO_TCP)
1951                         return (m);
1952
1953                 if (m->m_len < csum_start + sizeof(struct tcphdr)) {
1954                         m = m_pullup(m, csum_start + sizeof(struct tcphdr));
1955                         if (m == NULL)
1956                                 return (NULL);
1957                 }
1958
1959                 tcp = (struct tcphdr *)(mtod(m, uint8_t *) + csum_start);
1960                 hdr->gso_type = gso_type;
1961                 hdr->hdr_len = csum_start + (tcp->th_off << 2);
1962                 hdr->gso_size = m->m_pkthdr.tso_segsz;
1963
1964                 if (tcp->th_flags & TH_CWR) {
1965                         /*
1966                          * Drop if we did not negotiate VIRTIO_NET_F_HOST_ECN.
1967                          * ECN support is only configurable globally with the
1968                          * net.inet.tcp.ecn.enable sysctl knob.
1969                          */
1970                         if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) {
1971                                 if_printf(ifp, "TSO with ECN not supported "
1972                                     "by host\n");
1973                                 m_freem(m);
1974                                 return (NULL);
1975                         }
1976
1977                         hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN;
1978                 }
1979
1980                 sc->vtnet_stats.tx_tso_offloaded++;
1981         }
1982
1983         return (m);
1984 }
1985
1986 static int
1987 vtnet_enqueue_txbuf(struct vtnet_softc *sc, struct mbuf **m_head,
1988     struct vtnet_tx_header *txhdr)
1989 {
1990         struct sglist sg;
1991         struct sglist_seg segs[VTNET_MAX_TX_SEGS];
1992         struct virtqueue *vq;
1993         struct mbuf *m;
1994         int error;
1995
1996         vq = sc->vtnet_tx_vq;
1997         m = *m_head;
1998
1999         sglist_init(&sg, sc->vtnet_tx_nsegs, segs);
2000         error = sglist_append(&sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size);
2001         KASSERT(error == 0 && sg.sg_nseg == 1,
2002             ("%s: error %d adding header to sglist", __func__, error));
2003
2004         error = sglist_append_mbuf(&sg, m);
2005         if (error) {
2006                 m = m_defrag(m, M_NOWAIT);
2007                 if (m == NULL)
2008                         goto fail;
2009
2010                 *m_head = m;
2011                 sc->vtnet_stats.tx_defragged++;
2012
2013                 error = sglist_append_mbuf(&sg, m);
2014                 if (error)
2015                         goto fail;
2016         }
2017
2018         txhdr->vth_mbuf = m;
2019         error = virtqueue_enqueue(vq, txhdr, &sg, sg.sg_nseg, 0);
2020
2021         return (error);
2022
2023 fail:
2024         sc->vtnet_stats.tx_defrag_failed++;
2025         m_freem(*m_head);
2026         *m_head = NULL;
2027
2028         return (ENOBUFS);
2029 }
2030
2031 static struct mbuf *
2032 vtnet_vlan_tag_insert(struct mbuf *m)
2033 {
2034         struct mbuf *n;
2035         struct ether_vlan_header *evl;
2036
2037         if (M_WRITABLE(m) == 0) {
2038                 n = m_dup(m, M_NOWAIT);
2039                 m_freem(m);
2040                 if ((m = n) == NULL)
2041                         return (NULL);
2042         }
2043
2044         M_PREPEND(m, ETHER_VLAN_ENCAP_LEN, M_NOWAIT);
2045         if (m == NULL)
2046                 return (NULL);
2047         if (m->m_len < sizeof(struct ether_vlan_header)) {
2048                 m = m_pullup(m, sizeof(struct ether_vlan_header));
2049                 if (m == NULL)
2050                         return (NULL);
2051         }
2052
2053         /* Insert 802.1Q header into the existing Ethernet header. */
2054         evl = mtod(m, struct ether_vlan_header *);
2055         bcopy((char *) evl + ETHER_VLAN_ENCAP_LEN,
2056               (char *) evl, ETHER_HDR_LEN - ETHER_TYPE_LEN);
2057         evl->evl_encap_proto = htons(ETHERTYPE_VLAN);
2058         evl->evl_tag = htons(m->m_pkthdr.ether_vlantag);
2059         m->m_flags &= ~M_VLANTAG;
2060
2061         return (m);
2062 }
2063
2064 static int
2065 vtnet_encap(struct vtnet_softc *sc, struct mbuf **m_head)
2066 {
2067         struct vtnet_tx_header *txhdr;
2068         struct virtio_net_hdr *hdr;
2069         struct mbuf *m;
2070         int error;
2071
2072         txhdr = SLIST_FIRST(&sc->vtnet_txhdr_free);
2073         if (txhdr == NULL)
2074                 return (ENOBUFS);
2075         SLIST_REMOVE_HEAD(&sc->vtnet_txhdr_free, link);
2076
2077         /*
2078          * Always use the non-mergeable header to simplify things. When
2079          * the mergeable feature is negotiated, the num_buffers field
2080          * must be set to zero. We use vtnet_hdr_size later to enqueue
2081          * the correct header size to the host.
2082          */
2083         hdr = &txhdr->vth_uhdr.hdr;
2084         m = *m_head;
2085
2086         error = ENOBUFS;
2087
2088         if (m->m_flags & M_VLANTAG) {
2089                 //m = ether_vlanencap(m, m->m_pkthdr.ether_vtag);
2090                 m = vtnet_vlan_tag_insert(m);
2091                 if ((*m_head = m) == NULL)
2092                         goto fail;
2093                 m->m_flags &= ~M_VLANTAG;
2094         }
2095
2096         if (m->m_pkthdr.csum_flags != 0) {
2097                 m = vtnet_tx_offload(sc, m, hdr);
2098                 if ((*m_head = m) == NULL)
2099                         goto fail;
2100         }
2101
2102         error = vtnet_enqueue_txbuf(sc, m_head, txhdr);
2103 fail:
2104         if (error != 0)
2105                 vtnet_enqueue_txhdr(sc, txhdr);
2106         return (error);
2107 }
2108
2109 static void
2110 vtnet_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
2111 {
2112         struct vtnet_softc *sc;
2113         struct virtqueue *vq;
2114         struct mbuf *m0;
2115         int enq;
2116
2117         sc = ifp->if_softc;
2118         vq = sc->vtnet_tx_vq;
2119         enq = 0;
2120
2121         ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq);
2122         ASSERT_SERIALIZED(&sc->vtnet_tx_slz);
2123
2124         if ((ifp->if_flags & (IFF_RUNNING)) !=
2125             IFF_RUNNING || ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0))
2126                 return;
2127
2128 #ifdef VTNET_TX_INTR_MODERATION
2129         if (virtqueue_nused(vq) >= sc->vtnet_tx_size / 2)
2130                 vtnet_txeof(sc);
2131 #endif
2132
2133         while (!ifsq_is_empty(ifsq)) {
2134                 if (virtqueue_full(vq)) {
2135                         ifsq_set_oactive(ifsq);
2136                         break;
2137                 }
2138
2139                 m0 = ifsq_dequeue(ifsq);
2140                 if (m0 == NULL)
2141                         break;
2142
2143                 if (vtnet_encap(sc, &m0) != 0) {
2144                         if (m0 == NULL)
2145                                 break;
2146                         ifsq_prepend(ifsq, m0);
2147                         ifsq_set_oactive(ifsq);
2148                         break;
2149                 }
2150
2151                 enq++;
2152                 ETHER_BPF_MTAP(ifp, m0);
2153         }
2154
2155         if (enq > 0) {
2156                 virtqueue_notify(vq, NULL);
2157                 ifsq_watchdog_set_count(&sc->vtnet_tx_watchdog,
2158                                         VTNET_WATCHDOG_TIMEOUT);
2159         }
2160 }
2161
2162 static void
2163 vtnet_tx_msix_intr(void *xsc)
2164 {
2165         struct vtnet_softc *sc;
2166         struct ifnet *ifp;
2167         struct ifaltq_subque *ifsq;
2168
2169         sc = xsc;
2170         ifp = sc->vtnet_ifp;
2171         ifsq = ifq_get_subq_default(&ifp->if_snd);
2172
2173         if (!virtqueue_pending(sc->vtnet_tx_vq))
2174                 return;
2175
2176         vtnet_disable_tx_intr(sc);
2177 next:
2178         if ((ifp->if_flags & IFF_RUNNING) == 0) {
2179                 vtnet_enable_tx_intr(sc);
2180                 return;
2181         }
2182
2183         vtnet_txeof(sc);
2184
2185         if (!ifsq_is_empty(ifsq))
2186                 ifsq_devstart(ifsq);
2187
2188         if (vtnet_enable_tx_intr(sc) != 0) {
2189                 vtnet_disable_tx_intr(sc);
2190                 sc->vtnet_stats.tx_task_rescheduled++;
2191                 goto next;
2192         }
2193 }
2194
2195 static void
2196 vtnet_tx_vq_intr(void *xsc)
2197 {
2198         struct vtnet_softc *sc = xsc;
2199
2200         lwkt_serialize_enter(&sc->vtnet_tx_slz);
2201         vtnet_tx_msix_intr(xsc);
2202         lwkt_serialize_exit(&sc->vtnet_tx_slz);
2203 }
2204
2205 static void
2206 vtnet_config_intr(void *arg)
2207 {
2208         struct vtnet_softc *sc;
2209
2210         sc = arg;
2211
2212         vtnet_update_link_status(sc);
2213 }
2214
2215 static void
2216 vtnet_stop(struct vtnet_softc *sc)
2217 {
2218         device_t dev;
2219         struct ifnet *ifp;
2220
2221         dev = sc->vtnet_dev;
2222         ifp = sc->vtnet_ifp;
2223
2224         ASSERT_IFNET_SERIALIZED_ALL(ifp);
2225
2226         ifq_clr_oactive(&ifp->if_snd);
2227         ifsq_watchdog_stop(&sc->vtnet_tx_watchdog);
2228         ifp->if_flags &= ~(IFF_RUNNING);
2229
2230         vtnet_disable_rx_intr(sc);
2231         vtnet_disable_tx_intr(sc);
2232
2233         /*
2234          * Stop the host VirtIO adapter. Note this will reset the host
2235          * adapter's state back to the pre-initialized state, so in
2236          * order to make the device usable again, we must drive it
2237          * through virtio_reinit() and virtio_reinit_complete().
2238          */
2239         virtio_stop(dev);
2240
2241         sc->vtnet_flags &= ~VTNET_FLAG_LINK;
2242
2243         vtnet_free_rx_mbufs(sc);
2244         vtnet_free_tx_mbufs(sc);
2245 }
2246
2247 static int
2248 vtnet_virtio_reinit(struct vtnet_softc *sc)
2249 {
2250         device_t dev;
2251         struct ifnet *ifp;
2252         uint64_t features;
2253         int error;
2254
2255         dev = sc->vtnet_dev;
2256         ifp = sc->vtnet_ifp;
2257         features = sc->vtnet_features;
2258
2259         /*
2260          * Re-negotiate with the host, removing any disabled receive
2261          * features. Transmit features are disabled only on our side
2262          * via if_capenable and if_hwassist.
2263          */
2264
2265         if (ifp->if_capabilities & IFCAP_RXCSUM) {
2266                 if ((ifp->if_capenable & IFCAP_RXCSUM) == 0)
2267                         features &= ~VIRTIO_NET_F_GUEST_CSUM;
2268         }
2269
2270 #if 0   /* IFCAP_LRO doesn't exist in DragonFly. */
2271         if (ifp->if_capabilities & IFCAP_LRO) {
2272                 if ((ifp->if_capenable & IFCAP_LRO) == 0)
2273                         features &= ~VTNET_LRO_FEATURES;
2274         }
2275 #endif
2276
2277         if (ifp->if_capabilities & IFCAP_VLAN_HWFILTER) {
2278                 if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
2279                         features &= ~VIRTIO_NET_F_CTRL_VLAN;
2280         }
2281
2282         error = virtio_reinit(dev, features);
2283         if (error)
2284                 device_printf(dev, "virtio reinit error %d\n", error);
2285
2286         return (error);
2287 }
2288
2289 static void
2290 vtnet_init(void *xsc)
2291 {
2292         struct vtnet_softc *sc;
2293         device_t dev;
2294         struct ifnet *ifp;
2295         int error;
2296
2297         sc = xsc;
2298         dev = sc->vtnet_dev;
2299         ifp = sc->vtnet_ifp;
2300
2301         ASSERT_IFNET_SERIALIZED_ALL(ifp);
2302
2303         if (ifp->if_flags & IFF_RUNNING)
2304                 return;
2305
2306         /* Stop host's adapter, cancel any pending I/O. */
2307         vtnet_stop(sc);
2308
2309         /* Reinitialize the host device. */
2310         error = vtnet_virtio_reinit(sc);
2311         if (error) {
2312                 device_printf(dev,
2313                     "reinitialization failed, stopping device...\n");
2314                 vtnet_stop(sc);
2315                 return;
2316         }
2317
2318         /* Update host with assigned MAC address. */
2319         bcopy(IF_LLADDR(ifp), sc->vtnet_hwaddr, ETHER_ADDR_LEN);
2320         vtnet_set_hwaddr(sc);
2321
2322         ifp->if_hwassist = 0;
2323         if (ifp->if_capenable & IFCAP_TXCSUM)
2324                 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;
2325         if (ifp->if_capenable & IFCAP_TSO4)
2326                 ifp->if_hwassist |= CSUM_TSO;
2327
2328         error = vtnet_init_rx_vq(sc);
2329         if (error) {
2330                 device_printf(dev,
2331                     "cannot allocate mbufs for Rx virtqueue\n");
2332                 vtnet_stop(sc);
2333                 return;
2334         }
2335
2336         if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
2337                 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
2338                         /* Restore promiscuous and all-multicast modes. */
2339                         vtnet_rx_filter(sc);
2340
2341                         /* Restore filtered MAC addresses. */
2342                         vtnet_rx_filter_mac(sc);
2343                 }
2344
2345                 /* Restore VLAN filters. */
2346                 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
2347                         vtnet_rx_filter_vlan(sc);
2348         }
2349
2350 #ifdef IFPOLL_ENABLE
2351         if (!(ifp->if_flags & IFF_NPOLLING))
2352 #endif
2353         {
2354                 vtnet_enable_rx_intr(sc);
2355                 vtnet_enable_tx_intr(sc);
2356         }
2357
2358         ifp->if_flags |= IFF_RUNNING;
2359         ifq_clr_oactive(&ifp->if_snd);
2360         ifsq_watchdog_start(&sc->vtnet_tx_watchdog);
2361
2362         virtio_reinit_complete(dev);
2363
2364         vtnet_update_link_status(sc);
2365 }
2366
2367 static void
2368 vtnet_exec_ctrl_cmd(struct vtnet_softc *sc, void *cookie,
2369     struct sglist *sg, int readable, int writable)
2370 {
2371         struct virtqueue *vq;
2372         void *c;
2373
2374         vq = sc->vtnet_ctrl_vq;
2375
2376         ASSERT_IFNET_SERIALIZED_ALL(sc->vtnet_ifp);
2377         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ,
2378             ("no control virtqueue"));
2379         KASSERT(virtqueue_empty(vq),
2380             ("control command already enqueued"));
2381
2382         if (virtqueue_enqueue(vq, cookie, sg, readable, writable) != 0)
2383                 return;
2384
2385         /*
2386          * XXX We can safely drop the serializer between here, and the end of
2387          *     the function, when we can correctly sleep for this command to
2388          *     be finished.
2389          */
2390         virtqueue_notify(vq, NULL);
2391
2392         /*
2393          * Poll until the command is complete. Previously, we would
2394          * sleep until the control virtqueue interrupt handler woke
2395          * us up, but dropping the VTNET_MTX leads to serialization
2396          * difficulties.
2397          *
2398          * Furthermore, it appears QEMU/KVM only allocates three MSIX
2399          * vectors. Two of those vectors are needed for the Rx and Tx
2400          * virtqueues. We do not support sharing both a Vq and config
2401          * changed notification on the same MSIX vector.
2402          */
2403         c = virtqueue_poll(vq, NULL);
2404         KASSERT(c == cookie, ("unexpected control command response"));
2405 }
2406
2407 static int
2408 vtnet_ctrl_mac_cmd(struct vtnet_softc *sc, uint8_t *hwaddr)
2409 {
2410         struct {
2411                 struct virtio_net_ctrl_hdr hdr __aligned(2);
2412                 uint8_t pad1;
2413                 char aligned_hwaddr[ETHER_ADDR_LEN] __aligned(8);
2414                 uint8_t pad2;
2415                 uint8_t ack;
2416         } s;
2417         struct sglist_seg segs[3];
2418         struct sglist sg;
2419         int error;
2420
2421         s.hdr.class = VIRTIO_NET_CTRL_MAC;
2422         s.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET;
2423         s.ack = VIRTIO_NET_ERR;
2424
2425         /* Copy the mac address into physically contiguous memory */
2426         memcpy(s.aligned_hwaddr, hwaddr, ETHER_ADDR_LEN);
2427
2428         sglist_init(&sg, 3, segs);
2429         error = 0;
2430         error |= sglist_append(&sg, &s.hdr,
2431             sizeof(struct virtio_net_ctrl_hdr));
2432         error |= sglist_append(&sg, s.aligned_hwaddr, ETHER_ADDR_LEN);
2433         error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
2434         KASSERT(error == 0 && sg.sg_nseg == 3,
2435             ("%s: error %d adding set MAC msg to sglist", __func__, error));
2436
2437         vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
2438
2439         return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
2440 }
2441
2442 static void
2443 vtnet_rx_filter(struct vtnet_softc *sc)
2444 {
2445         device_t dev;
2446         struct ifnet *ifp;
2447
2448         dev = sc->vtnet_dev;
2449         ifp = sc->vtnet_ifp;
2450
2451         ASSERT_IFNET_SERIALIZED_ALL(ifp);
2452         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
2453             ("CTRL_RX feature not negotiated"));
2454
2455         if (vtnet_set_promisc(sc, ifp->if_flags & IFF_PROMISC) != 0)
2456                 device_printf(dev, "cannot %s promiscuous mode\n",
2457                     (ifp->if_flags & IFF_PROMISC) ? "enable" : "disable");
2458
2459         if (vtnet_set_allmulti(sc, ifp->if_flags & IFF_ALLMULTI) != 0)
2460                 device_printf(dev, "cannot %s all-multicast mode\n",
2461                     (ifp->if_flags & IFF_ALLMULTI) ? "enable" : "disable");
2462 }
2463
2464 static int
2465 vtnet_ctrl_rx_cmd(struct vtnet_softc *sc, int cmd, int on)
2466 {
2467         struct sglist_seg segs[3];
2468         struct sglist sg;
2469         struct {
2470                 struct virtio_net_ctrl_hdr hdr __aligned(2);
2471                 uint8_t pad1;
2472                 uint8_t onoff;
2473                 uint8_t pad2;
2474                 uint8_t ack;
2475         } s;
2476         int error;
2477
2478         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
2479             ("%s: CTRL_RX feature not negotiated", __func__));
2480
2481         s.hdr.class = VIRTIO_NET_CTRL_RX;
2482         s.hdr.cmd = cmd;
2483         s.onoff = !!on;
2484         s.ack = VIRTIO_NET_ERR;
2485
2486         sglist_init(&sg, 3, segs);
2487         error = 0;
2488         error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
2489         error |= sglist_append(&sg, &s.onoff, sizeof(uint8_t));
2490         error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
2491         KASSERT(error == 0 && sg.sg_nseg == 3,
2492             ("%s: error %d adding Rx message to sglist", __func__, error));
2493
2494         vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
2495
2496         return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
2497 }
2498
2499 static int
2500 vtnet_set_promisc(struct vtnet_softc *sc, int on)
2501 {
2502
2503         return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_PROMISC, on));
2504 }
2505
2506 static int
2507 vtnet_set_allmulti(struct vtnet_softc *sc, int on)
2508 {
2509
2510         return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, on));
2511 }
2512
2513 static void
2514 vtnet_rx_filter_mac(struct vtnet_softc *sc)
2515 {
2516         struct virtio_net_ctrl_hdr hdr __aligned(2);
2517         struct vtnet_mac_filter *filter;
2518         struct sglist_seg segs[4];
2519         struct sglist sg;
2520         struct ifnet *ifp;
2521         struct ifaddr *ifa;
2522         struct ifaddr_container *ifac;
2523         struct ifmultiaddr *ifma;
2524         int ucnt, mcnt, promisc, allmulti, error;
2525         uint8_t ack;
2526
2527         ifp = sc->vtnet_ifp;
2528         ucnt = 0;
2529         mcnt = 0;
2530         promisc = 0;
2531         allmulti = 0;
2532
2533         ASSERT_IFNET_SERIALIZED_ALL(ifp);
2534         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
2535             ("%s: CTRL_RX feature not negotiated", __func__));
2536
2537         /* Use the MAC filtering table allocated in vtnet_attach. */
2538         filter = sc->vtnet_macfilter;
2539         memset(filter, 0, sizeof(struct vtnet_mac_filter));
2540
2541         /* Unicast MAC addresses: */
2542         //if_addr_rlock(ifp);
2543         TAILQ_FOREACH(ifac, &ifp->if_addrheads[mycpuid], ifa_link) {
2544                 ifa = ifac->ifa;
2545                 if (ifa->ifa_addr->sa_family != AF_LINK)
2546                         continue;
2547                 else if (memcmp(LLADDR((struct sockaddr_dl *)ifa->ifa_addr),
2548                     sc->vtnet_hwaddr, ETHER_ADDR_LEN) == 0)
2549                         continue;
2550                 else if (ucnt == VTNET_MAX_MAC_ENTRIES) {
2551                         promisc = 1;
2552                         break;
2553                 }
2554
2555                 bcopy(LLADDR((struct sockaddr_dl *)ifa->ifa_addr),
2556                     &filter->vmf_unicast.macs[ucnt], ETHER_ADDR_LEN);
2557                 ucnt++;
2558         }
2559         //if_addr_runlock(ifp);
2560
2561         if (promisc != 0) {
2562                 filter->vmf_unicast.nentries = 0;
2563                 if_printf(ifp, "more than %d MAC addresses assigned, "
2564                     "falling back to promiscuous mode\n",
2565                     VTNET_MAX_MAC_ENTRIES);
2566         } else
2567                 filter->vmf_unicast.nentries = ucnt;
2568
2569         /* Multicast MAC addresses: */
2570         //if_maddr_rlock(ifp);
2571         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2572                 if (ifma->ifma_addr->sa_family != AF_LINK)
2573                         continue;
2574                 else if (mcnt == VTNET_MAX_MAC_ENTRIES) {
2575                         allmulti = 1;
2576                         break;
2577                 }
2578
2579                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2580                     &filter->vmf_multicast.macs[mcnt], ETHER_ADDR_LEN);
2581                 mcnt++;
2582         }
2583         //if_maddr_runlock(ifp);
2584
2585         if (allmulti != 0) {
2586                 filter->vmf_multicast.nentries = 0;
2587                 if_printf(ifp, "more than %d multicast MAC addresses "
2588                     "assigned, falling back to all-multicast mode\n",
2589                     VTNET_MAX_MAC_ENTRIES);
2590         } else
2591                 filter->vmf_multicast.nentries = mcnt;
2592
2593         if (promisc != 0 && allmulti != 0)
2594                 goto out;
2595
2596         hdr.class = VIRTIO_NET_CTRL_MAC;
2597         hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
2598         ack = VIRTIO_NET_ERR;
2599
2600         sglist_init(&sg, 4, segs);
2601         error = 0;
2602         error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
2603         error |= sglist_append(&sg, &filter->vmf_unicast,
2604             sizeof(uint32_t) + filter->vmf_unicast.nentries * ETHER_ADDR_LEN);
2605         error |= sglist_append(&sg, &filter->vmf_multicast,
2606             sizeof(uint32_t) + filter->vmf_multicast.nentries * ETHER_ADDR_LEN);
2607         error |= sglist_append(&sg, &ack, sizeof(uint8_t));
2608         KASSERT(error == 0 && sg.sg_nseg == 4,
2609             ("%s: error %d adding MAC filter msg to sglist", __func__, error));
2610
2611         vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
2612
2613         if (ack != VIRTIO_NET_OK)
2614                 if_printf(ifp, "error setting host MAC filter table\n");
2615
2616 out:
2617         if (promisc != 0 && vtnet_set_promisc(sc, 1) != 0)
2618                 if_printf(ifp, "cannot enable promiscuous mode\n");
2619         if (allmulti != 0 && vtnet_set_allmulti(sc, 1) != 0)
2620                 if_printf(ifp, "cannot enable all-multicast mode\n");
2621 }
2622
2623 static int
2624 vtnet_exec_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
2625 {
2626         struct sglist_seg segs[3];
2627         struct sglist sg;
2628         struct {
2629                 struct virtio_net_ctrl_hdr hdr __aligned(2);
2630                 uint8_t pad1;
2631                 uint16_t tag;
2632                 uint8_t pad2;
2633                 uint8_t ack;
2634         } s;
2635         int error;
2636
2637         s.hdr.class = VIRTIO_NET_CTRL_VLAN;
2638         s.hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
2639         s.tag = tag;
2640         s.ack = VIRTIO_NET_ERR;
2641
2642         sglist_init(&sg, 3, segs);
2643         error = 0;
2644         error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
2645         error |= sglist_append(&sg, &s.tag, sizeof(uint16_t));
2646         error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
2647         KASSERT(error == 0 && sg.sg_nseg == 3,
2648             ("%s: error %d adding VLAN message to sglist", __func__, error));
2649
2650         vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
2651
2652         return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
2653 }
2654
2655 static void
2656 vtnet_rx_filter_vlan(struct vtnet_softc *sc)
2657 {
2658         uint32_t w;
2659         uint16_t tag;
2660         int i, bit, nvlans;
2661
2662         ASSERT_IFNET_SERIALIZED_ALL(sc->vtnet_ifp);
2663         KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER,
2664             ("%s: VLAN_FILTER feature not negotiated", __func__));
2665
2666         nvlans = sc->vtnet_nvlans;
2667
2668         /* Enable the filter for each configured VLAN. */
2669         for (i = 0; i < VTNET_VLAN_SHADOW_SIZE && nvlans > 0; i++) {
2670                 w = sc->vtnet_vlan_shadow[i];
2671                 while ((bit = ffs(w) - 1) != -1) {
2672                         w &= ~(1 << bit);
2673                         tag = sizeof(w) * CHAR_BIT * i + bit;
2674                         nvlans--;
2675
2676                         if (vtnet_exec_vlan_filter(sc, 1, tag) != 0) {
2677                                 device_printf(sc->vtnet_dev,
2678                                     "cannot enable VLAN %d filter\n", tag);
2679                         }
2680                 }
2681         }
2682
2683         KASSERT(nvlans == 0, ("VLAN count incorrect"));
2684 }
2685
2686 static void
2687 vtnet_update_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
2688 {
2689         struct ifnet *ifp;
2690         int idx, bit;
2691
2692         ifp = sc->vtnet_ifp;
2693         idx = (tag >> 5) & 0x7F;
2694         bit = tag & 0x1F;
2695
2696         if (tag == 0 || tag > 4095)
2697                 return;
2698
2699         ifnet_serialize_all(ifp);
2700
2701         /* Update shadow VLAN table. */
2702         if (add) {
2703                 sc->vtnet_nvlans++;
2704                 sc->vtnet_vlan_shadow[idx] |= (1 << bit);
2705         } else {
2706                 sc->vtnet_nvlans--;
2707                 sc->vtnet_vlan_shadow[idx] &= ~(1 << bit);
2708         }
2709
2710         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER &&
2711             vtnet_exec_vlan_filter(sc, add, tag) != 0) {
2712                 device_printf(sc->vtnet_dev,
2713                     "cannot %s VLAN %d %s the host filter table\n",
2714                     add ? "add" : "remove", tag, add ? "to" : "from");
2715         }
2716
2717         ifnet_deserialize_all(ifp);
2718 }
2719
2720 static void
2721 vtnet_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
2722 {
2723
2724         if (ifp->if_softc != arg)
2725                 return;
2726
2727         vtnet_update_vlan_filter(arg, 1, tag);
2728 }
2729
2730 static void
2731 vtnet_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
2732 {
2733
2734         if (ifp->if_softc != arg)
2735                 return;
2736
2737         vtnet_update_vlan_filter(arg, 0, tag);
2738 }
2739
2740 static int
2741 vtnet_ifmedia_upd(struct ifnet *ifp)
2742 {
2743         struct vtnet_softc *sc;
2744         struct ifmedia *ifm;
2745
2746         sc = ifp->if_softc;
2747         ifm = &sc->vtnet_media;
2748
2749         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
2750                 return (EINVAL);
2751
2752         return (0);
2753 }
2754
2755 static void
2756 vtnet_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
2757 {
2758         struct vtnet_softc *sc;
2759
2760         sc = ifp->if_softc;
2761
2762         ifmr->ifm_status = IFM_AVALID;
2763         ifmr->ifm_active = IFM_ETHER;
2764
2765         if (vtnet_is_link_up(sc) != 0) {
2766                 ifmr->ifm_status |= IFM_ACTIVE;
2767                 ifmr->ifm_active |= VTNET_MEDIATYPE;
2768         } else
2769                 ifmr->ifm_active |= IFM_NONE;
2770 }
2771
2772 static void
2773 vtnet_add_statistics(struct vtnet_softc *sc)
2774 {
2775         device_t dev;
2776         struct vtnet_statistics *stats;
2777         struct sysctl_ctx_list *ctx;
2778         struct sysctl_oid *tree;
2779         struct sysctl_oid_list *child;
2780
2781         dev = sc->vtnet_dev;
2782         stats = &sc->vtnet_stats;
2783         ctx = device_get_sysctl_ctx(dev);
2784         tree = device_get_sysctl_tree(dev);
2785         child = SYSCTL_CHILDREN(tree);
2786
2787         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "mbuf_alloc_failed",
2788             CTLFLAG_RD, &stats->mbuf_alloc_failed, 0,
2789             "Mbuf cluster allocation failures");
2790
2791         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_frame_too_large",
2792             CTLFLAG_RD, &stats->rx_frame_too_large, 0,
2793             "Received frame larger than the mbuf chain");
2794         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_enq_replacement_failed",
2795             CTLFLAG_RD, &stats->rx_enq_replacement_failed, 0,
2796             "Enqueuing the replacement receive mbuf failed");
2797         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_mergeable_failed",
2798             CTLFLAG_RD, &stats->rx_mergeable_failed, 0,
2799             "Mergeable buffers receive failures");
2800         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ethtype",
2801             CTLFLAG_RD, &stats->rx_csum_bad_ethtype, 0,
2802             "Received checksum offloaded buffer with unsupported "
2803             "Ethernet type");
2804         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ipproto",
2805             CTLFLAG_RD, &stats->rx_csum_bad_ipproto, 0,
2806             "Received checksum offloaded buffer with incorrect IP protocol");
2807         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_offset",
2808             CTLFLAG_RD, &stats->rx_csum_bad_offset, 0,
2809             "Received checksum offloaded buffer with incorrect offset");
2810         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_failed",
2811             CTLFLAG_RD, &stats->rx_csum_failed, 0,
2812             "Received buffer checksum offload failed");
2813         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_offloaded",
2814             CTLFLAG_RD, &stats->rx_csum_offloaded, 0,
2815             "Received buffer checksum offload succeeded");
2816         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_task_rescheduled",
2817             CTLFLAG_RD, &stats->rx_task_rescheduled, 0,
2818             "Times the receive interrupt task rescheduled itself");
2819
2820         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_bad_ethtype",
2821             CTLFLAG_RD, &stats->tx_csum_bad_ethtype, 0,
2822             "Aborted transmit of checksum offloaded buffer with unknown "
2823             "Ethernet type");
2824         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_bad_ethtype",
2825             CTLFLAG_RD, &stats->tx_tso_bad_ethtype, 0,
2826             "Aborted transmit of TSO buffer with unknown Ethernet type");
2827         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defragged",
2828             CTLFLAG_RD, &stats->tx_defragged, 0,
2829             "Transmit mbufs defragged");
2830         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defrag_failed",
2831             CTLFLAG_RD, &stats->tx_defrag_failed, 0,
2832             "Aborted transmit of buffer because defrag failed");
2833         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_offloaded",
2834             CTLFLAG_RD, &stats->tx_csum_offloaded, 0,
2835             "Offloaded checksum of transmitted buffer");
2836         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_offloaded",
2837             CTLFLAG_RD, &stats->tx_tso_offloaded, 0,
2838             "Segmentation offload of transmitted buffer");
2839         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_task_rescheduled",
2840             CTLFLAG_RD, &stats->tx_task_rescheduled, 0,
2841             "Times the transmit interrupt task rescheduled itself");
2842 }
2843
2844 static int
2845 vtnet_enable_rx_intr(struct vtnet_softc *sc)
2846 {
2847
2848         return (virtqueue_enable_intr(sc->vtnet_rx_vq));
2849 }
2850
2851 static void
2852 vtnet_disable_rx_intr(struct vtnet_softc *sc)
2853 {
2854
2855         virtqueue_disable_intr(sc->vtnet_rx_vq);
2856 }
2857
2858 static int
2859 vtnet_enable_tx_intr(struct vtnet_softc *sc)
2860 {
2861
2862 #ifdef VTNET_TX_INTR_MODERATION
2863         return (0);
2864 #else
2865         return (virtqueue_enable_intr(sc->vtnet_tx_vq));
2866 #endif
2867 }
2868
2869 static void
2870 vtnet_disable_tx_intr(struct vtnet_softc *sc)
2871 {
2872
2873         virtqueue_disable_intr(sc->vtnet_tx_vq);
2874 }