sys/dev/virtual/vmware/vmxnet3/if_vmx.c

   1 /*-
   2  * Copyright (c) 2013 Tsubai Masanari
   3  * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org>
   4  *
   5  * Permission to use, copy, modify, and distribute this software for any
   6  * purpose with or without fee is hereby granted, provided that the above
   7  * copyright notice and this permission notice appear in all copies.
   8  *
   9  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  10  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  11  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  12  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  13  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  14  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  15  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  16  *
  17  * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $
  18  * $FreeBSD: head/sys/dev/vmware/vmxnet3/if_vmx.c 318867 2017-05-25 10:49:56Z avg $
  19  */
  20
  21 /* Driver for VMware vmxnet3 virtual ethernet devices. */
  22
  23 #include <sys/param.h>
  24 #include <sys/systm.h>
  25 #include <sys/device.h>
  26 #include <sys/eventhandler.h>
  27 #include <sys/kernel.h>
  28 #include <sys/endian.h>
  29 #include <sys/sockio.h>
  30 #include <sys/mbuf.h>
  31 #include <sys/malloc.h>
  32 #include <sys/module.h>
  33 #include <sys/socket.h>
  34 #include <sys/sysctl.h>
  35 #include <sys/taskqueue.h>
  36 #include <vm/vm.h>
  37 #include <vm/pmap.h>
  38
  39 #include <net/ethernet.h>
  40 #include <net/if.h>
  41 #include <net/if_var.h>
  42 #include <net/ifq_var.h>
  43 #include <net/if_arp.h>
  44 #include <net/if_dl.h>
  45 #include <net/if_types.h>
  46 #include <net/if_media.h>
  47 #include <net/vlan/if_vlan_ether.h>
  48 #include <net/vlan/if_vlan_var.h>
  49
  50 #include <net/bpf.h>
  51
  52 #include <netinet/in_systm.h>
  53 #include <netinet/in.h>
  54 #include <netinet/ip.h>
  55 #include <netinet/ip6.h>
  56 #include <netinet6/ip6_var.h>
  57 #include <netinet/udp.h>
  58 #include <netinet/tcp.h>
  59
  60 #include <sys/in_cksum.h>
  61
  62 #include <sys/bus.h>
  63 #include <sys/rman.h>
  64
  65 #include <bus/pci/pcireg.h>
  66 #include <bus/pci/pcivar.h>
  67
  68 #define VMXNET3_LEGACY_TX 1     /* XXX we need this at the moment */
  69 #include "if_vmxreg.h"
  70 #include "if_vmxvar.h"
  71
  72 #include "opt_inet.h"
  73 #include "opt_inet6.h"
  74
  75 #ifdef VMXNET3_FAILPOINTS
  76 #include <sys/fail.h>
  77 static SYSCTL_NODE(DEBUG_FP, OID_AUTO, vmxnet3, CTLFLAG_RW, 0,
  78     "vmxnet3 fail points");
  79 #define VMXNET3_FP      _debug_fail_point_vmxnet3
  80 #endif
  81
  82 static int      vmxnet3_probe(device_t);
  83 static int      vmxnet3_attach(device_t);
  84 static int      vmxnet3_detach(device_t);
  85 static int      vmxnet3_shutdown(device_t);
  86
  87 static int      vmxnet3_alloc_resources(struct vmxnet3_softc *);
  88 static void     vmxnet3_free_resources(struct vmxnet3_softc *);
  89 static int      vmxnet3_check_version(struct vmxnet3_softc *);
  90 static void     vmxnet3_initial_config(struct vmxnet3_softc *);
  91 static void     vmxnet3_check_multiqueue(struct vmxnet3_softc *);
  92
  93 #ifdef __FreeBSD__
  94 static int      vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *);
  95 static int      vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *);
  96 #else
  97 static int      vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *);
  98 #endif
  99 static int      vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *);
 100 static int      vmxnet3_alloc_interrupt(struct vmxnet3_softc *, int, int,
 101                     struct vmxnet3_interrupt *);
 102 static int      vmxnet3_alloc_intr_resources(struct vmxnet3_softc *);
 103 #ifdef __FreeBSD__
 104 static int      vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *);
 105 #endif
 106 static int      vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *);
 107 static int      vmxnet3_setup_interrupts(struct vmxnet3_softc *);
 108 static int      vmxnet3_alloc_interrupts(struct vmxnet3_softc *);
 109
 110 static void     vmxnet3_free_interrupt(struct vmxnet3_softc *,
 111                     struct vmxnet3_interrupt *);
 112 static void     vmxnet3_free_interrupts(struct vmxnet3_softc *);
 113
 114 #ifndef VMXNET3_LEGACY_TX
 115 static int      vmxnet3_alloc_taskqueue(struct vmxnet3_softc *);
 116 static void     vmxnet3_start_taskqueue(struct vmxnet3_softc *);
 117 static void     vmxnet3_drain_taskqueue(struct vmxnet3_softc *);
 118 static void     vmxnet3_free_taskqueue(struct vmxnet3_softc *);
 119 #endif
 120
 121 static int      vmxnet3_init_rxq(struct vmxnet3_softc *, int);
 122 static int      vmxnet3_init_txq(struct vmxnet3_softc *, int);
 123 static int      vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *);
 124 static void     vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *);
 125 static void     vmxnet3_destroy_txq(struct vmxnet3_txqueue *);
 126 static void     vmxnet3_free_rxtx_queues(struct vmxnet3_softc *);
 127
 128 static int      vmxnet3_alloc_shared_data(struct vmxnet3_softc *);
 129 static void     vmxnet3_free_shared_data(struct vmxnet3_softc *);
 130 static int      vmxnet3_alloc_txq_data(struct vmxnet3_softc *);
 131 static void     vmxnet3_free_txq_data(struct vmxnet3_softc *);
 132 static int      vmxnet3_alloc_rxq_data(struct vmxnet3_softc *);
 133 static void     vmxnet3_free_rxq_data(struct vmxnet3_softc *);
 134 static int      vmxnet3_alloc_queue_data(struct vmxnet3_softc *);
 135 static void     vmxnet3_free_queue_data(struct vmxnet3_softc *);
 136 static int      vmxnet3_alloc_mcast_table(struct vmxnet3_softc *);
 137 static void     vmxnet3_init_shared_data(struct vmxnet3_softc *);
 138 static void     vmxnet3_init_hwassist(struct vmxnet3_softc *);
 139 static void     vmxnet3_reinit_interface(struct vmxnet3_softc *);
 140 static void     vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *);
 141 static void     vmxnet3_reinit_shared_data(struct vmxnet3_softc *);
 142 static int      vmxnet3_alloc_data(struct vmxnet3_softc *);
 143 static void     vmxnet3_free_data(struct vmxnet3_softc *);
 144 static int      vmxnet3_setup_interface(struct vmxnet3_softc *);
 145
 146 static void     vmxnet3_evintr(struct vmxnet3_softc *);
 147 static void     vmxnet3_txq_eof(struct vmxnet3_txqueue *);
 148 static void     vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *);
 149 static int      vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *);
 150 static void     vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *,
 151                     struct vmxnet3_rxring *, int);
 152 static void     vmxnet3_rxq_eof(struct vmxnet3_rxqueue *);
 153 static void     vmxnet3_legacy_intr(void *);
 154 #ifdef __FreeBSD__
 155 static void     vmxnet3_txq_intr(void *);
 156 static void     vmxnet3_rxq_intr(void *);
 157 static void     vmxnet3_event_intr(void *);
 158 #endif
 159
 160 static void     vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
 161 static void     vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
 162 static void     vmxnet3_stop(struct vmxnet3_softc *);
 163
 164 static void     vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
 165 static int      vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
 166 static int      vmxnet3_reinit_queues(struct vmxnet3_softc *);
 167 static int      vmxnet3_enable_device(struct vmxnet3_softc *);
 168 static void     vmxnet3_reinit_rxfilters(struct vmxnet3_softc *);
 169 static int      vmxnet3_reinit(struct vmxnet3_softc *);
 170 static void     vmxnet3_init_locked(struct vmxnet3_softc *);
 171 static void     vmxnet3_init(void *);
 172
 173 static int      vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *,struct mbuf *,
 174                     int *, int *, int *);
 175 static int      vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **,
 176                     bus_dmamap_t, bus_dma_segment_t [], int *);
 177 static void     vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t);
 178 static int      vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **);
 179 #ifdef VMXNET3_LEGACY_TX
 180 static void     vmxnet3_start_locked(struct ifnet *);
 181 static void     vmxnet3_start(struct ifnet *, struct ifaltq_subque *);
 182 #else
 183 static int      vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *,
 184                     struct mbuf *);
 185 static int      vmxnet3_txq_mq_start(struct ifnet *, struct mbuf *);
 186 static void     vmxnet3_txq_tq_deferred(void *, int);
 187 #endif
 188 static void     vmxnet3_txq_start(struct vmxnet3_txqueue *);
 189 static void     vmxnet3_tx_start_all(struct vmxnet3_softc *);
 190
 191 static void     vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int,
 192                     uint16_t);
 193 static void     vmxnet3_register_vlan(void *, struct ifnet *, uint16_t);
 194 static void     vmxnet3_unregister_vlan(void *, struct ifnet *, uint16_t);
 195 static void     vmxnet3_set_rxfilter(struct vmxnet3_softc *);
 196 static int      vmxnet3_change_mtu(struct vmxnet3_softc *, int);
 197 static int      vmxnet3_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 198
 199 #ifndef VMXNET3_LEGACY_TX
 200 static void     vmxnet3_qflush(struct ifnet *);
 201 #endif
 202
 203 static int      vmxnet3_watchdog(struct vmxnet3_txqueue *);
 204 static void     vmxnet3_refresh_host_stats(struct vmxnet3_softc *);
 205 static void     vmxnet3_txq_accum_stats(struct vmxnet3_txqueue *,
 206                     struct vmxnet3_txq_stats *);
 207 static void     vmxnet3_rxq_accum_stats(struct vmxnet3_rxqueue *,
 208                     struct vmxnet3_rxq_stats *);
 209 static void     vmxnet3_tick(void *);
 210 static void     vmxnet3_link_status(struct vmxnet3_softc *);
 211 static void     vmxnet3_media_status(struct ifnet *, struct ifmediareq *);
 212 static int      vmxnet3_media_change(struct ifnet *);
 213 static void     vmxnet3_set_lladdr(struct vmxnet3_softc *);
 214 static void     vmxnet3_get_lladdr(struct vmxnet3_softc *);
 215
 216 static void     vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *,
 217                     struct sysctl_ctx_list *, struct sysctl_oid_list *);
 218 static void     vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *,
 219                     struct sysctl_ctx_list *, struct sysctl_oid_list *);
 220 static void     vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *,
 221                     struct sysctl_ctx_list *, struct sysctl_oid_list *);
 222 static void     vmxnet3_setup_sysctl(struct vmxnet3_softc *);
 223
 224 static void     vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t,
 225                     uint32_t);
 226 static uint32_t vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t);
 227 static void     vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t,
 228                     uint32_t);
 229 static void     vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t);
 230 static uint32_t vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t);
 231
 232 static void     vmxnet3_enable_intr(struct vmxnet3_softc *, int);
 233 static void     vmxnet3_disable_intr(struct vmxnet3_softc *, int);
 234 static void     vmxnet3_enable_all_intrs(struct vmxnet3_softc *);
 235 static void     vmxnet3_disable_all_intrs(struct vmxnet3_softc *);
 236
 237 static int      vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t,
 238                     bus_size_t, struct vmxnet3_dma_alloc *);
 239 static void     vmxnet3_dma_free(struct vmxnet3_softc *,
 240                     struct vmxnet3_dma_alloc *);
 241 static int      vmxnet3_tunable_int(struct vmxnet3_softc *,
 242                     const char *, int);
 243
 244 typedef enum {
 245         VMXNET3_BARRIER_RD,
 246         VMXNET3_BARRIER_WR,
 247         VMXNET3_BARRIER_RDWR,
 248 } vmxnet3_barrier_t;
 249
 250 static void     vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t);
 251
 252 /* Tunables. */
 253 static int vmxnet3_mq_disable = 0;
 254 TUNABLE_INT("hw.vmx.mq_disable", &vmxnet3_mq_disable);
 255 static int vmxnet3_default_txnqueue = VMXNET3_DEF_TX_QUEUES;
 256 TUNABLE_INT("hw.vmx.txnqueue", &vmxnet3_default_txnqueue);
 257 static int vmxnet3_default_rxnqueue = VMXNET3_DEF_RX_QUEUES;
 258 TUNABLE_INT("hw.vmx.rxnqueue", &vmxnet3_default_rxnqueue);
 259 static int vmxnet3_default_txndesc = VMXNET3_DEF_TX_NDESC;
 260 TUNABLE_INT("hw.vmx.txndesc", &vmxnet3_default_txndesc);
 261 static int vmxnet3_default_rxndesc = VMXNET3_DEF_RX_NDESC;
 262 TUNABLE_INT("hw.vmx.rxndesc", &vmxnet3_default_rxndesc);
 263
 264 static device_method_t vmxnet3_methods[] = {
 265         /* Device interface. */
 266         DEVMETHOD(device_probe,         vmxnet3_probe),
 267         DEVMETHOD(device_attach,        vmxnet3_attach),
 268         DEVMETHOD(device_detach,        vmxnet3_detach),
 269         DEVMETHOD(device_shutdown,      vmxnet3_shutdown),
 270
 271         DEVMETHOD_END
 272 };
 273
 274 static driver_t vmxnet3_driver = {
 275         "vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc)
 276 };
 277
 278 static devclass_t vmxnet3_devclass;
 279 DRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, NULL, NULL);
 280
 281 MODULE_DEPEND(vmx, pci, 1, 1, 1);
 282 MODULE_DEPEND(vmx, ether, 1, 1, 1);
 283
 284 #define VMXNET3_VMWARE_VENDOR_ID        0x15AD
 285 #define VMXNET3_VMWARE_DEVICE_ID        0x07B0
 286
 287 static int
 288 vmxnet3_probe(device_t dev)
 289 {
 290
 291         if (pci_get_vendor(dev) == VMXNET3_VMWARE_VENDOR_ID &&
 292             pci_get_device(dev) == VMXNET3_VMWARE_DEVICE_ID) {
 293                 device_set_desc(dev, "VMware VMXNET3 Ethernet Adapter");
 294                 return (BUS_PROBE_DEFAULT);
 295         }
 296
 297         return (ENXIO);
 298 }
 299
 300 static int
 301 vmxnet3_attach(device_t dev)
 302 {
 303         struct vmxnet3_softc *sc;
 304         int error;
 305
 306         sc = device_get_softc(dev);
 307         sc->vmx_dev = dev;
 308
 309         pci_enable_busmaster(dev);
 310
 311         VMXNET3_CORE_LOCK_INIT(sc, device_get_nameunit(dev));
 312         callout_init_lk(&sc->vmx_tick, &sc->vmx_lock);
 313
 314         vmxnet3_initial_config(sc);
 315
 316         error = vmxnet3_alloc_resources(sc);
 317         if (error)
 318                 goto fail;
 319
 320         error = vmxnet3_check_version(sc);
 321         if (error)
 322                 goto fail;
 323
 324         error = vmxnet3_alloc_rxtx_queues(sc);
 325         if (error)
 326                 goto fail;
 327
 328 #ifndef VMXNET3_LEGACY_TX
 329         error = vmxnet3_alloc_taskqueue(sc);
 330         if (error)
 331                 goto fail;
 332 #endif
 333
 334         error = vmxnet3_alloc_interrupts(sc);
 335         if (error)
 336                 goto fail;
 337
 338         vmxnet3_check_multiqueue(sc);
 339
 340         error = vmxnet3_alloc_data(sc);
 341         if (error)
 342                 goto fail;
 343
 344         error = vmxnet3_setup_interface(sc);
 345         if (error)
 346                 goto fail;
 347
 348         error = vmxnet3_setup_interrupts(sc);
 349         if (error) {
 350                 ether_ifdetach(sc->vmx_ifp);
 351                 device_printf(dev, "could not set up interrupt\n");
 352                 goto fail;
 353         }
 354
 355         vmxnet3_setup_sysctl(sc);
 356 #ifndef VMXNET3_LEGACY_TX
 357         vmxnet3_start_taskqueue(sc);
 358 #endif
 359
 360 fail:
 361         if (error)
 362                 vmxnet3_detach(dev);
 363
 364         return (error);
 365 }
 366
 367 static int
 368 vmxnet3_detach(device_t dev)
 369 {
 370         struct vmxnet3_softc *sc;
 371         struct ifnet *ifp;
 372
 373         sc = device_get_softc(dev);
 374         ifp = sc->vmx_ifp;
 375
 376         if (device_is_attached(dev)) {
 377                 VMXNET3_CORE_LOCK(sc);
 378                 vmxnet3_stop(sc);
 379                 VMXNET3_CORE_UNLOCK(sc);
 380
 381                 callout_stop_sync(&sc->vmx_tick);
 382 #ifndef VMXNET3_LEGACY_TX
 383                 vmxnet3_drain_taskqueue(sc);
 384 #endif
 385
 386                 ether_ifdetach(ifp);
 387         }
 388
 389         if (sc->vmx_vlan_attach != NULL) {
 390                 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_attach);
 391                 sc->vmx_vlan_attach = NULL;
 392         }
 393         if (sc->vmx_vlan_detach != NULL) {
 394                 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_detach);
 395                 sc->vmx_vlan_detach = NULL;
 396         }
 397
 398 #ifndef VMXNET3_LEGACY_TX
 399         vmxnet3_free_taskqueue(sc);
 400 #endif
 401         vmxnet3_free_interrupts(sc);
 402
 403         if (ifp != NULL) {
 404                 if_free(ifp);
 405                 sc->vmx_ifp = NULL;
 406         }
 407
 408         ifmedia_removeall(&sc->vmx_media);
 409
 410         vmxnet3_free_data(sc);
 411         vmxnet3_free_resources(sc);
 412         vmxnet3_free_rxtx_queues(sc);
 413
 414         VMXNET3_CORE_LOCK_DESTROY(sc);
 415
 416         return (0);
 417 }
 418
 419 static int
 420 vmxnet3_shutdown(device_t dev)
 421 {
 422
 423         return (0);
 424 }
 425
 426 static int
 427 vmxnet3_alloc_resources(struct vmxnet3_softc *sc)
 428 {
 429         device_t dev;
 430         int rid;
 431
 432         dev = sc->vmx_dev;
 433
 434         rid = PCIR_BAR(0);
 435         sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 436             RF_ACTIVE);
 437         if (sc->vmx_res0 == NULL) {
 438                 device_printf(dev,
 439                     "could not map BAR0 memory\n");
 440                 return (ENXIO);
 441         }
 442
 443         sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0);
 444         sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0);
 445
 446         rid = PCIR_BAR(1);
 447         sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 448             RF_ACTIVE);
 449         if (sc->vmx_res1 == NULL) {
 450                 device_printf(dev,
 451                     "could not map BAR1 memory\n");
 452                 return (ENXIO);
 453         }
 454
 455         sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1);
 456         sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1);
 457
 458         if (pci_find_extcap(dev, PCIY_MSIX, NULL) == 0) {
 459                 rid = PCIR_BAR(2);
 460                 sc->vmx_msix_res = bus_alloc_resource_any(dev,
 461                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
 462         }
 463
 464         if (sc->vmx_msix_res == NULL)
 465                 sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX;
 466
 467         return (0);
 468 }
 469
 470 static void
 471 vmxnet3_free_resources(struct vmxnet3_softc *sc)
 472 {
 473         device_t dev;
 474         int rid;
 475
 476         dev = sc->vmx_dev;
 477
 478         if (sc->vmx_res0 != NULL) {
 479                 rid = PCIR_BAR(0);
 480                 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res0);
 481                 sc->vmx_res0 = NULL;
 482         }
 483
 484         if (sc->vmx_res1 != NULL) {
 485                 rid = PCIR_BAR(1);
 486                 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res1);
 487                 sc->vmx_res1 = NULL;
 488         }
 489
 490         if (sc->vmx_msix_res != NULL) {
 491                 rid = PCIR_BAR(2);
 492                 bus_release_resource(dev, SYS_RES_MEMORY, rid,
 493                     sc->vmx_msix_res);
 494                 sc->vmx_msix_res = NULL;
 495         }
 496 }
 497
 498 static int
 499 vmxnet3_check_version(struct vmxnet3_softc *sc)
 500 {
 501         device_t dev;
 502         uint32_t version;
 503
 504         dev = sc->vmx_dev;
 505
 506         version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS);
 507         if ((version & 0x01) == 0) {
 508                 device_printf(dev, "unsupported hardware version %#x\n",
 509                     version);
 510                 return (ENOTSUP);
 511         }
 512         vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1);
 513
 514         version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS);
 515         if ((version & 0x01) == 0) {
 516                 device_printf(dev, "unsupported UPT version %#x\n", version);
 517                 return (ENOTSUP);
 518         }
 519         vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1);
 520
 521         return (0);
 522 }
 523
 524 static int
 525 trunc_powerof2(int val)
 526 {
 527
 528         return (1U << (fls(val) - 1));
 529 }
 530
 531 static void
 532 vmxnet3_initial_config(struct vmxnet3_softc *sc)
 533 {
 534         int nqueue, ndesc;
 535
 536         nqueue = vmxnet3_tunable_int(sc, "txnqueue", vmxnet3_default_txnqueue);
 537         if (nqueue > VMXNET3_MAX_TX_QUEUES || nqueue < 1)
 538                 nqueue = VMXNET3_DEF_TX_QUEUES;
 539         if (nqueue > ncpus)
 540                 nqueue = ncpus;
 541         sc->vmx_max_ntxqueues = trunc_powerof2(nqueue);
 542
 543         nqueue = vmxnet3_tunable_int(sc, "rxnqueue", vmxnet3_default_rxnqueue);
 544         if (nqueue > VMXNET3_MAX_RX_QUEUES || nqueue < 1)
 545                 nqueue = VMXNET3_DEF_RX_QUEUES;
 546         if (nqueue > ncpus)
 547                 nqueue = ncpus;
 548         sc->vmx_max_nrxqueues = trunc_powerof2(nqueue);
 549
 550         if (vmxnet3_tunable_int(sc, "mq_disable", vmxnet3_mq_disable)) {
 551                 sc->vmx_max_nrxqueues = 1;
 552                 sc->vmx_max_ntxqueues = 1;
 553         }
 554
 555         ndesc = vmxnet3_tunable_int(sc, "txd", vmxnet3_default_txndesc);
 556         if (ndesc > VMXNET3_MAX_TX_NDESC || ndesc < VMXNET3_MIN_TX_NDESC)
 557                 ndesc = VMXNET3_DEF_TX_NDESC;
 558         if (ndesc & VMXNET3_MASK_TX_NDESC)
 559                 ndesc &= ~VMXNET3_MASK_TX_NDESC;
 560         sc->vmx_ntxdescs = ndesc;
 561
 562         ndesc = vmxnet3_tunable_int(sc, "rxd", vmxnet3_default_rxndesc);
 563         if (ndesc > VMXNET3_MAX_RX_NDESC || ndesc < VMXNET3_MIN_RX_NDESC)
 564                 ndesc = VMXNET3_DEF_RX_NDESC;
 565         if (ndesc & VMXNET3_MASK_RX_NDESC)
 566                 ndesc &= ~VMXNET3_MASK_RX_NDESC;
 567         sc->vmx_nrxdescs = ndesc;
 568         sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS;
 569 }
 570
 571 static void
 572 vmxnet3_check_multiqueue(struct vmxnet3_softc *sc)
 573 {
 574
 575         if (sc->vmx_intr_type != VMXNET3_IT_MSIX)
 576                 goto out;
 577
 578         /* BMV: Just use the maximum configured for now. */
 579         sc->vmx_nrxqueues = sc->vmx_max_nrxqueues;
 580         sc->vmx_ntxqueues = sc->vmx_max_ntxqueues;
 581
 582         if (sc->vmx_nrxqueues > 1)
 583                 sc->vmx_flags |= VMXNET3_FLAG_RSS;
 584
 585         return;
 586
 587 out:
 588         sc->vmx_ntxqueues = 1;
 589         sc->vmx_nrxqueues = 1;
 590 }
 591
 592 #ifdef __FreeBSD__
 593 static int
 594 vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc)
 595 {
 596         device_t dev;
 597         int nmsix, cnt, required;
 598
 599         dev = sc->vmx_dev;
 600
 601         if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX)
 602                 return (1);
 603
 604         /* Allocate an additional vector for the events interrupt. */
 605         required = sc->vmx_max_nrxqueues + sc->vmx_max_ntxqueues + 1;
 606
 607         nmsix = pci_msix_count(dev);
 608         if (nmsix < required)
 609                 return (1);
 610
 611         cnt = required;
 612         if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) {
 613                 sc->vmx_nintrs = required;
 614                 return (0);
 615         } else
 616                 pci_release_msi(dev);
 617
 618         /* BMV TODO Fallback to sharing MSIX vectors if possible. */
 619
 620         return (1);
 621 }
 622
 623 static int
 624 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc)
 625 {
 626         device_t dev;
 627         int nmsi, cnt, required;
 628
 629         dev = sc->vmx_dev;
 630         required = 1;
 631
 632         nmsi = pci_msi_count(dev);
 633         if (nmsi < required)
 634                 return (1);
 635
 636         cnt = required;
 637         if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) {
 638                 sc->vmx_nintrs = 1;
 639                 return (0);
 640         } else
 641                 pci_release_msi(dev);
 642
 643         return (1);
 644 }
 645 #else
 646 static int
 647 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc)
 648 {
 649         int irq_flags, rid;
 650         int enable = 1;
 651
 652         sc->vmx_irq_type = pci_alloc_1intr(sc->vmx_dev, enable, &rid,
 653             &irq_flags);
 654         sc->vmx_irq_flags = irq_flags;
 655         sc->vmx_nintrs = 1;
 656         return (0);
 657 }
 658 #endif
 659
 660 static int
 661 vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc)
 662 {
 663
 664         sc->vmx_nintrs = 1;
 665         return (0);
 666 }
 667
 668 static int
 669 vmxnet3_alloc_interrupt(struct vmxnet3_softc *sc, int rid, int flags,
 670     struct vmxnet3_interrupt *intr)
 671 {
 672         struct resource *irq;
 673
 674         irq = bus_alloc_resource_any(sc->vmx_dev, SYS_RES_IRQ, &rid,
 675             sc->vmx_irq_flags);
 676         if (irq == NULL)
 677                 return (ENXIO);
 678
 679         intr->vmxi_irq = irq;
 680         intr->vmxi_rid = rid;
 681
 682         return (0);
 683 }
 684
 685 static int
 686 vmxnet3_alloc_intr_resources(struct vmxnet3_softc *sc)
 687 {
 688         int i, rid, flags, error;
 689
 690         rid = 0;
 691         flags = RF_ACTIVE;
 692
 693         if (sc->vmx_intr_type == VMXNET3_IT_LEGACY)
 694                 flags |= RF_SHAREABLE;
 695         else
 696                 rid = 1;
 697
 698         for (i = 0; i < sc->vmx_nintrs; i++, rid++) {
 699                 error = vmxnet3_alloc_interrupt(sc, rid, flags,
 700                     &sc->vmx_intrs[i]);
 701                 if (error)
 702                         return (error);
 703         }
 704
 705         return (0);
 706 }
 707
 708 #ifdef __FreeBSD__
 709 static int
 710 vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc)
 711 {
 712         device_t dev;
 713         struct vmxnet3_txqueue *txq;
 714         struct vmxnet3_rxqueue *rxq;
 715         struct vmxnet3_interrupt *intr;
 716         int i, error;
 717
 718         dev = sc->vmx_dev;
 719         intr = &sc->vmx_intrs[0];
 720
 721         for (i = 0; i < sc->vmx_ntxqueues; i++, intr++) {
 722                 txq = &sc->vmx_txq[i];
 723                 error = bus_setup_intr(dev, intr->vmxi_irq, INTR_MPSAFE,
 724                      vmxnet3_txq_intr, txq, &intr->vmxi_handler, NULL);
 725                 if (error)
 726                         return (error);
 727                 bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler,
 728                     "tq%d", i);
 729                 txq->vxtxq_intr_idx = intr->vmxi_rid - 1;
 730         }
 731
 732         for (i = 0; i < sc->vmx_nrxqueues; i++, intr++) {
 733                 rxq = &sc->vmx_rxq[i];
 734                 error = bus_setup_intr(dev, intr->vmxi_irq, INTR_MPSAFE,
 735                     vmxnet3_rxq_intr, rxq, &intr->vmxi_handler, NULL);
 736                 if (error)
 737                         return (error);
 738                 bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler,
 739                     "rq%d", i);
 740                 rxq->vxrxq_intr_idx = intr->vmxi_rid - 1;
 741         }
 742
 743         error = bus_setup_intr(dev, intr->vmxi_irq, INTR_MPSAFE,
 744             vmxnet3_event_intr, sc, &intr->vmxi_handler, NULL);
 745         if (error)
 746                 return (error);
 747         bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, "event");
 748         sc->vmx_event_intr_idx = intr->vmxi_rid - 1;
 749
 750         return (0);
 751 }
 752 #endif
 753
 754 static int
 755 vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc)
 756 {
 757         struct vmxnet3_interrupt *intr;
 758         int i, error;
 759
 760         intr = &sc->vmx_intrs[0];
 761         error = bus_setup_intr(sc->vmx_dev, intr->vmxi_irq,
 762             INTR_MPSAFE, vmxnet3_legacy_intr, sc,
 763             &intr->vmxi_handler, NULL);
 764
 765         for (i = 0; i < sc->vmx_ntxqueues; i++)
 766                 sc->vmx_txq[i].vxtxq_intr_idx = 0;
 767         for (i = 0; i < sc->vmx_nrxqueues; i++)
 768                 sc->vmx_rxq[i].vxrxq_intr_idx = 0;
 769         sc->vmx_event_intr_idx = 0;
 770
 771         return (error);
 772 }
 773
 774 static void
 775 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc)
 776 {
 777         struct vmxnet3_txqueue *txq;
 778         struct vmxnet3_txq_shared *txs;
 779         struct vmxnet3_rxqueue *rxq;
 780         struct vmxnet3_rxq_shared *rxs;
 781         int i;
 782
 783         sc->vmx_ds->evintr = sc->vmx_event_intr_idx;
 784
 785         for (i = 0; i < sc->vmx_ntxqueues; i++) {
 786                 txq = &sc->vmx_txq[i];
 787                 txs = txq->vxtxq_ts;
 788                 txs->intr_idx = txq->vxtxq_intr_idx;
 789         }
 790
 791         for (i = 0; i < sc->vmx_nrxqueues; i++) {
 792                 rxq = &sc->vmx_rxq[i];
 793                 rxs = rxq->vxrxq_rs;
 794                 rxs->intr_idx = rxq->vxrxq_intr_idx;
 795         }
 796 }
 797
 798 static int
 799 vmxnet3_setup_interrupts(struct vmxnet3_softc *sc)
 800 {
 801         int error;
 802
 803         error = vmxnet3_alloc_intr_resources(sc);
 804         if (error)
 805                 return (error);
 806
 807         switch (sc->vmx_intr_type) {
 808         case VMXNET3_IT_MSIX:
 809 #ifdef __FreeBSD__
 810                 error = vmxnet3_setup_msix_interrupts(sc);
 811 #else
 812                 device_printf(sc->vmx_dev, "VMXNET3_IT_MSIX unsupported\n");
 813                 error = ENXIO;
 814 #endif
 815                 break;
 816         case VMXNET3_IT_MSI:
 817         case VMXNET3_IT_LEGACY:
 818                 error = vmxnet3_setup_legacy_interrupt(sc);
 819                 break;
 820         default:
 821                 panic("%s: invalid interrupt type %d", __func__,
 822                     sc->vmx_intr_type);
 823         }
 824
 825         if (error == 0)
 826                 vmxnet3_set_interrupt_idx(sc);
 827
 828         return (error);
 829 }
 830
 831 #ifdef __FreeBSD__
 832 static int
 833 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc)
 834 {
 835         device_t dev;
 836         uint32_t config;
 837         int error;
 838
 839         dev = sc->vmx_dev;
 840         config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG);
 841
 842         sc->vmx_intr_type = config & 0x03;
 843         sc->vmx_intr_mask_mode = (config >> 2) & 0x03;
 844
 845         switch (sc->vmx_intr_type) {
 846         case VMXNET3_IT_AUTO:
 847                 sc->vmx_intr_type = VMXNET3_IT_MSIX;
 848                 /* FALLTHROUGH */
 849         case VMXNET3_IT_MSIX:
 850                 error = vmxnet3_alloc_msix_interrupts(sc);
 851                 if (error == 0)
 852                         break;
 853                 sc->vmx_intr_type = VMXNET3_IT_MSI;
 854                 /* FALLTHROUGH */
 855         case VMXNET3_IT_MSI:
 856                 error = vmxnet3_alloc_msi_interrupts(sc);
 857                 if (error == 0)
 858                         break;
 859                 sc->vmx_intr_type = VMXNET3_IT_LEGACY;
 860                 /* FALLTHROUGH */
 861         case VMXNET3_IT_LEGACY:
 862                 error = vmxnet3_alloc_legacy_interrupts(sc);
 863                 if (error == 0)
 864                         break;
 865                 /* FALLTHROUGH */
 866         default:
 867                 sc->vmx_intr_type = -1;
 868                 device_printf(dev, "cannot allocate any interrupt resources\n");
 869                 return (ENXIO);
 870         }
 871
 872         return (error);
 873 }
 874 #else
 875 static int
 876 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc)
 877 {
 878         device_t dev;
 879         uint32_t config;
 880         int error;
 881
 882         dev = sc->vmx_dev;
 883         config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG);
 884
 885         sc->vmx_intr_type = config & 0x03;
 886         sc->vmx_intr_mask_mode = (config >> 2) & 0x03;
 887
 888         switch (sc->vmx_intr_type) {
 889         case VMXNET3_IT_AUTO:
 890                 sc->vmx_intr_type = VMXNET3_IT_MSI;
 891                 /* FALLTHROUGH */
 892         case VMXNET3_IT_MSI:
 893                 error = vmxnet3_alloc_msi_interrupts(sc);
 894                 if (error == 0)
 895                         break;
 896                 sc->vmx_intr_type = VMXNET3_IT_LEGACY;
 897         case VMXNET3_IT_LEGACY:
 898                 error = vmxnet3_alloc_legacy_interrupts(sc);
 899                 if (error == 0)
 900                         break;
 901                 /* FALLTHROUGH */
 902         case VMXNET3_IT_MSIX:
 903                 /* FALLTHROUGH */
 904         default:
 905                 sc->vmx_intr_type = -1;
 906                 device_printf(dev, "cannot allocate any interrupt resources\n");
 907                 return (ENXIO);
 908         }
 909
 910         return (error);
 911 }
 912 #endif
 913
 914 static void
 915 vmxnet3_free_interrupt(struct vmxnet3_softc *sc,
 916     struct vmxnet3_interrupt *intr)
 917 {
 918         device_t dev;
 919
 920         dev = sc->vmx_dev;
 921
 922         if (intr->vmxi_handler != NULL) {
 923                 bus_teardown_intr(dev, intr->vmxi_irq, intr->vmxi_handler);
 924                 intr->vmxi_handler = NULL;
 925         }
 926
 927         if (intr->vmxi_irq != NULL) {
 928                 bus_release_resource(dev, SYS_RES_IRQ, intr->vmxi_rid,
 929                     intr->vmxi_irq);
 930                 intr->vmxi_irq = NULL;
 931                 intr->vmxi_rid = -1;
 932         }
 933 }
 934
 935 #ifdef __FreeBSD__
 936 static void
 937 vmxnet3_free_interrupts(struct vmxnet3_softc *sc)
 938 {
 939         int i;
 940
 941         for (i = 0; i < sc->vmx_nintrs; i++)
 942                 vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]);
 943
 944         if (sc->vmx_intr_type == VMXNET3_IT_MSI ||
 945             sc->vmx_intr_type == VMXNET3_IT_MSIX)
 946                 pci_release_msi(sc->vmx_dev);
 947 }
 948 #else
 949 static void
 950 vmxnet3_free_interrupts(struct vmxnet3_softc *sc)
 951 {
 952         int i;
 953
 954         for (i = 0; i < sc->vmx_nintrs; i++)
 955                 vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]);
 956
 957         if (sc->vmx_irq_type == PCI_INTR_TYPE_MSI)
 958                 pci_release_msi(sc->vmx_dev);
 959 }
 960 #endif
 961
 962 #ifndef VMXNET3_LEGACY_TX
 963 static int
 964 vmxnet3_alloc_taskqueue(struct vmxnet3_softc *sc)
 965 {
 966         device_t dev;
 967
 968         dev = sc->vmx_dev;
 969
 970         sc->vmx_tq = taskqueue_create(device_get_nameunit(dev), M_NOWAIT,
 971             taskqueue_thread_enqueue, &sc->vmx_tq);
 972         if (sc->vmx_tq == NULL)
 973                 return (ENOMEM);
 974
 975         return (0);
 976 }
 977
 978 static void
 979 vmxnet3_start_taskqueue(struct vmxnet3_softc *sc)
 980 {
 981         device_t dev;
 982         int nthreads, error;
 983
 984         dev = sc->vmx_dev;
 985
 986         /*
 987          * The taskqueue is typically not frequently used, so a dedicated
 988          * thread for each queue is unnecessary.
 989          */
 990         nthreads = MAX(1, sc->vmx_ntxqueues / 2);
 991
 992         /*
 993          * Most drivers just ignore the return value - it only fails
 994          * with ENOMEM so an error is not likely. It is hard for us
 995          * to recover from an error here.
 996          */
 997         error = taskqueue_start_threads(&sc->vmx_tq, nthreads, PI_NET,
 998             "%s taskq", device_get_nameunit(dev));
 999         if (error)
1000                 device_printf(dev, "failed to start taskqueue: %d", error);
1001 }
1002
1003 static void
1004 vmxnet3_drain_taskqueue(struct vmxnet3_softc *sc)
1005 {
1006         struct vmxnet3_txqueue *txq;
1007         int i;
1008
1009         if (sc->vmx_tq != NULL) {
1010                 for (i = 0; i < sc->vmx_max_ntxqueues; i++) {
1011                         txq = &sc->vmx_txq[i];
1012                         taskqueue_drain(sc->vmx_tq, &txq->vxtxq_defrtask);
1013                 }
1014         }
1015 }
1016
1017 static void
1018 vmxnet3_free_taskqueue(struct vmxnet3_softc *sc)
1019 {
1020         if (sc->vmx_tq != NULL) {
1021                 taskqueue_free(sc->vmx_tq);
1022                 sc->vmx_tq = NULL;
1023         }
1024 }
1025 #endif
1026
1027 static int
1028 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q)
1029 {
1030         struct vmxnet3_rxqueue *rxq;
1031         struct vmxnet3_rxring *rxr;
1032         int i;
1033
1034         rxq = &sc->vmx_rxq[q];
1035
1036         ksnprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d",
1037             device_get_nameunit(sc->vmx_dev), q);
1038         lockinit(&rxq->vxrxq_lock, rxq->vxrxq_name, 0, 0);
1039
1040         rxq->vxrxq_sc = sc;
1041         rxq->vxrxq_id = q;
1042
1043         for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1044                 rxr = &rxq->vxrxq_cmd_ring[i];
1045                 rxr->vxrxr_rid = i;
1046                 rxr->vxrxr_ndesc = sc->vmx_nrxdescs;
1047                 rxr->vxrxr_rxbuf = kmalloc(rxr->vxrxr_ndesc *
1048                     sizeof(struct vmxnet3_rxbuf), M_DEVBUF, M_INTWAIT | M_ZERO);
1049                 if (rxr->vxrxr_rxbuf == NULL)
1050                         return (ENOMEM);
1051
1052                 rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs;
1053         }
1054
1055         return (0);
1056 }
1057
1058 static int
1059 vmxnet3_init_txq(struct vmxnet3_softc *sc, int q)
1060 {
1061         struct vmxnet3_txqueue *txq;
1062         struct vmxnet3_txring *txr;
1063
1064         txq = &sc->vmx_txq[q];
1065         txr = &txq->vxtxq_cmd_ring;
1066
1067         ksnprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d",
1068             device_get_nameunit(sc->vmx_dev), q);
1069         lockinit(&txq->vxtxq_lock, txq->vxtxq_name, 0, 0);
1070
1071         txq->vxtxq_sc = sc;
1072         txq->vxtxq_id = q;
1073
1074         txr->vxtxr_ndesc = sc->vmx_ntxdescs;
1075         txr->vxtxr_txbuf = kmalloc(txr->vxtxr_ndesc *
1076             sizeof(struct vmxnet3_txbuf), M_DEVBUF, M_INTWAIT | M_ZERO);
1077         if (txr->vxtxr_txbuf == NULL)
1078                 return (ENOMEM);
1079
1080         txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs;
1081
1082 #ifndef VMXNET3_LEGACY_TX
1083         TASK_INIT(&txq->vxtxq_defrtask, 0, vmxnet3_txq_tq_deferred, txq);
1084
1085         txq->vxtxq_br = buf_ring_alloc(VMXNET3_DEF_BUFRING_SIZE, M_DEVBUF,
1086             M_NOWAIT, &txq->vxtxq_lock);
1087         if (txq->vxtxq_br == NULL)
1088                 return (ENOMEM);
1089 #endif
1090
1091         return (0);
1092 }
1093
1094 static int
1095 vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc)
1096 {
1097         int i, error;
1098
1099         /*
1100          * Only attempt to create multiple queues if MSIX is available. MSIX is
1101          * disabled by default because its apparently broken for devices passed
1102          * through by at least ESXi 5.1. The hw.pci.honor_msi_blacklist tunable
1103          * must be set to zero for MSIX. This check prevents us from allocating
1104          * queue structures that we will not use.
1105          */
1106         if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) {
1107                 sc->vmx_max_nrxqueues = 1;
1108                 sc->vmx_max_ntxqueues = 1;
1109         }
1110
1111         sc->vmx_rxq = kmalloc(sizeof(struct vmxnet3_rxqueue) *
1112             sc->vmx_max_nrxqueues, M_DEVBUF, M_INTWAIT | M_ZERO);
1113         sc->vmx_txq = kmalloc(sizeof(struct vmxnet3_txqueue) *
1114             sc->vmx_max_ntxqueues, M_DEVBUF, M_INTWAIT | M_ZERO);
1115         if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL)
1116                 return (ENOMEM);
1117
1118         for (i = 0; i < sc->vmx_max_nrxqueues; i++) {
1119                 error = vmxnet3_init_rxq(sc, i);
1120                 if (error)
1121                         return (error);
1122         }
1123
1124         for (i = 0; i < sc->vmx_max_ntxqueues; i++) {
1125                 error = vmxnet3_init_txq(sc, i);
1126                 if (error)
1127                         return (error);
1128         }
1129
1130         return (0);
1131 }
1132
1133 static void
1134 vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq)
1135 {
1136         struct vmxnet3_rxring *rxr;
1137         int i;
1138
1139         rxq->vxrxq_sc = NULL;
1140         rxq->vxrxq_id = -1;
1141
1142         for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1143                 rxr = &rxq->vxrxq_cmd_ring[i];
1144
1145                 if (rxr->vxrxr_rxbuf != NULL) {
1146                         kfree(rxr->vxrxr_rxbuf, M_DEVBUF);
1147                         rxr->vxrxr_rxbuf = NULL;
1148                 }
1149         }
1150
1151 #if 0 /* XXX */
1152         if (mtx_initialized(&rxq->vxrxq_lock) != 0)
1153 #endif
1154                 lockuninit(&rxq->vxrxq_lock);
1155 }
1156
1157 static void
1158 vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq)
1159 {
1160         struct vmxnet3_txring *txr;
1161
1162         txr = &txq->vxtxq_cmd_ring;
1163
1164         txq->vxtxq_sc = NULL;
1165         txq->vxtxq_id = -1;
1166
1167 #ifndef VMXNET3_LEGACY_TX
1168         if (txq->vxtxq_br != NULL) {
1169                 buf_ring_free(txq->vxtxq_br, M_DEVBUF);
1170                 txq->vxtxq_br = NULL;
1171         }
1172 #endif
1173
1174         if (txr->vxtxr_txbuf != NULL) {
1175                 kfree(txr->vxtxr_txbuf, M_DEVBUF);
1176                 txr->vxtxr_txbuf = NULL;
1177         }
1178
1179 #if 0 /* XXX */
1180         if (mtx_initialized(&txq->vxtxq_lock) != 0)
1181 #endif
1182                 lockuninit(&txq->vxtxq_lock);
1183 }
1184
1185 static void
1186 vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc)
1187 {
1188         int i;
1189
1190         if (sc->vmx_rxq != NULL) {
1191                 for (i = 0; i < sc->vmx_max_nrxqueues; i++)
1192                         vmxnet3_destroy_rxq(&sc->vmx_rxq[i]);
1193                 kfree(sc->vmx_rxq, M_DEVBUF);
1194                 sc->vmx_rxq = NULL;
1195         }
1196
1197         if (sc->vmx_txq != NULL) {
1198                 for (i = 0; i < sc->vmx_max_ntxqueues; i++)
1199                         vmxnet3_destroy_txq(&sc->vmx_txq[i]);
1200                 kfree(sc->vmx_txq, M_DEVBUF);
1201                 sc->vmx_txq = NULL;
1202         }
1203 }
1204
1205 static int
1206 vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc)
1207 {
1208         device_t dev;
1209         uint8_t *kva;
1210         size_t size;
1211         int i, error;
1212
1213         dev = sc->vmx_dev;
1214
1215         size = sizeof(struct vmxnet3_driver_shared);
1216         error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma);
1217         if (error) {
1218                 device_printf(dev, "cannot alloc shared memory\n");
1219                 return (error);
1220         }
1221         sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr;
1222
1223         size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) +
1224             sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared);
1225         error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma);
1226         if (error) {
1227                 device_printf(dev, "cannot alloc queue shared memory\n");
1228                 return (error);
1229         }
1230         sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr;
1231         kva = sc->vmx_qs;
1232
1233         for (i = 0; i < sc->vmx_ntxqueues; i++) {
1234                 sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva;
1235                 kva += sizeof(struct vmxnet3_txq_shared);
1236         }
1237         for (i = 0; i < sc->vmx_nrxqueues; i++) {
1238                 sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva;
1239                 kva += sizeof(struct vmxnet3_rxq_shared);
1240         }
1241
1242         if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1243                 size = sizeof(struct vmxnet3_rss_shared);
1244                 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma);
1245                 if (error) {
1246                         device_printf(dev, "cannot alloc rss shared memory\n");
1247                         return (error);
1248                 }
1249                 sc->vmx_rss =
1250                     (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr;
1251         }
1252
1253         return (0);
1254 }
1255
1256 static void
1257 vmxnet3_free_shared_data(struct vmxnet3_softc *sc)
1258 {
1259
1260         if (sc->vmx_rss != NULL) {
1261                 vmxnet3_dma_free(sc, &sc->vmx_rss_dma);
1262                 sc->vmx_rss = NULL;
1263         }
1264
1265         if (sc->vmx_qs != NULL) {
1266                 vmxnet3_dma_free(sc, &sc->vmx_qs_dma);
1267                 sc->vmx_qs = NULL;
1268         }
1269
1270         if (sc->vmx_ds != NULL) {
1271                 vmxnet3_dma_free(sc, &sc->vmx_ds_dma);
1272                 sc->vmx_ds = NULL;
1273         }
1274 }
1275
1276 static int
1277 vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc)
1278 {
1279         device_t dev;
1280         struct vmxnet3_txqueue *txq;
1281         struct vmxnet3_txring *txr;
1282         struct vmxnet3_comp_ring *txc;
1283         size_t descsz, compsz;
1284         int i, q, error;
1285
1286         dev = sc->vmx_dev;
1287
1288         for (q = 0; q < sc->vmx_ntxqueues; q++) {
1289                 txq = &sc->vmx_txq[q];
1290                 txr = &txq->vxtxq_cmd_ring;
1291                 txc = &txq->vxtxq_comp_ring;
1292
1293                 descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc);
1294                 compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc);
1295
1296                 error = bus_dma_tag_create(bus_get_dma_tag(dev),
1297                     1, 0,                       /* alignment, boundary */
1298                     BUS_SPACE_MAXADDR,          /* lowaddr */
1299                     BUS_SPACE_MAXADDR,          /* highaddr */
1300                     NULL, NULL,                 /* filter, filterarg */
1301                     VMXNET3_TX_MAXSIZE,         /* maxsize */
1302                     VMXNET3_TX_MAXSEGS,         /* nsegments */
1303                     VMXNET3_TX_MAXSEGSIZE,      /* maxsegsize */
1304                     0,                          /* flags */
1305                     &txr->vxtxr_txtag);
1306                 if (error) {
1307                         device_printf(dev,
1308                             "unable to create Tx buffer tag for queue %d\n", q);
1309                         return (error);
1310                 }
1311
1312                 error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma);
1313                 if (error) {
1314                         device_printf(dev, "cannot alloc Tx descriptors for "
1315                             "queue %d error %d\n", q, error);
1316                         return (error);
1317                 }
1318                 txr->vxtxr_txd =
1319                     (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr;
1320
1321                 error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma);
1322                 if (error) {
1323                         device_printf(dev, "cannot alloc Tx comp descriptors "
1324                            "for queue %d error %d\n", q, error);
1325                         return (error);
1326                 }
1327                 txc->vxcr_u.txcd =
1328                     (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr;
1329
1330                 for (i = 0; i < txr->vxtxr_ndesc; i++) {
1331                         error = bus_dmamap_create(txr->vxtxr_txtag, 0,
1332                             &txr->vxtxr_txbuf[i].vtxb_dmamap);
1333                         if (error) {
1334                                 device_printf(dev, "unable to create Tx buf "
1335                                     "dmamap for queue %d idx %d\n", q, i);
1336                                 return (error);
1337                         }
1338                 }
1339         }
1340
1341         return (0);
1342 }
1343
1344 static void
1345 vmxnet3_free_txq_data(struct vmxnet3_softc *sc)
1346 {
1347         device_t dev;
1348         struct vmxnet3_txqueue *txq;
1349         struct vmxnet3_txring *txr;
1350         struct vmxnet3_comp_ring *txc;
1351         struct vmxnet3_txbuf *txb;
1352         int i, q;
1353
1354         dev = sc->vmx_dev;
1355
1356         for (q = 0; q < sc->vmx_ntxqueues; q++) {
1357                 txq = &sc->vmx_txq[q];
1358                 txr = &txq->vxtxq_cmd_ring;
1359                 txc = &txq->vxtxq_comp_ring;
1360
1361                 for (i = 0; i < txr->vxtxr_ndesc; i++) {
1362                         txb = &txr->vxtxr_txbuf[i];
1363                         if (txb->vtxb_dmamap != NULL) {
1364                                 bus_dmamap_destroy(txr->vxtxr_txtag,
1365                                     txb->vtxb_dmamap);
1366                                 txb->vtxb_dmamap = NULL;
1367                         }
1368                 }
1369
1370                 if (txc->vxcr_u.txcd != NULL) {
1371                         vmxnet3_dma_free(sc, &txc->vxcr_dma);
1372                         txc->vxcr_u.txcd = NULL;
1373                 }
1374
1375                 if (txr->vxtxr_txd != NULL) {
1376                         vmxnet3_dma_free(sc, &txr->vxtxr_dma);
1377                         txr->vxtxr_txd = NULL;
1378                 }
1379
1380                 if (txr->vxtxr_txtag != NULL) {
1381                         bus_dma_tag_destroy(txr->vxtxr_txtag);
1382                         txr->vxtxr_txtag = NULL;
1383                 }
1384         }
1385 }
1386
1387 static int
1388 vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc)
1389 {
1390         device_t dev;
1391         struct vmxnet3_rxqueue *rxq;
1392         struct vmxnet3_rxring *rxr;
1393         struct vmxnet3_comp_ring *rxc;
1394         int descsz, compsz;
1395         int i, j, q, error;
1396
1397         dev = sc->vmx_dev;
1398
1399         for (q = 0; q < sc->vmx_nrxqueues; q++) {
1400                 rxq = &sc->vmx_rxq[q];
1401                 rxc = &rxq->vxrxq_comp_ring;
1402                 compsz = 0;
1403
1404                 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1405                         rxr = &rxq->vxrxq_cmd_ring[i];
1406
1407                         descsz = rxr->vxrxr_ndesc *
1408                             sizeof(struct vmxnet3_rxdesc);
1409                         compsz += rxr->vxrxr_ndesc *
1410                             sizeof(struct vmxnet3_rxcompdesc);
1411
1412                         error = bus_dma_tag_create(bus_get_dma_tag(dev),
1413                             1, 0,               /* alignment, boundary */
1414                             BUS_SPACE_MAXADDR,  /* lowaddr */
1415                             BUS_SPACE_MAXADDR,  /* highaddr */
1416                             NULL, NULL,         /* filter, filterarg */
1417                             MJUMPAGESIZE,       /* maxsize */
1418                             1,                  /* nsegments */
1419                             MJUMPAGESIZE,       /* maxsegsize */
1420                             0,                  /* flags */
1421                             &rxr->vxrxr_rxtag);
1422                         if (error) {
1423                                 device_printf(dev,
1424                                     "unable to create Rx buffer tag for "
1425                                     "queue %d\n", q);
1426                                 return (error);
1427                         }
1428
1429                         error = vmxnet3_dma_malloc(sc, descsz, 512,
1430                             &rxr->vxrxr_dma);
1431                         if (error) {
1432                                 device_printf(dev, "cannot allocate Rx "
1433                                     "descriptors for queue %d/%d error %d\n",
1434                                     i, q, error);
1435                                 return (error);
1436                         }
1437                         rxr->vxrxr_rxd =
1438                             (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr;
1439                 }
1440
1441                 error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma);
1442                 if (error) {
1443                         device_printf(dev, "cannot alloc Rx comp descriptors "
1444                             "for queue %d error %d\n", q, error);
1445                         return (error);
1446                 }
1447                 rxc->vxcr_u.rxcd =
1448                     (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr;
1449
1450                 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1451                         rxr = &rxq->vxrxq_cmd_ring[i];
1452
1453                         error = bus_dmamap_create(rxr->vxrxr_rxtag, 0,
1454                             &rxr->vxrxr_spare_dmap);
1455                         if (error) {
1456                                 device_printf(dev, "unable to create spare "
1457                                     "dmamap for queue %d/%d error %d\n",
1458                                     q, i, error);
1459                                 return (error);
1460                         }
1461
1462                         for (j = 0; j < rxr->vxrxr_ndesc; j++) {
1463                                 error = bus_dmamap_create(rxr->vxrxr_rxtag, 0,
1464                                     &rxr->vxrxr_rxbuf[j].vrxb_dmamap);
1465                                 if (error) {
1466                                         device_printf(dev, "unable to create "
1467                                             "dmamap for queue %d/%d slot %d "
1468                                             "error %d\n",
1469                                             q, i, j, error);
1470                                         return (error);
1471                                 }
1472                         }
1473                 }
1474         }
1475
1476         return (0);
1477 }
1478
1479 static void
1480 vmxnet3_free_rxq_data(struct vmxnet3_softc *sc)
1481 {
1482         device_t dev;
1483         struct vmxnet3_rxqueue *rxq;
1484         struct vmxnet3_rxring *rxr;
1485         struct vmxnet3_comp_ring *rxc;
1486         struct vmxnet3_rxbuf *rxb;
1487         int i, j, q;
1488
1489         dev = sc->vmx_dev;
1490
1491         for (q = 0; q < sc->vmx_nrxqueues; q++) {
1492                 rxq = &sc->vmx_rxq[q];
1493                 rxc = &rxq->vxrxq_comp_ring;
1494
1495                 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1496                         rxr = &rxq->vxrxq_cmd_ring[i];
1497
1498                         if (rxr->vxrxr_spare_dmap != NULL) {
1499                                 bus_dmamap_destroy(rxr->vxrxr_rxtag,
1500                                     rxr->vxrxr_spare_dmap);
1501                                 rxr->vxrxr_spare_dmap = NULL;
1502                         }
1503
1504                         for (j = 0; j < rxr->vxrxr_ndesc; j++) {
1505                                 rxb = &rxr->vxrxr_rxbuf[j];
1506                                 if (rxb->vrxb_dmamap != NULL) {
1507                                         bus_dmamap_destroy(rxr->vxrxr_rxtag,
1508                                             rxb->vrxb_dmamap);
1509                                         rxb->vrxb_dmamap = NULL;
1510                                 }
1511                         }
1512                 }
1513
1514                 if (rxc->vxcr_u.rxcd != NULL) {
1515                         vmxnet3_dma_free(sc, &rxc->vxcr_dma);
1516                         rxc->vxcr_u.rxcd = NULL;
1517                 }
1518
1519                 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1520                         rxr = &rxq->vxrxq_cmd_ring[i];
1521
1522                         if (rxr->vxrxr_rxd != NULL) {
1523                                 vmxnet3_dma_free(sc, &rxr->vxrxr_dma);
1524                                 rxr->vxrxr_rxd = NULL;
1525                         }
1526
1527                         if (rxr->vxrxr_rxtag != NULL) {
1528                                 bus_dma_tag_destroy(rxr->vxrxr_rxtag);
1529                                 rxr->vxrxr_rxtag = NULL;
1530                         }
1531                 }
1532         }
1533 }
1534
1535 static int
1536 vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc)
1537 {
1538         int error;
1539
1540         error = vmxnet3_alloc_txq_data(sc);
1541         if (error)
1542                 return (error);
1543
1544         error = vmxnet3_alloc_rxq_data(sc);
1545         if (error)
1546                 return (error);
1547
1548         return (0);
1549 }
1550
1551 static void
1552 vmxnet3_free_queue_data(struct vmxnet3_softc *sc)
1553 {
1554
1555         if (sc->vmx_rxq != NULL)
1556                 vmxnet3_free_rxq_data(sc);
1557
1558         if (sc->vmx_txq != NULL)
1559                 vmxnet3_free_txq_data(sc);
1560 }
1561
1562 static int
1563 vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc)
1564 {
1565         int error;
1566
1567         error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN,
1568             32, &sc->vmx_mcast_dma);
1569         if (error)
1570                 device_printf(sc->vmx_dev, "unable to alloc multicast table\n");
1571         else
1572                 sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr;
1573
1574         return (error);
1575 }
1576
1577 static void
1578 vmxnet3_free_mcast_table(struct vmxnet3_softc *sc)
1579 {
1580
1581         if (sc->vmx_mcast != NULL) {
1582                 vmxnet3_dma_free(sc, &sc->vmx_mcast_dma);
1583                 sc->vmx_mcast = NULL;
1584         }
1585 }
1586
1587 static void
1588 vmxnet3_init_shared_data(struct vmxnet3_softc *sc)
1589 {
1590         struct vmxnet3_driver_shared *ds;
1591         struct vmxnet3_txqueue *txq;
1592         struct vmxnet3_txq_shared *txs;
1593         struct vmxnet3_rxqueue *rxq;
1594         struct vmxnet3_rxq_shared *rxs;
1595         int i;
1596
1597         ds = sc->vmx_ds;
1598
1599         /*
1600          * Initialize fields of the shared data that remains the same across
1601          * reinits. Note the shared data is zero'd when allocated.
1602          */
1603
1604         ds->magic = VMXNET3_REV1_MAGIC;
1605
1606         /* DriverInfo */
1607         ds->version = VMXNET3_DRIVER_VERSION;
1608         ds->guest = VMXNET3_GOS_FREEBSD |
1609 #ifdef __LP64__
1610             VMXNET3_GOS_64BIT;
1611 #else
1612             VMXNET3_GOS_32BIT;
1613 #endif
1614         ds->vmxnet3_revision = 1;
1615         ds->upt_version = 1;
1616
1617         /* Misc. conf */
1618         ds->driver_data = vtophys(sc);
1619         ds->driver_data_len = sizeof(struct vmxnet3_softc);
1620         ds->queue_shared = sc->vmx_qs_dma.dma_paddr;
1621         ds->queue_shared_len = sc->vmx_qs_dma.dma_size;
1622         ds->nrxsg_max = sc->vmx_max_rxsegs;
1623
1624         /* RSS conf */
1625         if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1626                 ds->rss.version = 1;
1627                 ds->rss.paddr = sc->vmx_rss_dma.dma_paddr;
1628                 ds->rss.len = sc->vmx_rss_dma.dma_size;
1629         }
1630
1631         /* Interrupt control. */
1632         ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO;
1633         ds->nintr = sc->vmx_nintrs;
1634         ds->evintr = sc->vmx_event_intr_idx;
1635         ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL;
1636
1637         for (i = 0; i < sc->vmx_nintrs; i++)
1638                 ds->modlevel[i] = UPT1_IMOD_ADAPTIVE;
1639
1640         /* Receive filter. */
1641         ds->mcast_table = sc->vmx_mcast_dma.dma_paddr;
1642         ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size;
1643
1644         /* Tx queues */
1645         for (i = 0; i < sc->vmx_ntxqueues; i++) {
1646                 txq = &sc->vmx_txq[i];
1647                 txs = txq->vxtxq_ts;
1648
1649                 txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr;
1650                 txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc;
1651                 txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr;
1652                 txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc;
1653                 txs->driver_data = vtophys(txq);
1654                 txs->driver_data_len = sizeof(struct vmxnet3_txqueue);
1655         }
1656
1657         /* Rx queues */
1658         for (i = 0; i < sc->vmx_nrxqueues; i++) {
1659                 rxq = &sc->vmx_rxq[i];
1660                 rxs = rxq->vxrxq_rs;
1661
1662                 rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr;
1663                 rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc;
1664                 rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr;
1665                 rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc;
1666                 rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr;
1667                 rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc;
1668                 rxs->driver_data = vtophys(rxq);
1669                 rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue);
1670         }
1671 }
1672
1673 static void
1674 vmxnet3_init_hwassist(struct vmxnet3_softc *sc)
1675 {
1676         struct ifnet *ifp = sc->vmx_ifp;
1677         uint64_t hwassist;
1678
1679         hwassist = 0;
1680         if (ifp->if_capenable & IFCAP_TXCSUM)
1681                 hwassist |= VMXNET3_CSUM_OFFLOAD;
1682         if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
1683                 hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6;
1684 #if 0 /* XXX TSO */
1685         if (ifp->if_capenable & IFCAP_TSO4)
1686                 hwassist |= CSUM_IP_TSO;
1687         if (ifp->if_capenable & IFCAP_TSO6)
1688                 hwassist |= CSUM_IP6_TSO;
1689 #endif
1690         ifp->if_hwassist = hwassist;
1691 }
1692
1693 static void
1694 vmxnet3_reinit_interface(struct vmxnet3_softc *sc)
1695 {
1696         struct ifnet *ifp;
1697
1698         ifp = sc->vmx_ifp;
1699
1700         /* Use the current MAC address. */
1701         bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN);
1702         vmxnet3_set_lladdr(sc);
1703
1704         vmxnet3_init_hwassist(sc);
1705 }
1706
1707 static void
1708 vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc)
1709 {
1710         /*
1711          * Use the same key as the Linux driver until FreeBSD can do
1712          * RSS (presumably Toeplitz) in software.
1713          */
1714         static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {
1715             0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
1716             0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
1717             0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
1718             0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
1719             0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
1720         };
1721
1722         struct vmxnet3_driver_shared *ds;
1723         struct vmxnet3_rss_shared *rss;
1724         int i;
1725
1726         ds = sc->vmx_ds;
1727         rss = sc->vmx_rss;
1728
1729         rss->hash_type =
1730             UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 |
1731             UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6;
1732         rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ;
1733         rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE;
1734         rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE;
1735         memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE);
1736
1737         for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++)
1738                 rss->ind_table[i] = i % sc->vmx_nrxqueues;
1739 }
1740
1741 static void
1742 vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc)
1743 {
1744         struct ifnet *ifp;
1745         struct vmxnet3_driver_shared *ds;
1746
1747         ifp = sc->vmx_ifp;
1748         ds = sc->vmx_ds;
1749
1750         ds->mtu = ifp->if_mtu;
1751         ds->ntxqueue = sc->vmx_ntxqueues;
1752         ds->nrxqueue = sc->vmx_nrxqueues;
1753
1754         ds->upt_features = 0;
1755         if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6))
1756                 ds->upt_features |= UPT1_F_CSUM;
1757         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1758                 ds->upt_features |= UPT1_F_VLAN;
1759 #if 0 /* XXX LRO */
1760         if (ifp->if_capenable & IFCAP_LRO)
1761                 ds->upt_features |= UPT1_F_LRO;
1762 #endif
1763
1764         if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1765                 ds->upt_features |= UPT1_F_RSS;
1766                 vmxnet3_reinit_rss_shared_data(sc);
1767         }
1768
1769         vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr);
1770         vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH,
1771             (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32);
1772 }
1773
1774 static int
1775 vmxnet3_alloc_data(struct vmxnet3_softc *sc)
1776 {
1777         int error;
1778
1779         error = vmxnet3_alloc_shared_data(sc);
1780         if (error)
1781                 return (error);
1782
1783         error = vmxnet3_alloc_queue_data(sc);
1784         if (error)
1785                 return (error);
1786
1787         error = vmxnet3_alloc_mcast_table(sc);
1788         if (error)
1789                 return (error);
1790
1791         vmxnet3_init_shared_data(sc);
1792
1793         return (0);
1794 }
1795
1796 static void
1797 vmxnet3_free_data(struct vmxnet3_softc *sc)
1798 {
1799
1800         vmxnet3_free_mcast_table(sc);
1801         vmxnet3_free_queue_data(sc);
1802         vmxnet3_free_shared_data(sc);
1803 }
1804
1805 static int
1806 vmxnet3_setup_interface(struct vmxnet3_softc *sc)
1807 {
1808         device_t dev;
1809         struct ifnet *ifp;
1810
1811         dev = sc->vmx_dev;
1812
1813         ifp = sc->vmx_ifp = if_alloc(IFT_ETHER);
1814         if (ifp == NULL) {
1815                 device_printf(dev, "cannot allocate ifnet structure\n");
1816                 return (ENOSPC);
1817         }
1818
1819         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1820         ifp->if_baudrate = IF_Gbps(10ULL);
1821         ifp->if_softc = sc;
1822         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1823         ifp->if_init = vmxnet3_init;
1824         ifp->if_ioctl = vmxnet3_ioctl;
1825 #if 0 /* XXX TSO */
1826         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1827         ifp->if_hw_tsomaxsegcount = VMXNET3_TX_MAXSEGS;
1828         ifp->if_hw_tsomaxsegsize = VMXNET3_TX_MAXSEGSIZE;
1829 #endif
1830
1831 #ifdef VMXNET3_LEGACY_TX
1832         ifp->if_start = vmxnet3_start;
1833         ifq_set_maxlen(&ifp->if_snd, sc->vmx_ntxdescs - 1);
1834         ifq_set_ready(&ifp->if_snd);
1835 #else
1836         ifp->if_transmit = vmxnet3_txq_mq_start;
1837         ifp->if_qflush = vmxnet3_qflush;
1838 #endif
1839
1840         vmxnet3_get_lladdr(sc);
1841         ether_ifattach(ifp, sc->vmx_lladdr, NULL);
1842
1843         ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM;
1844         ifp->if_capabilities |= IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6;
1845 #if 0 /* XXX TSO */
1846         ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6;
1847 #endif
1848         ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING |
1849             IFCAP_VLAN_HWCSUM;
1850         ifp->if_capenable = ifp->if_capabilities;
1851
1852 #if 0 /* XXX LRO / VLAN_HWFILTER */
1853         /* These capabilities are not enabled by default. */
1854         ifp->if_capabilities |= /* IFCAP_LRO | */ IFCAP_VLAN_HWFILTER;
1855 #endif
1856
1857         sc->vmx_vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
1858             vmxnet3_register_vlan, sc, EVENTHANDLER_PRI_FIRST);
1859         sc->vmx_vlan_detach = EVENTHANDLER_REGISTER(vlan_config,
1860             vmxnet3_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST);
1861
1862         ifmedia_init(&sc->vmx_media, 0, vmxnet3_media_change,
1863             vmxnet3_media_status);
1864         ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL);
1865         ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO);
1866
1867         return (0);
1868 }
1869
1870 static void
1871 vmxnet3_evintr(struct vmxnet3_softc *sc)
1872 {
1873         device_t dev;
1874         struct ifnet *ifp;
1875         struct vmxnet3_txq_shared *ts;
1876         struct vmxnet3_rxq_shared *rs;
1877         uint32_t event;
1878         int reset;
1879
1880         dev = sc->vmx_dev;
1881         ifp = sc->vmx_ifp;
1882         reset = 0;
1883
1884         VMXNET3_CORE_LOCK(sc);
1885
1886         /* Clear events. */
1887         event = sc->vmx_ds->event;
1888         vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event);
1889
1890         if (event & VMXNET3_EVENT_LINK) {
1891                 vmxnet3_link_status(sc);
1892                 if (sc->vmx_link_active != 0)
1893                         vmxnet3_tx_start_all(sc);
1894         }
1895
1896         if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) {
1897                 reset = 1;
1898                 vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS);
1899                 ts = sc->vmx_txq[0].vxtxq_ts;
1900                 if (ts->stopped != 0)
1901                         device_printf(dev, "Tx queue error %#x\n", ts->error);
1902                 rs = sc->vmx_rxq[0].vxrxq_rs;
1903                 if (rs->stopped != 0)
1904                         device_printf(dev, "Rx queue error %#x\n", rs->error);
1905                 device_printf(dev, "Rx/Tx queue error event ... resetting\n");
1906         }
1907
1908         if (event & VMXNET3_EVENT_DIC)
1909                 device_printf(dev, "device implementation change event\n");
1910         if (event & VMXNET3_EVENT_DEBUG)
1911                 device_printf(dev, "debug event\n");
1912
1913         if (reset != 0) {
1914                 ifp->if_flags &= ~IFF_RUNNING;
1915                 vmxnet3_init_locked(sc);
1916         }
1917
1918         VMXNET3_CORE_UNLOCK(sc);
1919 }
1920
1921 static void
1922 vmxnet3_txq_eof(struct vmxnet3_txqueue *txq)
1923 {
1924         struct vmxnet3_softc *sc;
1925         struct ifnet *ifp;
1926         struct vmxnet3_txring *txr;
1927         struct vmxnet3_comp_ring *txc;
1928         struct vmxnet3_txcompdesc *txcd;
1929         struct vmxnet3_txbuf *txb;
1930         struct mbuf *m;
1931         u_int sop;
1932
1933         sc = txq->vxtxq_sc;
1934         ifp = sc->vmx_ifp;
1935         txr = &txq->vxtxq_cmd_ring;
1936         txc = &txq->vxtxq_comp_ring;
1937
1938         VMXNET3_TXQ_LOCK_ASSERT(txq);
1939
1940         for (;;) {
1941                 txcd = &txc->vxcr_u.txcd[txc->vxcr_next];
1942                 if (txcd->gen != txc->vxcr_gen)
1943                         break;
1944                 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
1945
1946                 if (++txc->vxcr_next == txc->vxcr_ndesc) {
1947                         txc->vxcr_next = 0;
1948                         txc->vxcr_gen ^= 1;
1949                 }
1950
1951                 sop = txr->vxtxr_next;
1952                 txb = &txr->vxtxr_txbuf[sop];
1953
1954                 if ((m = txb->vtxb_m) != NULL) {
1955                         bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap,
1956                             BUS_DMASYNC_POSTWRITE);
1957                         bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap);
1958
1959                         txq->vxtxq_stats.vmtxs_opackets++;
1960                         txq->vxtxq_stats.vmtxs_obytes += m->m_pkthdr.len;
1961                         if (m->m_flags & M_MCAST)
1962                                 txq->vxtxq_stats.vmtxs_omcasts++;
1963
1964                         m_freem(m);
1965                         txb->vtxb_m = NULL;
1966                 }
1967
1968                 txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc;
1969         }
1970
1971         if (txr->vxtxr_head == txr->vxtxr_next)
1972                 txq->vxtxq_watchdog = 0;
1973 }
1974
1975 static int
1976 vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr)
1977 {
1978         struct ifnet *ifp;
1979         struct mbuf *m;
1980         struct vmxnet3_rxdesc *rxd;
1981         struct vmxnet3_rxbuf *rxb;
1982         bus_dma_tag_t tag;
1983         bus_dmamap_t dmap;
1984         bus_dma_segment_t segs[1];
1985         int idx, clsize, btype, flags, nsegs, error;
1986
1987         ifp = sc->vmx_ifp;
1988         tag = rxr->vxrxr_rxtag;
1989         dmap = rxr->vxrxr_spare_dmap;
1990         idx = rxr->vxrxr_fill;
1991         rxd = &rxr->vxrxr_rxd[idx];
1992         rxb = &rxr->vxrxr_rxbuf[idx];
1993
1994 #ifdef VMXNET3_FAILPOINTS
1995         KFAIL_POINT_CODE(VMXNET3_FP, newbuf, return ENOBUFS);
1996         if (rxr->vxrxr_rid != 0)
1997                 KFAIL_POINT_CODE(VMXNET3_FP, newbuf_body_only, return ENOBUFS);
1998 #endif
1999
2000         if (rxr->vxrxr_rid == 0 && (idx % sc->vmx_rx_max_chain) == 0) {
2001                 flags = M_PKTHDR;
2002                 clsize = MCLBYTES;
2003                 btype = VMXNET3_BTYPE_HEAD;
2004         } else {
2005                 flags = M_PKTHDR;
2006                 clsize = MJUMPAGESIZE;
2007                 btype = VMXNET3_BTYPE_BODY;
2008         }
2009
2010         m = m_getjcl(M_NOWAIT, MT_DATA, flags, clsize);
2011         if (m == NULL) {
2012                 sc->vmx_stats.vmst_mgetcl_failed++;
2013                 return (ENOBUFS);
2014         }
2015
2016         if (btype == VMXNET3_BTYPE_HEAD) {
2017                 m->m_len = m->m_pkthdr.len = clsize;
2018                 m_adj(m, ETHER_ALIGN);
2019         } else
2020                 m->m_len = clsize;
2021
2022         error = bus_dmamap_load_mbuf_segment(tag, dmap, m, &segs[0], 1, &nsegs,
2023             BUS_DMA_NOWAIT);
2024         if (error) {
2025                 m_freem(m);
2026                 sc->vmx_stats.vmst_mbuf_load_failed++;
2027                 return (error);
2028         }
2029         KASSERT(nsegs == 1,
2030             ("%s: mbuf %p with too many segments %d", __func__, m, nsegs));
2031         if (btype == VMXNET3_BTYPE_BODY)
2032                 m->m_flags &= ~M_PKTHDR;
2033
2034         if (rxb->vrxb_m != NULL) {
2035                 bus_dmamap_sync(tag, rxb->vrxb_dmamap, BUS_DMASYNC_POSTREAD);
2036                 bus_dmamap_unload(tag, rxb->vrxb_dmamap);
2037         }
2038
2039         rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap;
2040         rxb->vrxb_dmamap = dmap;
2041         rxb->vrxb_m = m;
2042
2043         rxd->addr = segs[0].ds_addr;
2044         rxd->len = segs[0].ds_len;
2045         rxd->btype = btype;
2046         rxd->gen = rxr->vxrxr_gen;
2047
2048         vmxnet3_rxr_increment_fill(rxr);
2049         return (0);
2050 }
2051
2052 static void
2053 vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq,
2054     struct vmxnet3_rxring *rxr, int idx)
2055 {
2056         struct vmxnet3_rxdesc *rxd;
2057
2058         rxd = &rxr->vxrxr_rxd[idx];
2059         rxd->gen = rxr->vxrxr_gen;
2060         vmxnet3_rxr_increment_fill(rxr);
2061 }
2062
2063 static void
2064 vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq)
2065 {
2066         struct vmxnet3_softc *sc;
2067         struct vmxnet3_rxring *rxr;
2068         struct vmxnet3_comp_ring *rxc;
2069         struct vmxnet3_rxcompdesc *rxcd;
2070         int idx, eof;
2071
2072         sc = rxq->vxrxq_sc;
2073         rxc = &rxq->vxrxq_comp_ring;
2074
2075         do {
2076                 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next];
2077                 if (rxcd->gen != rxc->vxcr_gen)
2078                         break;          /* Not expected. */
2079                 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
2080
2081                 if (++rxc->vxcr_next == rxc->vxcr_ndesc) {
2082                         rxc->vxcr_next = 0;
2083                         rxc->vxcr_gen ^= 1;
2084                 }
2085
2086                 idx = rxcd->rxd_idx;
2087                 eof = rxcd->eop;
2088                 if (rxcd->qid < sc->vmx_nrxqueues)
2089                         rxr = &rxq->vxrxq_cmd_ring[0];
2090                 else
2091                         rxr = &rxq->vxrxq_cmd_ring[1];
2092                 vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2093         } while (!eof);
2094 }
2095
2096 static void
2097 vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m)
2098 {
2099
2100         if (rxcd->ipv4) {
2101                 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
2102                 if (rxcd->ipcsum_ok)
2103                         m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2104         }
2105
2106         if (!rxcd->fragment) {
2107                 if (rxcd->csum_ok && (rxcd->tcp || rxcd->udp)) {
2108                         m->m_pkthdr.csum_flags |= CSUM_DATA_VALID |
2109                             CSUM_PSEUDO_HDR;
2110                         m->m_pkthdr.csum_data = 0xFFFF;
2111                 }
2112         }
2113 }
2114
2115 static void
2116 vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq,
2117     struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m)
2118 {
2119         struct vmxnet3_softc *sc;
2120         struct ifnet *ifp;
2121
2122         sc = rxq->vxrxq_sc;
2123         ifp = sc->vmx_ifp;
2124
2125         if (rxcd->error) {
2126                 rxq->vxrxq_stats.vmrxs_ierrors++;
2127                 m_freem(m);
2128                 return;
2129         }
2130
2131 #if 0
2132 #ifdef notyet
2133         switch (rxcd->rss_type) {
2134         case VMXNET3_RCD_RSS_TYPE_IPV4:
2135                 m->m_pkthdr.flowid = rxcd->rss_hash;
2136                 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV4);
2137                 break;
2138         case VMXNET3_RCD_RSS_TYPE_TCPIPV4:
2139                 m->m_pkthdr.flowid = rxcd->rss_hash;
2140                 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV4);
2141                 break;
2142         case VMXNET3_RCD_RSS_TYPE_IPV6:
2143                 m->m_pkthdr.flowid = rxcd->rss_hash;
2144                 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV6);
2145                 break;
2146         case VMXNET3_RCD_RSS_TYPE_TCPIPV6:
2147                 m->m_pkthdr.flowid = rxcd->rss_hash;
2148                 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV6);
2149                 break;
2150         default: /* VMXNET3_RCD_RSS_TYPE_NONE */
2151                 m->m_pkthdr.flowid = rxq->vxrxq_id;
2152                 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
2153                 break;
2154         }
2155 #else
2156         m->m_pkthdr.flowid = rxq->vxrxq_id;
2157         M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
2158 #endif
2159 #endif
2160
2161         if (!rxcd->no_csum)
2162                 vmxnet3_rx_csum(rxcd, m);
2163         if (rxcd->vlan) {
2164                 m->m_flags |= M_VLANTAG;
2165                 m->m_pkthdr.ether_vlantag = rxcd->vtag;
2166         }
2167
2168         rxq->vxrxq_stats.vmrxs_ipackets++;
2169         rxq->vxrxq_stats.vmrxs_ibytes += m->m_pkthdr.len;
2170
2171         VMXNET3_RXQ_UNLOCK(rxq);
2172         (*ifp->if_input)(ifp, m, NULL, -1);
2173         VMXNET3_RXQ_LOCK(rxq);
2174 }
2175
2176 static void
2177 vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq)
2178 {
2179         struct vmxnet3_softc *sc;
2180         struct ifnet *ifp;
2181         struct vmxnet3_rxring *rxr;
2182         struct vmxnet3_comp_ring *rxc;
2183         struct vmxnet3_rxdesc *rxd;
2184         struct vmxnet3_rxcompdesc *rxcd;
2185         struct mbuf *m, *m_head, *m_tail;
2186         int idx, length;
2187
2188         sc = rxq->vxrxq_sc;
2189         ifp = sc->vmx_ifp;
2190         rxc = &rxq->vxrxq_comp_ring;
2191
2192         VMXNET3_RXQ_LOCK_ASSERT(rxq);
2193
2194         if ((ifp->if_flags & IFF_RUNNING) == 0)
2195                 return;
2196
2197         m_head = rxq->vxrxq_mhead;
2198         rxq->vxrxq_mhead = NULL;
2199         m_tail = rxq->vxrxq_mtail;
2200         rxq->vxrxq_mtail = NULL;
2201         KKASSERT(m_head == NULL || m_tail != NULL);
2202
2203         for (;;) {
2204                 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next];
2205                 if (rxcd->gen != rxc->vxcr_gen) {
2206                         rxq->vxrxq_mhead = m_head;
2207                         rxq->vxrxq_mtail = m_tail;
2208                         break;
2209                 }
2210                 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
2211
2212                 if (++rxc->vxcr_next == rxc->vxcr_ndesc) {
2213                         rxc->vxcr_next = 0;
2214                         rxc->vxcr_gen ^= 1;
2215                 }
2216
2217                 idx = rxcd->rxd_idx;
2218                 length = rxcd->len;
2219                 if (rxcd->qid < sc->vmx_nrxqueues)
2220                         rxr = &rxq->vxrxq_cmd_ring[0];
2221                 else
2222                         rxr = &rxq->vxrxq_cmd_ring[1];
2223                 rxd = &rxr->vxrxr_rxd[idx];
2224
2225                 m = rxr->vxrxr_rxbuf[idx].vrxb_m;
2226                 KASSERT(m != NULL, ("%s: queue %d idx %d without mbuf",
2227                     __func__, rxcd->qid, idx));
2228
2229                 /*
2230                  * The host may skip descriptors. We detect this when this
2231                  * descriptor does not match the previous fill index. Catch
2232                  * up with the host now.
2233                  */
2234                 if (__predict_false(rxr->vxrxr_fill != idx)) {
2235                         while (rxr->vxrxr_fill != idx) {
2236                                 rxr->vxrxr_rxd[rxr->vxrxr_fill].gen =
2237                                     rxr->vxrxr_gen;
2238                                 vmxnet3_rxr_increment_fill(rxr);
2239                         }
2240                 }
2241
2242                 if (rxcd->sop) {
2243                         KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD,
2244                             ("%s: start of frame w/o head buffer", __func__));
2245                         KASSERT(rxr == &rxq->vxrxq_cmd_ring[0],
2246                             ("%s: start of frame not in ring 0", __func__));
2247                         KASSERT((idx % sc->vmx_rx_max_chain) == 0,
2248                             ("%s: start of frame at unexcepted index %d (%d)",
2249                              __func__, idx, sc->vmx_rx_max_chain));
2250                         KASSERT(m_head == NULL,
2251                             ("%s: duplicate start of frame?", __func__));
2252
2253                         if (length == 0) {
2254                                 /* Just ignore this descriptor. */
2255                                 vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2256                                 goto nextp;
2257                         }
2258
2259                         if (vmxnet3_newbuf(sc, rxr) != 0) {
2260                                 rxq->vxrxq_stats.vmrxs_iqdrops++;
2261                                 vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2262                                 if (!rxcd->eop)
2263                                         vmxnet3_rxq_discard_chain(rxq);
2264                                 goto nextp;
2265                         }
2266
2267                         m->m_pkthdr.rcvif = ifp;
2268                         m->m_pkthdr.len = m->m_len = length;
2269                         m->m_pkthdr.csum_flags = 0;
2270                         m_head = m_tail = m;
2271
2272                 } else {
2273                         KASSERT(rxd->btype == VMXNET3_BTYPE_BODY,
2274                             ("%s: non start of frame w/o body buffer", __func__));
2275
2276                         if (m_head == NULL && m_tail == NULL) {
2277                                 /*
2278                                  * This is a continuation of a packet that we
2279                                  * started to drop, but could not drop entirely
2280                                  * because this segment was still owned by the
2281                                  * host.  So, drop the remainder now.
2282                                  */
2283                                 vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2284                                 if (!rxcd->eop)
2285                                         vmxnet3_rxq_discard_chain(rxq);
2286                                 goto nextp;
2287                         }
2288
2289                         KASSERT(m_head != NULL,
2290                             ("%s: frame not started?", __func__));
2291
2292                         if (vmxnet3_newbuf(sc, rxr) != 0) {
2293                                 rxq->vxrxq_stats.vmrxs_iqdrops++;
2294                                 vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2295                                 if (!rxcd->eop)
2296                                         vmxnet3_rxq_discard_chain(rxq);
2297                                 m_freem(m_head);
2298                                 m_head = m_tail = NULL;
2299                                 goto nextp;
2300                         }
2301
2302                         m->m_len = length;
2303                         m_head->m_pkthdr.len += length;
2304                         m_tail->m_next = m;
2305                         m_tail = m;
2306                 }
2307
2308                 if (rxcd->eop) {
2309                         vmxnet3_rxq_input(rxq, rxcd, m_head);
2310                         m_head = m_tail = NULL;
2311
2312                         /* Must recheck after dropping the Rx lock. */
2313                         if ((ifp->if_flags & IFF_RUNNING) == 0)
2314                                 break;
2315                 }
2316
2317 nextp:
2318                 if (__predict_false(rxq->vxrxq_rs->update_rxhead)) {
2319                         int qid = rxcd->qid;
2320                         bus_size_t r;
2321
2322                         idx = (idx + 1) % rxr->vxrxr_ndesc;
2323                         if (qid >= sc->vmx_nrxqueues) {
2324                                 qid -= sc->vmx_nrxqueues;
2325                                 r = VMXNET3_BAR0_RXH2(qid);
2326                         } else
2327                                 r = VMXNET3_BAR0_RXH1(qid);
2328                         vmxnet3_write_bar0(sc, r, idx);
2329                 }
2330         }
2331 }
2332
2333 static void
2334 vmxnet3_legacy_intr(void *xsc)
2335 {
2336         struct vmxnet3_softc *sc;
2337         struct vmxnet3_rxqueue *rxq;
2338         struct vmxnet3_txqueue *txq;
2339
2340         sc = xsc;
2341         rxq = &sc->vmx_rxq[0];
2342         txq = &sc->vmx_txq[0];
2343
2344         if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) {
2345                 if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0)
2346                         return;
2347         }
2348         if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2349                 vmxnet3_disable_all_intrs(sc);
2350
2351         if (sc->vmx_ds->event != 0)
2352                 vmxnet3_evintr(sc);
2353
2354         VMXNET3_RXQ_LOCK(rxq);
2355         vmxnet3_rxq_eof(rxq);
2356         VMXNET3_RXQ_UNLOCK(rxq);
2357
2358         VMXNET3_TXQ_LOCK(txq);
2359         vmxnet3_txq_eof(txq);
2360         vmxnet3_txq_start(txq);
2361         VMXNET3_TXQ_UNLOCK(txq);
2362
2363         vmxnet3_enable_all_intrs(sc);
2364 }
2365
2366 #ifdef __FreeBSD__
2367 static void
2368 vmxnet3_txq_intr(void *xtxq)
2369 {
2370         struct vmxnet3_softc *sc;
2371         struct vmxnet3_txqueue *txq;
2372
2373         txq = xtxq;
2374         sc = txq->vxtxq_sc;
2375
2376         if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2377                 vmxnet3_disable_intr(sc, txq->vxtxq_intr_idx);
2378
2379         VMXNET3_TXQ_LOCK(txq);
2380         vmxnet3_txq_eof(txq);
2381         vmxnet3_txq_start(txq);
2382         VMXNET3_TXQ_UNLOCK(txq);
2383
2384         vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx);
2385 }
2386
2387 static void
2388 vmxnet3_rxq_intr(void *xrxq)
2389 {
2390         struct vmxnet3_softc *sc;
2391         struct vmxnet3_rxqueue *rxq;
2392
2393         rxq = xrxq;
2394         sc = rxq->vxrxq_sc;
2395
2396         if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2397                 vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx);
2398
2399         VMXNET3_RXQ_LOCK(rxq);
2400         vmxnet3_rxq_eof(rxq);
2401         VMXNET3_RXQ_UNLOCK(rxq);
2402
2403         vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx);
2404 }
2405
2406 static void
2407 vmxnet3_event_intr(void *xsc)
2408 {
2409         struct vmxnet3_softc *sc;
2410
2411         sc = xsc;
2412
2413         if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2414                 vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx);
2415
2416         if (sc->vmx_ds->event != 0)
2417                 vmxnet3_evintr(sc);
2418
2419         vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx);
2420 }
2421 #endif
2422
2423 static void
2424 vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
2425 {
2426         struct vmxnet3_txring *txr;
2427         struct vmxnet3_txbuf *txb;
2428         int i;
2429
2430         txr = &txq->vxtxq_cmd_ring;
2431
2432         for (i = 0; i < txr->vxtxr_ndesc; i++) {
2433                 txb = &txr->vxtxr_txbuf[i];
2434
2435                 if (txb->vtxb_m == NULL)
2436                         continue;
2437
2438                 bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap,
2439                     BUS_DMASYNC_POSTWRITE);
2440                 bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap);
2441                 m_freem(txb->vtxb_m);
2442                 txb->vtxb_m = NULL;
2443         }
2444 }
2445
2446 static void
2447 vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
2448 {
2449         struct vmxnet3_rxring *rxr;
2450         struct vmxnet3_rxbuf *rxb;
2451         int i, j;
2452
2453         if (rxq->vxrxq_mhead != NULL) {
2454                 m_freem(rxq->vxrxq_mhead);
2455                 rxq->vxrxq_mhead = NULL;
2456                 rxq->vxrxq_mtail = NULL;
2457         }
2458
2459         for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
2460                 rxr = &rxq->vxrxq_cmd_ring[i];
2461
2462                 for (j = 0; j < rxr->vxrxr_ndesc; j++) {
2463                         rxb = &rxr->vxrxr_rxbuf[j];
2464
2465                         if (rxb->vrxb_m == NULL)
2466                                 continue;
2467
2468                         bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap,
2469                             BUS_DMASYNC_POSTREAD);
2470                         bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap);
2471                         m_freem(rxb->vrxb_m);
2472                         rxb->vrxb_m = NULL;
2473                 }
2474         }
2475 }
2476
2477 static void
2478 vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc)
2479 {
2480         struct vmxnet3_rxqueue *rxq;
2481         struct vmxnet3_txqueue *txq;
2482         int i;
2483
2484         for (i = 0; i < sc->vmx_nrxqueues; i++) {
2485                 rxq = &sc->vmx_rxq[i];
2486                 VMXNET3_RXQ_LOCK(rxq);
2487                 VMXNET3_RXQ_UNLOCK(rxq);
2488         }
2489
2490         for (i = 0; i < sc->vmx_ntxqueues; i++) {
2491                 txq = &sc->vmx_txq[i];
2492                 VMXNET3_TXQ_LOCK(txq);
2493                 VMXNET3_TXQ_UNLOCK(txq);
2494         }
2495 }
2496
2497 static void
2498 vmxnet3_stop(struct vmxnet3_softc *sc)
2499 {
2500         struct ifnet *ifp;
2501         int q;
2502
2503         ifp = sc->vmx_ifp;
2504         VMXNET3_CORE_LOCK_ASSERT(sc);
2505
2506         ifp->if_flags &= ~IFF_RUNNING;
2507         sc->vmx_link_active = 0;
2508         callout_stop(&sc->vmx_tick);
2509
2510         /* Disable interrupts. */
2511         vmxnet3_disable_all_intrs(sc);
2512         vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE);
2513
2514         vmxnet3_stop_rendezvous(sc);
2515
2516         for (q = 0; q < sc->vmx_ntxqueues; q++)
2517                 vmxnet3_txstop(sc, &sc->vmx_txq[q]);
2518         for (q = 0; q < sc->vmx_nrxqueues; q++)
2519                 vmxnet3_rxstop(sc, &sc->vmx_rxq[q]);
2520
2521         vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET);
2522 }
2523
2524 static void
2525 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
2526 {
2527         struct vmxnet3_txring *txr;
2528         struct vmxnet3_comp_ring *txc;
2529
2530         txr = &txq->vxtxq_cmd_ring;
2531         txr->vxtxr_head = 0;
2532         txr->vxtxr_next = 0;
2533         txr->vxtxr_gen = VMXNET3_INIT_GEN;
2534         bzero(txr->vxtxr_txd,
2535             txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc));
2536
2537         txc = &txq->vxtxq_comp_ring;
2538         txc->vxcr_next = 0;
2539         txc->vxcr_gen = VMXNET3_INIT_GEN;
2540         bzero(txc->vxcr_u.txcd,
2541             txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc));
2542 }
2543
2544 static int
2545 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
2546 {
2547         struct ifnet *ifp;
2548         struct vmxnet3_rxring *rxr;
2549         struct vmxnet3_comp_ring *rxc;
2550         int i, populate, idx, frame_size, error;
2551
2552         ifp = sc->vmx_ifp;
2553         frame_size = ETHER_ALIGN + sizeof(struct ether_vlan_header) +
2554             ifp->if_mtu;
2555
2556         /*
2557          * If the MTU causes us to exceed what a regular sized cluster can
2558          * handle, we allocate a second MJUMPAGESIZE cluster after it in
2559          * ring 0. If in use, ring 1 always contains MJUMPAGESIZE clusters.
2560          *
2561          * Keep rx_max_chain a divisor of the maximum Rx ring size to make
2562          * our life easier. We do not support changing the ring size after
2563          * the attach.
2564          */
2565         if (frame_size <= MCLBYTES)
2566                 sc->vmx_rx_max_chain = 1;
2567         else
2568                 sc->vmx_rx_max_chain = 2;
2569
2570         /*
2571          * Only populate ring 1 if the configuration will take advantage
2572          * of it. That is either when LRO is enabled or the frame size
2573          * exceeds what ring 0 can contain.
2574          */
2575 #if 0 /* XXX LRO */
2576         if ((ifp->if_capenable & IFCAP_LRO) == 0 &&
2577 #else
2578         if (
2579 #endif
2580             frame_size <= MCLBYTES + MJUMPAGESIZE)
2581                 populate = 1;
2582         else
2583                 populate = VMXNET3_RXRINGS_PERQ;
2584
2585         for (i = 0; i < populate; i++) {
2586                 rxr = &rxq->vxrxq_cmd_ring[i];
2587                 rxr->vxrxr_fill = 0;
2588                 rxr->vxrxr_gen = VMXNET3_INIT_GEN;
2589                 bzero(rxr->vxrxr_rxd,
2590                     rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
2591
2592                 for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) {
2593                         error = vmxnet3_newbuf(sc, rxr);
2594                         if (error)
2595                                 return (error);
2596                 }
2597         }
2598
2599         for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) {
2600                 rxr = &rxq->vxrxq_cmd_ring[i];
2601                 rxr->vxrxr_fill = 0;
2602                 rxr->vxrxr_gen = 0;
2603                 bzero(rxr->vxrxr_rxd,
2604                     rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
2605         }
2606
2607         rxc = &rxq->vxrxq_comp_ring;
2608         rxc->vxcr_next = 0;
2609         rxc->vxcr_gen = VMXNET3_INIT_GEN;
2610         bzero(rxc->vxcr_u.rxcd,
2611             rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc));
2612
2613         return (0);
2614 }
2615
2616 static int
2617 vmxnet3_reinit_queues(struct vmxnet3_softc *sc)
2618 {
2619         device_t dev;
2620         int q, error;
2621
2622         dev = sc->vmx_dev;
2623
2624         for (q = 0; q < sc->vmx_ntxqueues; q++)
2625                 vmxnet3_txinit(sc, &sc->vmx_txq[q]);
2626
2627         for (q = 0; q < sc->vmx_nrxqueues; q++) {
2628                 error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]);
2629                 if (error) {
2630                         device_printf(dev, "cannot populate Rx queue %d\n", q);
2631                         return (error);
2632                 }
2633         }
2634
2635         return (0);
2636 }
2637
2638 static int
2639 vmxnet3_enable_device(struct vmxnet3_softc *sc)
2640 {
2641         int q;
2642
2643         if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) {
2644                 device_printf(sc->vmx_dev, "device enable command failed!\n");
2645                 return (1);
2646         }
2647
2648         /* Reset the Rx queue heads. */
2649         for (q = 0; q < sc->vmx_nrxqueues; q++) {
2650                 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0);
2651                 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0);
2652         }
2653
2654         return (0);
2655 }
2656
2657 static void
2658 vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc)
2659 {
2660         struct ifnet *ifp;
2661
2662         ifp = sc->vmx_ifp;
2663
2664         vmxnet3_set_rxfilter(sc);
2665
2666 #if 0 /* VLAN_HWFILTER */
2667         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
2668                 bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter,
2669                     sizeof(sc->vmx_ds->vlan_filter));
2670         else
2671 #endif
2672                 bzero(sc->vmx_ds->vlan_filter,
2673                     sizeof(sc->vmx_ds->vlan_filter));
2674         vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER);
2675 }
2676
2677 static int
2678 vmxnet3_reinit(struct vmxnet3_softc *sc)
2679 {
2680
2681         vmxnet3_reinit_interface(sc);
2682         vmxnet3_reinit_shared_data(sc);
2683
2684         if (vmxnet3_reinit_queues(sc) != 0)
2685                 return (ENXIO);
2686
2687         if (vmxnet3_enable_device(sc) != 0)
2688                 return (ENXIO);
2689
2690         vmxnet3_reinit_rxfilters(sc);
2691
2692         return (0);
2693 }
2694
2695 static void
2696 vmxnet3_init_locked(struct vmxnet3_softc *sc)
2697 {
2698         struct ifnet *ifp;
2699
2700         ifp = sc->vmx_ifp;
2701
2702         if (ifp->if_flags & IFF_RUNNING)
2703                 return;
2704
2705         vmxnet3_stop(sc);
2706
2707         if (vmxnet3_reinit(sc) != 0) {
2708                 vmxnet3_stop(sc);
2709                 return;
2710         }
2711
2712         ifp->if_flags |= IFF_RUNNING;
2713         vmxnet3_link_status(sc);
2714
2715         vmxnet3_enable_all_intrs(sc);
2716         callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc);
2717 }
2718
2719 static void
2720 vmxnet3_init(void *xsc)
2721 {
2722         struct vmxnet3_softc *sc;
2723
2724         sc = xsc;
2725
2726         VMXNET3_CORE_LOCK(sc);
2727         vmxnet3_init_locked(sc);
2728         VMXNET3_CORE_UNLOCK(sc);
2729 }
2730
2731 /*
2732  * BMV: Much of this can go away once we finally have offsets in
2733  * the mbuf packet header. Bug andre@.
2734  */
2735 static int
2736 vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m,
2737     int *etype, int *proto, int *start)
2738 {
2739         struct ether_vlan_header *evh;
2740         int offset;
2741 #if defined(INET)
2742         struct ip *ip = NULL;
2743 #endif
2744 #if defined(INET6)
2745         struct ip6_hdr *ip6 = NULL;
2746 #endif
2747
2748         evh = mtod(m, struct ether_vlan_header *);
2749         if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
2750                 /* BMV: We should handle nested VLAN tags too. */
2751                 *etype = ntohs(evh->evl_proto);
2752                 offset = sizeof(struct ether_vlan_header);
2753         } else {
2754                 *etype = ntohs(evh->evl_encap_proto);
2755                 offset = sizeof(struct ether_header);
2756         }
2757
2758         switch (*etype) {
2759 #if defined(INET)
2760         case ETHERTYPE_IP:
2761                 if (__predict_false(m->m_len < offset + sizeof(struct ip))) {
2762                         m = m_pullup(m, offset + sizeof(struct ip));
2763                         if (m == NULL)
2764                                 return (EINVAL);
2765                 }
2766
2767                 ip = (struct ip *)(mtod(m, uint8_t *) + offset);
2768                 *proto = ip->ip_p;
2769                 *start = offset + (ip->ip_hl << 2);
2770                 break;
2771 #endif
2772 #if defined(INET6)
2773         case ETHERTYPE_IPV6:
2774                 if (__predict_false(m->m_len <
2775                     offset + sizeof(struct ip6_hdr))) {
2776                         m = m_pullup(m, offset + sizeof(struct ip6_hdr));
2777                         if (m == NULL)
2778                                 return (EINVAL);
2779                 }
2780
2781                 ip6 = (struct ip6_hdr *)(mtod(m, uint8_t *) + offset);
2782                 *proto = -1;
2783                 *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto);
2784                 /* Assert the network stack sent us a valid packet. */
2785                 KASSERT(*start > offset,
2786                     ("%s: mbuf %p start %d offset %d proto %d", __func__, m,
2787                     *start, offset, *proto));
2788                 break;
2789 #endif
2790         default:
2791                 return (EINVAL);
2792         }
2793
2794 #if 0 /* XXX TSO */
2795         if (m->m_pkthdr.csum_flags & CSUM_TSO) {
2796                 struct tcphdr *tcp;
2797
2798                 if (__predict_false(*proto != IPPROTO_TCP)) {
2799                         /* Likely failed to correctly parse the mbuf. */
2800                         return (EINVAL);
2801                 }
2802
2803                 if (m->m_len < *start + sizeof(struct tcphdr)) {
2804                         m = m_pullup(m, *start + sizeof(struct tcphdr));
2805                         if (m == NULL)
2806                                 return (EINVAL);
2807                 }
2808
2809                 tcp = (struct tcphdr *)(mtod(m, uint8_t *) + *start);
2810                 *start += (tcp->th_off << 2);
2811
2812                 txq->vxtxq_stats.vmtxs_tso++;
2813         } else
2814 #endif
2815                 txq->vxtxq_stats.vmtxs_csum++;
2816
2817         return (0);
2818 }
2819
2820 static int
2821 vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0,
2822     bus_dmamap_t dmap, bus_dma_segment_t segs[], int *nsegs)
2823 {
2824         struct vmxnet3_txring *txr;
2825         struct mbuf *m;
2826         bus_dma_tag_t tag;
2827         int error;
2828
2829         txr = &txq->vxtxq_cmd_ring;
2830         m = *m0;
2831         tag = txr->vxtxr_txtag;
2832
2833         error = bus_dmamap_load_mbuf_segment(tag, dmap, m, segs, 1, nsegs,
2834             BUS_DMA_NOWAIT);
2835         if (error == 0 || error != EFBIG)
2836                 return (error);
2837
2838         m = m_defrag(m, M_NOWAIT);
2839         if (m != NULL) {
2840                 *m0 = m;
2841                 error = bus_dmamap_load_mbuf_segment(tag, dmap, m, segs,
2842                     1, nsegs, BUS_DMA_NOWAIT);
2843         } else
2844                 error = ENOBUFS;
2845
2846         if (error) {
2847                 m_freem(*m0);
2848                 *m0 = NULL;
2849                 txq->vxtxq_sc->vmx_stats.vmst_defrag_failed++;
2850         } else
2851                 txq->vxtxq_sc->vmx_stats.vmst_defragged++;
2852
2853         return (error);
2854 }
2855
2856 static void
2857 vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap)
2858 {
2859         struct vmxnet3_txring *txr;
2860
2861         txr = &txq->vxtxq_cmd_ring;
2862         bus_dmamap_unload(txr->vxtxr_txtag, dmap);
2863 }
2864
2865 static int
2866 vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0)
2867 {
2868         struct vmxnet3_softc *sc;
2869         struct vmxnet3_txring *txr;
2870         struct vmxnet3_txdesc *txd, *sop;
2871         struct mbuf *m;
2872         bus_dmamap_t dmap;
2873         bus_dma_segment_t segs[VMXNET3_TX_MAXSEGS];
2874         int i, gen, nsegs, etype, proto, start, error;
2875
2876         sc = txq->vxtxq_sc;
2877         start = 0;
2878         txd = NULL;
2879         txr = &txq->vxtxq_cmd_ring;
2880         dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap;
2881
2882         error = vmxnet3_txq_load_mbuf(txq, m0, dmap, segs, &nsegs);
2883         if (error)
2884                 return (error);
2885
2886         m = *m0;
2887         M_ASSERTPKTHDR(m);
2888         KASSERT(nsegs <= VMXNET3_TX_MAXSEGS,
2889             ("%s: mbuf %p with too many segments %d", __func__, m, nsegs));
2890
2891         if (VMXNET3_TXRING_AVAIL(txr) < nsegs) {
2892                 txq->vxtxq_stats.vmtxs_full++;
2893                 vmxnet3_txq_unload_mbuf(txq, dmap);
2894                 return (ENOSPC);
2895         } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) {
2896                 error = vmxnet3_txq_offload_ctx(txq, m, &etype, &proto, &start);
2897                 if (error) {
2898                         txq->vxtxq_stats.vmtxs_offload_failed++;
2899                         vmxnet3_txq_unload_mbuf(txq, dmap);
2900                         m_freem(m);
2901                         *m0 = NULL;
2902                         return (error);
2903                 }
2904         }
2905
2906         txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m;
2907         sop = &txr->vxtxr_txd[txr->vxtxr_head];
2908         gen = txr->vxtxr_gen ^ 1;       /* Owned by cpu (yet) */
2909
2910         for (i = 0; i < nsegs; i++) {
2911                 txd = &txr->vxtxr_txd[txr->vxtxr_head];
2912
2913                 txd->addr = segs[i].ds_addr;
2914                 txd->len = segs[i].ds_len;
2915                 txd->gen = gen;
2916                 txd->dtype = 0;
2917                 txd->offload_mode = VMXNET3_OM_NONE;
2918                 txd->offload_pos = 0;
2919                 txd->hlen = 0;
2920                 txd->eop = 0;
2921                 txd->compreq = 0;
2922                 txd->vtag_mode = 0;
2923                 txd->vtag = 0;
2924
2925                 if (++txr->vxtxr_head == txr->vxtxr_ndesc) {
2926                         txr->vxtxr_head = 0;
2927                         txr->vxtxr_gen ^= 1;
2928                 }
2929                 gen = txr->vxtxr_gen;
2930         }
2931         txd->eop = 1;
2932         txd->compreq = 1;
2933
2934         if (m->m_flags & M_VLANTAG) {
2935                 sop->vtag_mode = 1;
2936                 sop->vtag = m->m_pkthdr.ether_vlantag;
2937         }
2938
2939
2940 #if 0 /* XXX TSO */
2941         if (m->m_pkthdr.csum_flags & CSUM_TSO) {
2942                 sop->offload_mode = VMXNET3_OM_TSO;
2943                 sop->hlen = start;
2944                 sop->offload_pos = m->m_pkthdr.tso_segsz;
2945         } else
2946 #endif
2947         if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD |
2948             VMXNET3_CSUM_OFFLOAD_IPV6)) {
2949                 sop->offload_mode = VMXNET3_OM_CSUM;
2950                 sop->hlen = start;
2951                 sop->offload_pos = start + m->m_pkthdr.csum_data;
2952         }
2953
2954         /* Finally, change the ownership. */
2955         vmxnet3_barrier(sc, VMXNET3_BARRIER_WR);
2956         sop->gen ^= 1;
2957
2958         txq->vxtxq_ts->npending += nsegs;
2959         if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) {
2960                 txq->vxtxq_ts->npending = 0;
2961                 vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id),
2962                     txr->vxtxr_head);
2963         }
2964
2965         return (0);
2966 }
2967
2968 #ifdef VMXNET3_LEGACY_TX
2969
2970 static void
2971 vmxnet3_start_locked(struct ifnet *ifp)
2972 {
2973         struct vmxnet3_softc *sc;
2974         struct vmxnet3_txqueue *txq;
2975         struct vmxnet3_txring *txr;
2976         struct mbuf *m_head;
2977         int tx, avail;
2978
2979         sc = ifp->if_softc;
2980         txq = &sc->vmx_txq[0];
2981         txr = &txq->vxtxq_cmd_ring;
2982         tx = 0;
2983
2984         VMXNET3_TXQ_LOCK_ASSERT(txq);
2985
2986         if ((ifp->if_flags & IFF_RUNNING) == 0 ||
2987             sc->vmx_link_active == 0)
2988                 return;
2989
2990         while (!ifq_is_empty(&ifp->if_snd)) {
2991                 if ((avail = VMXNET3_TXRING_AVAIL(txr)) < 2)
2992                         break;
2993
2994                 m_head = ifq_dequeue(&ifp->if_snd);
2995                 if (m_head == NULL)
2996                         break;
2997
2998                 /* Assume worse case if this mbuf is the head of a chain. */
2999                 if (m_head->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) {
3000                         ifq_prepend(&ifp->if_snd, m_head);
3001                         break;
3002                 }
3003
3004                 if (vmxnet3_txq_encap(txq, &m_head) != 0) {
3005                         if (m_head != NULL)
3006                                 ifq_prepend(&ifp->if_snd, m_head);
3007                         break;
3008                 }
3009
3010                 tx++;
3011                 ETHER_BPF_MTAP(ifp, m_head);
3012         }
3013
3014         if (tx > 0)
3015                 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT;
3016 }
3017
3018 static void
3019 vmxnet3_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
3020 {
3021         struct vmxnet3_softc *sc;
3022         struct vmxnet3_txqueue *txq;
3023
3024         sc = ifp->if_softc;
3025         txq = &sc->vmx_txq[0];
3026
3027         VMXNET3_TXQ_LOCK(txq);
3028         vmxnet3_start_locked(ifp);
3029         VMXNET3_TXQ_UNLOCK(txq);
3030 }
3031
3032 #else /* !VMXNET3_LEGACY_TX */
3033
3034 static int
3035 vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *txq, struct mbuf *m)
3036 {
3037         struct vmxnet3_softc *sc;
3038         struct vmxnet3_txring *txr;
3039         struct buf_ring *br;
3040         struct ifnet *ifp;
3041         int tx, avail, error;
3042
3043         sc = txq->vxtxq_sc;
3044         br = txq->vxtxq_br;
3045         ifp = sc->vmx_ifp;
3046         txr = &txq->vxtxq_cmd_ring;
3047         tx = 0;
3048         error = 0;
3049
3050         VMXNET3_TXQ_LOCK_ASSERT(txq);
3051
3052         if ((ifp->if_flags & IFF_RUNNING) == 0 ||
3053             sc->vmx_link_active == 0) {
3054                 if (m != NULL)
3055                         error = drbr_enqueue(ifp, br, m);
3056                 return (error);
3057         }
3058
3059         if (m != NULL) {
3060                 error = drbr_enqueue(ifp, br, m);
3061                 if (error)
3062                         return (error);
3063         }
3064
3065         while ((avail = VMXNET3_TXRING_AVAIL(txr)) >= 2) {
3066                 m = drbr_peek(ifp, br);
3067                 if (m == NULL)
3068                         break;
3069
3070                 /* Assume worse case if this mbuf is the head of a chain. */
3071                 if (m->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) {
3072                         drbr_putback(ifp, br, m);
3073                         break;
3074                 }
3075
3076                 if (vmxnet3_txq_encap(txq, &m) != 0) {
3077                         if (m != NULL)
3078                                 drbr_putback(ifp, br, m);
3079                         else
3080                                 drbr_advance(ifp, br);
3081                         break;
3082                 }
3083                 drbr_advance(ifp, br);
3084
3085                 tx++;
3086                 ETHER_BPF_MTAP(ifp, m);
3087         }
3088
3089         if (tx > 0)
3090                 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT;
3091
3092         return (0);
3093 }
3094
3095 static int
3096 vmxnet3_txq_mq_start(struct ifnet *ifp, struct mbuf *m)
3097 {
3098         struct vmxnet3_softc *sc;
3099         struct vmxnet3_txqueue *txq;
3100         int i, ntxq, error;
3101
3102         sc = ifp->if_softc;
3103         ntxq = sc->vmx_ntxqueues;
3104
3105         /* check if flowid is set */
3106         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
3107                 i = m->m_pkthdr.flowid % ntxq;
3108         else
3109                 i = curcpu % ntxq;
3110
3111         txq = &sc->vmx_txq[i];
3112
3113         if (VMXNET3_TXQ_TRYLOCK(txq) != 0) {
3114                 error = vmxnet3_txq_mq_start_locked(txq, m);
3115                 VMXNET3_TXQ_UNLOCK(txq);
3116         } else {
3117                 error = drbr_enqueue(ifp, txq->vxtxq_br, m);
3118                 taskqueue_enqueue(sc->vmx_tq, &txq->vxtxq_defrtask);
3119         }
3120
3121         return (error);
3122 }
3123
3124 static void
3125 vmxnet3_txq_tq_deferred(void *xtxq, int pending)
3126 {
3127         struct vmxnet3_softc *sc;
3128         struct vmxnet3_txqueue *txq;
3129
3130         txq = xtxq;
3131         sc = txq->vxtxq_sc;
3132
3133         VMXNET3_TXQ_LOCK(txq);
3134         if (!drbr_empty(sc->vmx_ifp, txq->vxtxq_br))
3135                 vmxnet3_txq_mq_start_locked(txq, NULL);
3136         VMXNET3_TXQ_UNLOCK(txq);
3137 }
3138
3139 #endif /* VMXNET3_LEGACY_TX */
3140
3141 static void
3142 vmxnet3_txq_start(struct vmxnet3_txqueue *txq)
3143 {
3144         struct vmxnet3_softc *sc;
3145         struct ifnet *ifp;
3146
3147         sc = txq->vxtxq_sc;
3148         ifp = sc->vmx_ifp;
3149
3150 #ifdef VMXNET3_LEGACY_TX
3151         if (!ifq_is_empty(&ifp->if_snd))
3152                 vmxnet3_start_locked(ifp);
3153 #else
3154         if (!drbr_empty(ifp, txq->vxtxq_br))
3155                 vmxnet3_txq_mq_start_locked(txq, NULL);
3156 #endif
3157 }
3158
3159 static void
3160 vmxnet3_tx_start_all(struct vmxnet3_softc *sc)
3161 {
3162         struct vmxnet3_txqueue *txq;
3163         int i;
3164
3165         VMXNET3_CORE_LOCK_ASSERT(sc);
3166
3167         for (i = 0; i < sc->vmx_ntxqueues; i++) {
3168                 txq = &sc->vmx_txq[i];
3169
3170                 VMXNET3_TXQ_LOCK(txq);
3171                 vmxnet3_txq_start(txq);
3172                 VMXNET3_TXQ_UNLOCK(txq);
3173         }
3174 }
3175
3176 static void
3177 vmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag)
3178 {
3179         struct ifnet *ifp;
3180         int idx, bit;
3181
3182         ifp = sc->vmx_ifp;
3183         idx = (tag >> 5) & 0x7F;
3184         bit = tag & 0x1F;
3185
3186         if (tag == 0 || tag > 4095)
3187                 return;
3188
3189         VMXNET3_CORE_LOCK(sc);
3190
3191         /* Update our private VLAN bitvector. */
3192         if (add)
3193                 sc->vmx_vlan_filter[idx] |= (1 << bit);
3194         else
3195                 sc->vmx_vlan_filter[idx] &= ~(1 << bit);
3196
3197 #if 0 /* VLAN_HWFILTER */
3198         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
3199                 if (add)
3200                         sc->vmx_ds->vlan_filter[idx] |= (1 << bit);
3201                 else
3202                         sc->vmx_ds->vlan_filter[idx] &= ~(1 << bit);
3203                 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER);
3204         }
3205 #endif
3206
3207         VMXNET3_CORE_UNLOCK(sc);
3208 }
3209
3210 static void
3211 vmxnet3_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
3212 {
3213
3214         if (ifp->if_softc == arg)
3215                 vmxnet3_update_vlan_filter(arg, 1, tag);
3216 }
3217
3218 static void
3219 vmxnet3_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
3220 {
3221
3222         if (ifp->if_softc == arg)
3223                 vmxnet3_update_vlan_filter(arg, 0, tag);
3224 }
3225
3226 static void
3227 vmxnet3_set_rxfilter(struct vmxnet3_softc *sc)
3228 {
3229         struct ifnet *ifp;
3230         struct vmxnet3_driver_shared *ds;
3231         struct ifmultiaddr *ifma;
3232         u_int mode;
3233
3234         ifp = sc->vmx_ifp;
3235         ds = sc->vmx_ds;
3236
3237         mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST;
3238         if (ifp->if_flags & IFF_PROMISC)
3239                 mode |= VMXNET3_RXMODE_PROMISC;
3240         if (ifp->if_flags & IFF_ALLMULTI)
3241                 mode |= VMXNET3_RXMODE_ALLMULTI;
3242         else {
3243                 int cnt = 0, overflow = 0;
3244
3245                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3246                         if (ifma->ifma_addr->sa_family != AF_LINK)
3247                                 continue;
3248                         else if (cnt == VMXNET3_MULTICAST_MAX) {
3249                                 overflow = 1;
3250                                 break;
3251                         }
3252
3253                         bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
3254                            &sc->vmx_mcast[cnt*ETHER_ADDR_LEN], ETHER_ADDR_LEN);
3255                         cnt++;
3256                 }
3257
3258                 if (overflow != 0) {
3259                         cnt = 0;
3260                         mode |= VMXNET3_RXMODE_ALLMULTI;
3261                 } else if (cnt > 0)
3262                         mode |= VMXNET3_RXMODE_MCAST;
3263                 ds->mcast_tablelen = cnt * ETHER_ADDR_LEN;
3264         }
3265
3266         ds->rxmode = mode;
3267
3268         vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER);
3269         vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE);
3270 }
3271
3272 static int
3273 vmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu)
3274 {
3275         struct ifnet *ifp;
3276
3277         ifp = sc->vmx_ifp;
3278
3279         if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU)
3280                 return (EINVAL);
3281
3282         ifp->if_mtu = mtu;
3283
3284         if (ifp->if_flags & IFF_RUNNING) {
3285                 ifp->if_flags &= ~IFF_RUNNING;
3286                 vmxnet3_init_locked(sc);
3287         }
3288
3289         return (0);
3290 }
3291
3292 static int
3293 vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cred)
3294 {
3295         struct vmxnet3_softc *sc;
3296         struct ifreq *ifr;
3297         int reinit, mask, error;
3298
3299         sc = ifp->if_softc;
3300         ifr = (struct ifreq *) data;
3301         error = 0;
3302
3303         switch (cmd) {
3304         case SIOCSIFMTU:
3305                 if (ifp->if_mtu != ifr->ifr_mtu) {
3306                         VMXNET3_CORE_LOCK(sc);
3307                         error = vmxnet3_change_mtu(sc, ifr->ifr_mtu);
3308                         VMXNET3_CORE_UNLOCK(sc);
3309                 }
3310                 break;
3311
3312         case SIOCSIFFLAGS:
3313                 VMXNET3_CORE_LOCK(sc);
3314                 if (ifp->if_flags & IFF_UP) {
3315                         if ((ifp->if_flags & IFF_RUNNING)) {
3316                                 if ((ifp->if_flags ^ sc->vmx_if_flags) &
3317                                     (IFF_PROMISC | IFF_ALLMULTI)) {
3318                                         vmxnet3_set_rxfilter(sc);
3319                                 }
3320                         } else
3321                                 vmxnet3_init_locked(sc);
3322                 } else {
3323                         if (ifp->if_flags & IFF_RUNNING)
3324                                 vmxnet3_stop(sc);
3325                 }
3326                 sc->vmx_if_flags = ifp->if_flags;
3327                 VMXNET3_CORE_UNLOCK(sc);
3328                 break;
3329
3330         case SIOCADDMULTI:
3331         case SIOCDELMULTI:
3332                 VMXNET3_CORE_LOCK(sc);
3333                 if (ifp->if_flags & IFF_RUNNING)
3334                         vmxnet3_set_rxfilter(sc);
3335                 VMXNET3_CORE_UNLOCK(sc);
3336                 break;
3337
3338         case SIOCSIFMEDIA:
3339         case SIOCGIFMEDIA:
3340                 error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd);
3341                 break;
3342
3343         case SIOCSIFCAP:
3344                 VMXNET3_CORE_LOCK(sc);
3345                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3346
3347                 if (mask & IFCAP_TXCSUM)
3348                         ifp->if_capenable ^= IFCAP_TXCSUM;
3349                 if (mask & IFCAP_TXCSUM_IPV6)
3350                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
3351 #if 0 /* XXX TSO */
3352                 if (mask & IFCAP_TSO4)
3353                         ifp->if_capenable ^= IFCAP_TSO4;
3354                 if (mask & IFCAP_TSO6)
3355                         ifp->if_capenable ^= IFCAP_TSO6;
3356 #endif
3357
3358                 if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | /* IFCAP_LRO | */
3359                     IFCAP_VLAN_HWTAGGING /* | IFCAP_VLAN_HWFILTER */)) {
3360                         /* Changing these features requires us to reinit. */
3361                         reinit = 1;
3362
3363                         if (mask & IFCAP_RXCSUM)
3364                                 ifp->if_capenable ^= IFCAP_RXCSUM;
3365                         if (mask & IFCAP_RXCSUM_IPV6)
3366                                 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
3367 #if 0 /* XXX LRO */
3368                         if (mask & IFCAP_LRO)
3369                                 ifp->if_capenable ^= IFCAP_LRO;
3370 #endif
3371                         if (mask & IFCAP_VLAN_HWTAGGING)
3372                                 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
3373 #if 0 /* XXX VLAN_HWFILTER */
3374                         if (mask & IFCAP_VLAN_HWFILTER)
3375                                 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
3376 #endif
3377                 } else
3378                         reinit = 0;
3379
3380 #if 0 /* XXX TSO */
3381                 if (mask & IFCAP_VLAN_HWTSO)
3382                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
3383 #endif
3384
3385                 if (reinit && (ifp->if_flags & IFF_RUNNING)) {
3386                         ifp->if_flags &= ~IFF_RUNNING;
3387                         vmxnet3_init_locked(sc);
3388                 } else {
3389                         vmxnet3_init_hwassist(sc);
3390                 }
3391
3392                 VMXNET3_CORE_UNLOCK(sc);
3393 #if 0 /* XXX */
3394                 VLAN_CAPABILITIES(ifp);
3395 #endif
3396                 break;
3397
3398         default:
3399                 error = ether_ioctl(ifp, cmd, data);
3400                 break;
3401         }
3402
3403         VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(sc);
3404
3405         return (error);
3406 }
3407
3408 #ifndef VMXNET3_LEGACY_TX
3409 static void
3410 vmxnet3_qflush(struct ifnet *ifp)
3411 {
3412         struct vmxnet3_softc *sc;
3413         struct vmxnet3_txqueue *txq;
3414         struct mbuf *m;
3415         int i;
3416
3417         sc = ifp->if_softc;
3418
3419         for (i = 0; i < sc->vmx_ntxqueues; i++) {
3420                 txq = &sc->vmx_txq[i];
3421
3422                 VMXNET3_TXQ_LOCK(txq);
3423                 while ((m = buf_ring_dequeue_sc(txq->vxtxq_br)) != NULL)
3424                         m_freem(m);
3425                 VMXNET3_TXQ_UNLOCK(txq);
3426         }
3427
3428         if_qflush(ifp);
3429 }
3430 #endif
3431
3432 static int
3433 vmxnet3_watchdog(struct vmxnet3_txqueue *txq)
3434 {
3435         struct vmxnet3_softc *sc;
3436
3437         sc = txq->vxtxq_sc;
3438
3439         VMXNET3_TXQ_LOCK(txq);
3440         if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) {
3441                 VMXNET3_TXQ_UNLOCK(txq);
3442                 return (0);
3443         }
3444         VMXNET3_TXQ_UNLOCK(txq);
3445
3446         if_printf(sc->vmx_ifp, "watchdog timeout on queue %d\n",
3447             txq->vxtxq_id);
3448         return (1);
3449 }
3450
3451 static void
3452 vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc)
3453 {
3454
3455         vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS);
3456 }
3457
3458 static void
3459 vmxnet3_txq_accum_stats(struct vmxnet3_txqueue *txq,
3460     struct vmxnet3_txq_stats *accum)
3461 {
3462         struct vmxnet3_txq_stats *st;
3463
3464         st = &txq->vxtxq_stats;
3465
3466         accum->vmtxs_opackets += st->vmtxs_opackets;
3467         accum->vmtxs_obytes += st->vmtxs_obytes;
3468         accum->vmtxs_omcasts += st->vmtxs_omcasts;
3469         accum->vmtxs_csum += st->vmtxs_csum;
3470         accum->vmtxs_tso += st->vmtxs_tso;
3471         accum->vmtxs_full += st->vmtxs_full;
3472         accum->vmtxs_offload_failed += st->vmtxs_offload_failed;
3473 }
3474
3475 static void
3476 vmxnet3_rxq_accum_stats(struct vmxnet3_rxqueue *rxq,
3477     struct vmxnet3_rxq_stats *accum)
3478 {
3479         struct vmxnet3_rxq_stats *st;
3480
3481         st = &rxq->vxrxq_stats;
3482
3483         accum->vmrxs_ipackets += st->vmrxs_ipackets;
3484         accum->vmrxs_ibytes += st->vmrxs_ibytes;
3485         accum->vmrxs_iqdrops += st->vmrxs_iqdrops;
3486         accum->vmrxs_ierrors += st->vmrxs_ierrors;
3487 }
3488
3489 static void
3490 vmxnet3_accumulate_stats(struct vmxnet3_softc *sc)
3491 {
3492         struct ifnet *ifp;
3493         struct vmxnet3_statistics *st;
3494         struct vmxnet3_txq_stats txaccum;
3495         struct vmxnet3_rxq_stats rxaccum;
3496         int i;
3497
3498         ifp = sc->vmx_ifp;
3499         st = &sc->vmx_stats;
3500
3501         bzero(&txaccum, sizeof(struct vmxnet3_txq_stats));
3502         bzero(&rxaccum, sizeof(struct vmxnet3_rxq_stats));
3503
3504         for (i = 0; i < sc->vmx_ntxqueues; i++)
3505                 vmxnet3_txq_accum_stats(&sc->vmx_txq[i], &txaccum);
3506         for (i = 0; i < sc->vmx_nrxqueues; i++)
3507                 vmxnet3_rxq_accum_stats(&sc->vmx_rxq[i], &rxaccum);
3508
3509         /*
3510          * With the exception of if_ierrors, these ifnet statistics are
3511          * only updated in the driver, so just set them to our accumulated
3512          * values. if_ierrors is updated in ether_input() for malformed
3513          * frames that we should have already discarded.
3514          */
3515         ifp->if_ipackets = rxaccum.vmrxs_ipackets;
3516         ifp->if_iqdrops = rxaccum.vmrxs_iqdrops;
3517         ifp->if_ierrors = rxaccum.vmrxs_ierrors;
3518         ifp->if_opackets = txaccum.vmtxs_opackets;
3519 #ifndef VMXNET3_LEGACY_TX
3520         ifp->if_obytes = txaccum.vmtxs_obytes;
3521         ifp->if_omcasts = txaccum.vmtxs_omcasts;
3522 #endif
3523 }
3524
3525 static void
3526 vmxnet3_tick(void *xsc)
3527 {
3528         struct vmxnet3_softc *sc;
3529         struct ifnet *ifp;
3530         int i, timedout;
3531
3532         sc = xsc;
3533         ifp = sc->vmx_ifp;
3534         timedout = 0;
3535
3536         VMXNET3_CORE_LOCK_ASSERT(sc);
3537
3538         vmxnet3_accumulate_stats(sc);
3539         vmxnet3_refresh_host_stats(sc);
3540
3541         for (i = 0; i < sc->vmx_ntxqueues; i++)
3542                 timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]);
3543
3544         if (timedout != 0) {
3545                 ifp->if_flags &= ~IFF_RUNNING;
3546                 vmxnet3_init_locked(sc);
3547         } else
3548                 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc);
3549 }
3550
3551 static int
3552 vmxnet3_link_is_up(struct vmxnet3_softc *sc)
3553 {
3554         uint32_t status;
3555
3556         /* Also update the link speed while here. */
3557         status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK);
3558         sc->vmx_link_speed = status >> 16;
3559         return !!(status & 0x1);
3560 }
3561
3562 static void
3563 vmxnet3_link_status(struct vmxnet3_softc *sc)
3564 {
3565         struct ifnet *ifp;
3566         int link;
3567
3568         ifp = sc->vmx_ifp;
3569         link = vmxnet3_link_is_up(sc);
3570
3571         if (link != 0 && sc->vmx_link_active == 0) {
3572                 sc->vmx_link_active = 1;
3573                 ifp->if_link_state = LINK_STATE_UP;
3574                 if_link_state_change(ifp);
3575         } else if (link == 0 && sc->vmx_link_active != 0) {
3576                 sc->vmx_link_active = 0;
3577                 ifp->if_link_state = LINK_STATE_DOWN;
3578                 if_link_state_change(ifp);
3579         }
3580 }
3581
3582 static void
3583 vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
3584 {
3585         struct vmxnet3_softc *sc;
3586
3587         sc = ifp->if_softc;
3588
3589         ifmr->ifm_active = IFM_ETHER | IFM_AUTO;
3590         ifmr->ifm_status = IFM_AVALID;
3591
3592         VMXNET3_CORE_LOCK(sc);
3593         if (vmxnet3_link_is_up(sc) != 0)
3594                 ifmr->ifm_status |= IFM_ACTIVE;
3595         else
3596                 ifmr->ifm_status |= IFM_NONE;
3597         VMXNET3_CORE_UNLOCK(sc);
3598 }
3599
3600 static int
3601 vmxnet3_media_change(struct ifnet *ifp)
3602 {
3603
3604         /* Ignore. */
3605         return (0);
3606 }
3607
3608 static void
3609 vmxnet3_set_lladdr(struct vmxnet3_softc *sc)
3610 {
3611         uint32_t ml, mh;
3612
3613         ml  = sc->vmx_lladdr[0];
3614         ml |= sc->vmx_lladdr[1] << 8;
3615         ml |= sc->vmx_lladdr[2] << 16;
3616         ml |= sc->vmx_lladdr[3] << 24;
3617         vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml);
3618
3619         mh  = sc->vmx_lladdr[4];
3620         mh |= sc->vmx_lladdr[5] << 8;
3621         vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh);
3622 }
3623
3624 static void
3625 vmxnet3_get_lladdr(struct vmxnet3_softc *sc)
3626 {
3627         uint32_t ml, mh;
3628
3629         ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL);
3630         mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH);
3631
3632         sc->vmx_lladdr[0] = ml;
3633         sc->vmx_lladdr[1] = ml >> 8;
3634         sc->vmx_lladdr[2] = ml >> 16;
3635         sc->vmx_lladdr[3] = ml >> 24;
3636         sc->vmx_lladdr[4] = mh;
3637         sc->vmx_lladdr[5] = mh >> 8;
3638 }
3639
3640 static void
3641 vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq,
3642     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3643 {
3644         struct sysctl_oid *node, *txsnode;
3645         struct sysctl_oid_list *list, *txslist;
3646         struct vmxnet3_txq_stats *stats;
3647         struct UPT1_TxStats *txstats;
3648         char namebuf[16];
3649
3650         stats = &txq->vxtxq_stats;
3651         txstats = &txq->vxtxq_ts->stats;
3652
3653         ksnprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id);
3654         node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD,
3655             NULL, "Transmit Queue");
3656         txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node);
3657
3658         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD,
3659             &stats->vmtxs_opackets, 0, "Transmit packets");
3660         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD,
3661             &stats->vmtxs_obytes, 0, "Transmit bytes");
3662         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD,
3663             &stats->vmtxs_omcasts, 0, "Transmit multicasts");
3664         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD,
3665             &stats->vmtxs_csum, 0, "Transmit checksum offloaded");
3666         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD,
3667             &stats->vmtxs_tso, 0, "Transmit TCP segmentation offloaded");
3668         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ringfull", CTLFLAG_RD,
3669             &stats->vmtxs_full, 0, "Transmit ring full");
3670         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "offload_failed", CTLFLAG_RD,
3671             &stats->vmtxs_offload_failed, 0, "Transmit checksum offload failed");
3672
3673         /*
3674          * Add statistics reported by the host. These are updated once
3675          * per second.
3676          */
3677         txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD,
3678             NULL, "Host Statistics");
3679         txslist = SYSCTL_CHILDREN(txsnode);
3680 #if 0 /* XXX TSO */
3681         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD,
3682             &txstats->TSO_packets, 0, "TSO packets");
3683         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD,
3684             &txstats->TSO_bytes, 0, "TSO bytes");
3685 #endif
3686         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD,
3687             &txstats->ucast_packets, 0, "Unicast packets");
3688         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD,
3689             &txstats->ucast_bytes, 0, "Unicast bytes");
3690         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD,
3691             &txstats->mcast_packets, 0, "Multicast packets");
3692         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD,
3693             &txstats->mcast_bytes, 0, "Multicast bytes");
3694         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD,
3695             &txstats->error, 0, "Errors");
3696         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD,
3697             &txstats->discard, 0, "Discards");
3698 }
3699
3700 static void
3701 vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq,
3702     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3703 {
3704         struct sysctl_oid *node, *rxsnode;
3705         struct sysctl_oid_list *list, *rxslist;
3706         struct vmxnet3_rxq_stats *stats;
3707         struct UPT1_RxStats *rxstats;
3708         char namebuf[16];
3709
3710         stats = &rxq->vxrxq_stats;
3711         rxstats = &rxq->vxrxq_rs->stats;
3712
3713         ksnprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id);
3714         node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD,
3715             NULL, "Receive Queue");
3716         rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node);
3717
3718         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD,
3719             &stats->vmrxs_ipackets, 0, "Receive packets");
3720         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD,
3721             &stats->vmrxs_ibytes, 0, "Receive bytes");
3722         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD,
3723             &stats->vmrxs_iqdrops, 0, "Receive drops");
3724         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD,
3725             &stats->vmrxs_ierrors, 0, "Receive errors");
3726
3727         /*
3728          * Add statistics reported by the host. These are updated once
3729          * per second.
3730          */
3731         rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD,
3732             NULL, "Host Statistics");
3733         rxslist = SYSCTL_CHILDREN(rxsnode);
3734 #if 0 /* XXX LRO */
3735         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD,
3736             &rxstats->LRO_packets, 0, "LRO packets");
3737         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD,
3738             &rxstats->LRO_bytes, 0, "LRO bytes");
3739 #endif
3740         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD,
3741             &rxstats->ucast_packets, 0, "Unicast packets");
3742         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD,
3743             &rxstats->ucast_bytes, 0, "Unicast bytes");
3744         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD,
3745             &rxstats->mcast_packets, 0, "Multicast packets");
3746         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD,
3747             &rxstats->mcast_bytes, 0, "Multicast bytes");
3748         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD,
3749             &rxstats->bcast_packets, 0, "Broadcast packets");
3750         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD,
3751             &rxstats->bcast_bytes, 0, "Broadcast bytes");
3752         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD,
3753             &rxstats->nobuffer, 0, "No buffer");
3754         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD,
3755             &rxstats->error, 0, "Errors");
3756 }
3757
3758 static void
3759 vmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc,
3760     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3761 {
3762         struct sysctl_oid *node;
3763         struct sysctl_oid_list *list;
3764         int i;
3765
3766         for (i = 0; i < sc->vmx_ntxqueues; i++) {
3767                 struct vmxnet3_txqueue *txq = &sc->vmx_txq[i];
3768
3769                 node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO,
3770                     "debug", CTLFLAG_RD, NULL, "");
3771                 list = SYSCTL_CHILDREN(node);
3772
3773                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_head", CTLFLAG_RD,
3774                     &txq->vxtxq_cmd_ring.vxtxr_head, 0, "");
3775                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD,
3776                     &txq->vxtxq_cmd_ring.vxtxr_next, 0, "");
3777                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD,
3778                     &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, "");
3779                 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD,
3780                     &txq->vxtxq_cmd_ring.vxtxr_gen, 0, "");
3781                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD,
3782                     &txq->vxtxq_comp_ring.vxcr_next, 0, "");
3783                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD,
3784                     &txq->vxtxq_comp_ring.vxcr_ndesc, 0,"");
3785                 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD,
3786                     &txq->vxtxq_comp_ring.vxcr_gen, 0, "");
3787         }
3788
3789         for (i = 0; i < sc->vmx_nrxqueues; i++) {
3790                 struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i];
3791
3792                 node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO,
3793                     "debug", CTLFLAG_RD, NULL, "");
3794                 list = SYSCTL_CHILDREN(node);
3795
3796                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_fill", CTLFLAG_RD,
3797                     &rxq->vxrxq_cmd_ring[0].vxrxr_fill, 0, "");
3798                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD,
3799                     &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, "");
3800                 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD,
3801                     &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, "");
3802                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_fill", CTLFLAG_RD,
3803                     &rxq->vxrxq_cmd_ring[1].vxrxr_fill, 0, "");
3804                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD,
3805                     &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, "");
3806                 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD,
3807                     &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, "");
3808                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD,
3809                     &rxq->vxrxq_comp_ring.vxcr_next, 0, "");
3810                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD,
3811                     &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,"");
3812                 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD,
3813                     &rxq->vxrxq_comp_ring.vxcr_gen, 0, "");
3814         }
3815 }
3816
3817 static void
3818 vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc,
3819     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3820 {
3821         int i;
3822
3823         for (i = 0; i < sc->vmx_ntxqueues; i++)
3824                 vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child);
3825         for (i = 0; i < sc->vmx_nrxqueues; i++)
3826                 vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child);
3827
3828         vmxnet3_setup_debug_sysctl(sc, ctx, child);
3829 }
3830
3831 static void
3832 vmxnet3_setup_sysctl(struct vmxnet3_softc *sc)
3833 {
3834         device_t dev;
3835         struct vmxnet3_statistics *stats;
3836         struct sysctl_ctx_list *ctx;
3837         struct sysctl_oid *tree;
3838         struct sysctl_oid_list *child;
3839
3840         dev = sc->vmx_dev;
3841         ctx = device_get_sysctl_ctx(dev);
3842         tree = device_get_sysctl_tree(dev);
3843         child = SYSCTL_CHILDREN(tree);
3844
3845         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_ntxqueues", CTLFLAG_RD,
3846             &sc->vmx_max_ntxqueues, 0, "Maximum number of Tx queues");
3847         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_nrxqueues", CTLFLAG_RD,
3848             &sc->vmx_max_nrxqueues, 0, "Maximum number of Rx queues");
3849         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "ntxqueues", CTLFLAG_RD,
3850             &sc->vmx_ntxqueues, 0, "Number of Tx queues");
3851         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nrxqueues", CTLFLAG_RD,
3852             &sc->vmx_nrxqueues, 0, "Number of Rx queues");
3853
3854         stats = &sc->vmx_stats;
3855         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defragged", CTLFLAG_RD,
3856             &stats->vmst_defragged, 0, "Tx mbuf chains defragged");
3857         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defrag_failed", CTLFLAG_RD,
3858             &stats->vmst_defrag_failed, 0,
3859             "Tx mbuf dropped because defrag failed");
3860         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mgetcl_failed", CTLFLAG_RD,
3861             &stats->vmst_mgetcl_failed, 0, "mbuf cluster allocation failed");
3862         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mbuf_load_failed", CTLFLAG_RD,
3863             &stats->vmst_mbuf_load_failed, 0, "mbuf load segments failed");
3864
3865         vmxnet3_setup_queue_sysctl(sc, ctx, child);
3866 }
3867
3868 static void
3869 vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
3870 {
3871
3872         bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v);
3873 }
3874
3875 static uint32_t
3876 vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r)
3877 {
3878
3879         return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r));
3880 }
3881
3882 static void
3883 vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
3884 {
3885
3886         bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v);
3887 }
3888
3889 static void
3890 vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
3891 {
3892
3893         vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd);
3894 }
3895
3896 static uint32_t
3897 vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
3898 {
3899
3900         vmxnet3_write_cmd(sc, cmd);
3901         bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0,
3902             BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
3903         return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD));
3904 }
3905
3906 static void
3907 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq)
3908 {
3909
3910         vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0);
3911 }
3912
3913 static void
3914 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq)
3915 {
3916
3917         vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1);
3918 }
3919
3920 static void
3921 vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc)
3922 {
3923         int i;
3924
3925         sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL;
3926         for (i = 0; i < sc->vmx_nintrs; i++)
3927                 vmxnet3_enable_intr(sc, i);
3928 }
3929
3930 static void
3931 vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc)
3932 {
3933         int i;
3934
3935         sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL;
3936         for (i = 0; i < sc->vmx_nintrs; i++)
3937                 vmxnet3_disable_intr(sc, i);
3938 }
3939
3940 static void
3941 vmxnet3_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3942 {
3943         bus_addr_t *baddr = arg;
3944
3945         if (error == 0)
3946                 *baddr = segs->ds_addr;
3947 }
3948
3949 static int
3950 vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align,
3951     struct vmxnet3_dma_alloc *dma)
3952 {
3953         device_t dev;
3954         int error;
3955
3956         dev = sc->vmx_dev;
3957         bzero(dma, sizeof(struct vmxnet3_dma_alloc));
3958
3959         error = bus_dma_tag_create(bus_get_dma_tag(dev),
3960             align, 0,           /* alignment, bounds */
3961             BUS_SPACE_MAXADDR,  /* lowaddr */
3962             BUS_SPACE_MAXADDR,  /* highaddr */
3963             NULL, NULL,         /* filter, filterarg */
3964             size,               /* maxsize */
3965             1,                  /* nsegments */
3966             size,               /* maxsegsize */
3967             BUS_DMA_ALLOCNOW,   /* flags */
3968             &dma->dma_tag);
3969         if (error) {
3970                 device_printf(dev, "bus_dma_tag_create failed: %d\n", error);
3971                 goto fail;
3972         }
3973
3974         error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
3975             BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map);
3976         if (error) {
3977                 device_printf(dev, "bus_dmamem_alloc failed: %d\n", error);
3978                 goto fail;
3979         }
3980
3981         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3982             size, vmxnet3_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT);
3983         if (error) {
3984                 device_printf(dev, "bus_dmamap_load failed: %d\n", error);
3985                 goto fail;
3986         }
3987
3988         dma->dma_size = size;
3989
3990 fail:
3991         if (error)
3992                 vmxnet3_dma_free(sc, dma);
3993
3994         return (error);
3995 }
3996
3997 static void
3998 vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma)
3999 {
4000
4001         if (dma->dma_tag != NULL) {
4002                 if (dma->dma_paddr != 0) {
4003                         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
4004                             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4005                         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
4006                 }
4007
4008                 if (dma->dma_vaddr != NULL) {
4009                         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr,
4010                             dma->dma_map);
4011                 }
4012
4013                 bus_dma_tag_destroy(dma->dma_tag);
4014         }
4015         bzero(dma, sizeof(struct vmxnet3_dma_alloc));
4016 }
4017
4018 static int
4019 vmxnet3_tunable_int(struct vmxnet3_softc *sc, const char *knob, int def)
4020 {
4021         char path[64];
4022
4023         ksnprintf(path, sizeof(path),
4024             "hw.vmx.%d.%s", device_get_unit(sc->vmx_dev), knob);
4025         TUNABLE_INT_FETCH(path, &def);
4026
4027         return (def);
4028 }
4029
4030 #define mb()    __asm volatile("mfence" ::: "memory")
4031 #define wmb()   __asm volatile("sfence" ::: "memory")
4032 #define rmb()   __asm volatile("lfence" ::: "memory")
4033
4034 /*
4035  * Since this is a purely paravirtualized device, we do not have
4036  * to worry about DMA coherency. But at times, we must make sure
4037  * both the compiler and CPU do not reorder memory operations.
4038  */
4039 static inline void
4040 vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type)
4041 {
4042
4043         switch (type) {
4044         case VMXNET3_BARRIER_RD:
4045                 rmb();
4046                 break;
4047         case VMXNET3_BARRIER_WR:
4048                 wmb();
4049                 break;
4050         case VMXNET3_BARRIER_RDWR:
4051                 mb();
4052                 break;
4053         default:
4054                 panic("%s: bad barrier type %d", __func__, type);
4055         }
4056 }