sys/dev/netif/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2012, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
  34
  35 #include "opt_inet.h"
  36 #include "opt_inet6.h"
  37
  38 #include "ixgbe.h"
  39
  40 /*********************************************************************
  41  *  Set this to one to display debug statistics
  42  *********************************************************************/
  43 int             ixgbe_display_debug_stats = 0;
  44
  45 /*********************************************************************
  46  *  Driver version
  47  *********************************************************************/
  48 char ixgbe_driver_version[] = "2.4.8";
  49
  50 /*********************************************************************
  51  *  PCI Device ID Table
  52  *
  53  *  Used by probe to select devices to load on
  54  *  Last field stores an index into ixgbe_strings
  55  *  Last entry must be all 0s
  56  *
  57  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  58  *********************************************************************/
  59
  60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  61 {
  62         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
  81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
  82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
  83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
  84         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
  85         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
  86         /* required last entry */
  87         {0, 0, 0, 0, 0}
  88 };
  89
  90 /*********************************************************************
  91  *  Table of branding strings
  92  *********************************************************************/
  93
  94 static char    *ixgbe_strings[] = {
  95         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  96 };
  97
  98 /*********************************************************************
  99  *  Function prototypes
 100  *********************************************************************/
 101 static int      ixgbe_probe(device_t);
 102 static int      ixgbe_attach(device_t);
 103 static int      ixgbe_detach(device_t);
 104 static int      ixgbe_shutdown(device_t);
 105 static void     ixgbe_start(struct ifnet *);
 106 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
 107 #if 0 /* __FreeBSD_version >= 800000 */
 108 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
 109 static int      ixgbe_mq_start_locked(struct ifnet *,
 110                     struct tx_ring *, struct mbuf *);
 111 static void     ixgbe_qflush(struct ifnet *);
 112 #endif
 113 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 114 static void     ixgbe_init(void *);
 115 static void     ixgbe_init_locked(struct adapter *);
 116 static void     ixgbe_stop(void *);
 117 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 118 static int      ixgbe_media_change(struct ifnet *);
 119 static void     ixgbe_identify_hardware(struct adapter *);
 120 static int      ixgbe_allocate_pci_resources(struct adapter *);
 121 static int      ixgbe_allocate_msix(struct adapter *);
 122 static int      ixgbe_allocate_legacy(struct adapter *);
 123 static int      ixgbe_allocate_queues(struct adapter *);
 124 static int      ixgbe_setup_msix(struct adapter *);
 125 static void     ixgbe_free_pci_resources(struct adapter *);
 126 static void     ixgbe_local_timer(void *);
 127 static int      ixgbe_setup_interface(device_t, struct adapter *);
 128 static void     ixgbe_config_link(struct adapter *);
 129
 130 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 131 static int      ixgbe_setup_transmit_structures(struct adapter *);
 132 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 133 static void     ixgbe_initialize_transmit_units(struct adapter *);
 134 static void     ixgbe_free_transmit_structures(struct adapter *);
 135 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 136
 137 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 138 static int      ixgbe_setup_receive_structures(struct adapter *);
 139 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 140 static void     ixgbe_initialize_receive_units(struct adapter *);
 141 static void     ixgbe_free_receive_structures(struct adapter *);
 142 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 143 #if 0   /* NET_LRO */
 144 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
 145 #endif
 146
 147 static void     ixgbe_enable_intr(struct adapter *);
 148 static void     ixgbe_disable_intr(struct adapter *);
 149 static void     ixgbe_update_stats_counters(struct adapter *);
 150 static void     ixgbe_txeof(struct tx_ring *);
 151 static void     ixgbe_rxeof(struct ix_queue *, int);
 152 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
 153 static void     ixgbe_set_promisc(struct adapter *);
 154 static void     ixgbe_set_multi(struct adapter *);
 155 static void     ixgbe_update_link_status(struct adapter *);
 156 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
 157 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 158 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 159 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
 160 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
 161 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 162                     struct ixgbe_dma_alloc *, int);
 163 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 164 static void     ixgbe_add_rx_process_limit(struct adapter *, const char *,
 165                     const char *, int *, int);
 166 static bool     ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 167 static bool     ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
 168 static int      ixgbe_tso_pullup(struct tx_ring *, struct mbuf **);
 169 static void     ixgbe_add_sysctl(struct adapter *);
 170 static void     ixgbe_set_eitr(struct adapter *, int, int);
 171 static int      ixgbe_sysctl_intr_rate(SYSCTL_HANDLER_ARGS);
 172 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 173 static void     ixgbe_configure_ivars(struct adapter *);
 174 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 175
 176 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 177 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 178 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 179
 180 static void     ixgbe_add_hw_stats(struct adapter *adapter);
 181
 182 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
 183 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
 184                     struct mbuf *, u32);
 185
 186 /* Support for pluggable optic modules */
 187 static bool     ixgbe_sfp_probe(struct adapter *);
 188 static void     ixgbe_setup_optics(struct adapter *);
 189
 190 /* Legacy (single vector interrupt handler */
 191 static void     ixgbe_legacy_irq(void *);
 192
 193 /* The MSI/X Interrupt handlers */
 194 static void     ixgbe_msix_que(void *);
 195 static void     ixgbe_msix_link(void *);
 196
 197 /* Deferred interrupt tasklets */
 198 static void     ixgbe_handle_msf(void *, int);
 199 static void     ixgbe_handle_mod(void *, int);
 200
 201 #ifdef IXGBE_FDIR
 202 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
 203 static void     ixgbe_reinit_fdir(void *, int);
 204 #endif
 205
 206 /*********************************************************************
 207  *  FreeBSD Device Interface Entry Points
 208  *********************************************************************/
 209
 210 static device_method_t ixgbe_methods[] = {
 211         /* Device interface */
 212         DEVMETHOD(device_probe, ixgbe_probe),
 213         DEVMETHOD(device_attach, ixgbe_attach),
 214         DEVMETHOD(device_detach, ixgbe_detach),
 215         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 216         {0, 0}
 217 };
 218
 219 static driver_t ixgbe_driver = {
 220         "ix", ixgbe_methods, sizeof(struct adapter),
 221 };
 222
 223 devclass_t ixgbe_devclass;
 224 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
 225
 226 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 227 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 228
 229 /*
 230 ** TUNEABLE PARAMETERS:
 231 */
 232
 233 /* How many packets rxeof tries to clean at a time */
 234 static int ixgbe_rx_process_limit = 128;
 235 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 236
 237 /*
 238 ** Smart speed setting, default to on
 239 ** this only works as a compile option
 240 ** right now as its during attach, set
 241 ** this to 'ixgbe_smart_speed_off' to
 242 ** disable.
 243 */
 244 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 245
 246 static int ixgbe_msi_enable = 1;
 247 TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
 248
 249 /*
 250  * MSIX should be the default for best performance,
 251  * but this allows it to be forced off for testing.
 252  */
 253 static int ixgbe_enable_msix = 1;
 254 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 255
 256 /*
 257  * Header split: this causes the hardware to DMA
 258  * the header into a separate mbuf from the payload,
 259  * it can be a performance win in some workloads, but
 260  * in others it actually hurts, its off by default.
 261  */
 262 static int ixgbe_header_split = FALSE;
 263 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
 264
 265 /*
 266  * Number of Queues, can be set to 0,
 267  * it then autoconfigures based on the
 268  * number of cpus with a max of 8. This
 269  * can be overriden manually here.
 270  */
 271 static int ixgbe_num_queues = 0;
 272 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
 273
 274 /*
 275 ** Number of TX descriptors per ring,
 276 ** setting higher than RX as this seems
 277 ** the better performing choice.
 278 */
 279 static int ixgbe_txd = PERFORM_TXD;
 280 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 281
 282 /* Number of RX descriptors per ring */
 283 static int ixgbe_rxd = PERFORM_RXD;
 284 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 285
 286 /* Keep running tab on them for sanity check */
 287 static int ixgbe_total_ports;
 288
 289 #ifdef IXGBE_FDIR
 290 /*
 291 ** For Flow Director: this is the
 292 ** number of TX packets we sample
 293 ** for the filter pool, this means
 294 ** every 20th packet will be probed.
 295 **
 296 ** This feature can be disabled by
 297 ** setting this to 0.
 298 */
 299 static int atr_sample_rate = 20;
 300 /*
 301 ** Flow Director actually 'steals'
 302 ** part of the packet buffer as its
 303 ** filter pool, this variable controls
 304 ** how much it uses:
 305 **  0 = 64K, 1 = 128K, 2 = 256K
 306 */
 307 static int fdir_pballoc = 1;
 308 #endif
 309
 310 #ifdef DEV_NETMAP
 311 /*
 312  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
 313  * be a reference on how to implement netmap support in a driver.
 314  * Additional comments are in ixgbe_netmap.h .
 315  *
 316  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
 317  * that extend the standard driver.
 318  */
 319 #include <dev/netmap/ixgbe_netmap.h>
 320 #endif /* DEV_NETMAP */
 321
 322 /*********************************************************************
 323  *  Device identification routine
 324  *
 325  *  ixgbe_probe determines if the driver should be loaded on
 326  *  adapter based on PCI vendor/device id of the adapter.
 327  *
 328  *  return BUS_PROBE_DEFAULT on success, positive on failure
 329  *********************************************************************/
 330
 331 static int
 332 ixgbe_probe(device_t dev)
 333 {
 334         ixgbe_vendor_info_t *ent;
 335
 336         u16     pci_vendor_id = 0;
 337         u16     pci_device_id = 0;
 338         u16     pci_subvendor_id = 0;
 339         u16     pci_subdevice_id = 0;
 340         char    adapter_name[256];
 341
 342         INIT_DEBUGOUT("ixgbe_probe: begin");
 343
 344         pci_vendor_id = pci_get_vendor(dev);
 345         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 346                 return (ENXIO);
 347
 348         pci_device_id = pci_get_device(dev);
 349         pci_subvendor_id = pci_get_subvendor(dev);
 350         pci_subdevice_id = pci_get_subdevice(dev);
 351
 352         ent = ixgbe_vendor_info_array;
 353         while (ent->vendor_id != 0) {
 354                 if ((pci_vendor_id == ent->vendor_id) &&
 355                     (pci_device_id == ent->device_id) &&
 356
 357                     ((pci_subvendor_id == ent->subvendor_id) ||
 358                      (ent->subvendor_id == 0)) &&
 359
 360                     ((pci_subdevice_id == ent->subdevice_id) ||
 361                      (ent->subdevice_id == 0))) {
 362                         ksprintf(adapter_name, "%s, Version - %s",
 363                                 ixgbe_strings[ent->index],
 364                                 ixgbe_driver_version);
 365                         device_set_desc_copy(dev, adapter_name);
 366                         ++ixgbe_total_ports;
 367                         return (BUS_PROBE_DEFAULT);
 368                 }
 369                 ent++;
 370         }
 371         return (ENXIO);
 372 }
 373
 374 /*********************************************************************
 375  *  Device initialization routine
 376  *
 377  *  The attach entry point is called when the driver is being loaded.
 378  *  This routine identifies the type of hardware, allocates all resources
 379  *  and initializes the hardware.
 380  *
 381  *  return 0 on success, positive on failure
 382  *********************************************************************/
 383
 384 static int
 385 ixgbe_attach(device_t dev)
 386 {
 387         struct adapter *adapter;
 388         struct ixgbe_hw *hw;
 389         int             error = 0;
 390         u16             csum;
 391         u32             ctrl_ext;
 392
 393         INIT_DEBUGOUT("ixgbe_attach: begin");
 394
 395         if (resource_disabled("ixgbe", device_get_unit(dev))) {
 396                 device_printf(dev, "Disabled by device hint\n");
 397                 return (ENXIO);
 398         }
 399
 400         /* Allocate, clear, and link in our adapter structure */
 401         adapter = device_get_softc(dev);
 402         adapter->dev = adapter->osdep.dev = dev;
 403         hw = &adapter->hw;
 404
 405         /* Core Lock Init*/
 406         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 407
 408         /* Set up the timer callout */
 409         callout_init_mp(&adapter->timer);
 410
 411         /* Determine hardware revision */
 412         ixgbe_identify_hardware(adapter);
 413
 414         /* Enable bus mastering */
 415         pci_enable_busmaster(dev);
 416
 417         /* Do base PCI setup - map BAR0 */
 418         if (ixgbe_allocate_pci_resources(adapter)) {
 419                 device_printf(dev, "Allocation of PCI resources failed\n");
 420                 error = ENXIO;
 421                 goto err_out;
 422         }
 423
 424         /* Do descriptor calc and sanity checks */
 425         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 426             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 427                 device_printf(dev, "TXD config issue, using default!\n");
 428                 adapter->num_tx_desc = DEFAULT_TXD;
 429         } else
 430                 adapter->num_tx_desc = ixgbe_txd;
 431
 432         /*
 433         ** With many RX rings it is easy to exceed the
 434         ** system mbuf allocation. Tuning nmbclusters
 435         ** can alleviate this.
 436         */
 437         if (nmbclusters > 0 ) {
 438                 int s;
 439                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 440                 if (s > nmbclusters) {
 441                         device_printf(dev, "RX Descriptors exceed "
 442                             "system mbuf max, using default instead!\n");
 443                         ixgbe_rxd = DEFAULT_RXD;
 444                 }
 445         }
 446
 447         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 448             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 449                 device_printf(dev, "RXD config issue, using default!\n");
 450                 adapter->num_rx_desc = DEFAULT_RXD;
 451         } else
 452                 adapter->num_rx_desc = ixgbe_rxd;
 453
 454         /* Allocate our TX/RX Queues */
 455         if (ixgbe_allocate_queues(adapter)) {
 456                 error = ENOMEM;
 457                 goto err_out;
 458         }
 459
 460         /* Allocate multicast array memory. */
 461         adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
 462             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 463         if (adapter->mta == NULL) {
 464                 device_printf(dev, "Can not allocate multicast setup array\n");
 465                 error = ENOMEM;
 466                 goto err_late;
 467         }
 468
 469         /* Initialize the shared code */
 470         error = ixgbe_init_shared_code(hw);
 471         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 472                 /*
 473                 ** No optics in this port, set up
 474                 ** so the timer routine will probe
 475                 ** for later insertion.
 476                 */
 477                 adapter->sfp_probe = TRUE;
 478                 error = 0;
 479         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 480                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 481                 error = EIO;
 482                 goto err_late;
 483         } else if (error) {
 484                 device_printf(dev,"Unable to initialize the shared code\n");
 485                 error = EIO;
 486                 goto err_late;
 487         }
 488
 489         /* Make sure we have a good EEPROM before we read from it */
 490         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 491                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 492                 error = EIO;
 493                 goto err_late;
 494         }
 495
 496         error = ixgbe_init_hw(hw);
 497         switch (error) {
 498         case IXGBE_ERR_EEPROM_VERSION:
 499                 device_printf(dev, "This device is a pre-production adapter/"
 500                     "LOM.  Please be aware there may be issues associated "
 501                     "with your hardware.\n If you are experiencing problems "
 502                     "please contact your Intel or hardware representative "
 503                     "who provided you with this hardware.\n");
 504                 break;
 505         case IXGBE_ERR_SFP_NOT_SUPPORTED:
 506                 device_printf(dev,"Unsupported SFP+ Module\n");
 507                 error = EIO;
 508                 device_printf(dev,"Hardware Initialization Failure\n");
 509                 goto err_late;
 510         case IXGBE_ERR_SFP_NOT_PRESENT:
 511                 device_printf(dev,"No SFP+ Module found\n");
 512                 /* falls thru */
 513         default:
 514                 break;
 515         }
 516
 517         /* Detect and set physical type */
 518         ixgbe_setup_optics(adapter);
 519
 520         if ((adapter->msix > 1) && (ixgbe_enable_msix)) {
 521                 adapter->intr_type = PCI_INTR_TYPE_MSIX;
 522                 error = ixgbe_allocate_msix(adapter);
 523         } else {
 524                 error = ixgbe_allocate_legacy(adapter);
 525         }
 526         if (error)
 527                 goto err_late;
 528
 529         /* Setup OS specific network interface */
 530         if (ixgbe_setup_interface(dev, adapter) != 0)
 531                 goto err_late;
 532
 533         /* Add sysctl tree */
 534         ixgbe_add_sysctl(adapter);
 535
 536         /* Initialize statistics */
 537         ixgbe_update_stats_counters(adapter);
 538
 539         /* Register for VLAN events */
 540         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 541             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 542         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 543             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 544
 545         /* Print PCIE bus type/speed/width info */
 546         ixgbe_get_bus_info(hw);
 547         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
 548             ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
 549             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
 550             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
 551             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
 552             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
 553             ("Unknown"));
 554
 555         if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
 556             (hw->bus.speed == ixgbe_bus_speed_2500)) {
 557                 device_printf(dev, "PCI-Express bandwidth available"
 558                     " for this card\n     is not sufficient for"
 559                     " optimal performance.\n");
 560                 device_printf(dev, "For optimal performance a x8 "
 561                     "PCIE, or x4 PCIE 2 slot is required.\n");
 562         }
 563
 564         /* let hardware know driver is loaded */
 565         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 566         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 567         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 568
 569         ixgbe_add_hw_stats(adapter);
 570
 571 #ifdef DEV_NETMAP
 572         ixgbe_netmap_attach(adapter);
 573 #endif /* DEV_NETMAP */
 574         INIT_DEBUGOUT("ixgbe_attach: end");
 575         return (0);
 576 err_late:
 577         ixgbe_free_transmit_structures(adapter);
 578         ixgbe_free_receive_structures(adapter);
 579 err_out:
 580         if (adapter->ifp != NULL)
 581                 if_free(adapter->ifp);
 582         ixgbe_free_pci_resources(adapter);
 583         kfree(adapter->mta, M_DEVBUF);
 584         return (error);
 585
 586 }
 587
 588 /*********************************************************************
 589  *  Device removal routine
 590  *
 591  *  The detach entry point is called when the driver is being removed.
 592  *  This routine stops the adapter and deallocates all the resources
 593  *  that were allocated for driver operation.
 594  *
 595  *  return 0 on success, positive on failure
 596  *********************************************************************/
 597
 598 static int
 599 ixgbe_detach(device_t dev)
 600 {
 601         struct adapter *adapter = device_get_softc(dev);
 602         u32     ctrl_ext;
 603
 604         INIT_DEBUGOUT("ixgbe_detach: begin");
 605
 606         /* Make sure VLANS are not using driver */
 607         if (adapter->ifp->if_vlantrunks != NULL) {
 608                 device_printf(dev,"Vlan in use, detach first\n");
 609                 return (EBUSY);
 610         }
 611
 612         IXGBE_CORE_LOCK(adapter);
 613         ixgbe_stop(adapter);
 614         IXGBE_CORE_UNLOCK(adapter);
 615
 616         /* Drain the Link queue */
 617         if (adapter->tq) {
 618                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 619                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 620 #ifdef IXGBE_FDIR
 621                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 622 #endif
 623                 taskqueue_free(adapter->tq);
 624         }
 625
 626         /* let hardware know driver is unloading */
 627         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 628         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 629         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 630
 631         /* Unregister VLAN events */
 632         if (adapter->vlan_attach != NULL)
 633                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 634         if (adapter->vlan_detach != NULL)
 635                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 636
 637         ether_ifdetach(adapter->ifp);
 638         callout_stop(&adapter->timer);
 639 #ifdef DEV_NETMAP
 640         netmap_detach(adapter->ifp);
 641 #endif /* DEV_NETMAP */
 642         ixgbe_free_pci_resources(adapter);
 643         bus_generic_detach(dev);
 644         if_free(adapter->ifp);
 645
 646         ixgbe_free_transmit_structures(adapter);
 647         ixgbe_free_receive_structures(adapter);
 648         kfree(adapter->mta, M_DEVBUF);
 649         sysctl_ctx_free(&adapter->sysctl_ctx);
 650
 651         IXGBE_CORE_LOCK_DESTROY(adapter);
 652         return (0);
 653 }
 654
 655 /*********************************************************************
 656  *
 657  *  Shutdown entry point
 658  *
 659  **********************************************************************/
 660
 661 static int
 662 ixgbe_shutdown(device_t dev)
 663 {
 664         struct adapter *adapter = device_get_softc(dev);
 665         IXGBE_CORE_LOCK(adapter);
 666         ixgbe_stop(adapter);
 667         IXGBE_CORE_UNLOCK(adapter);
 668         return (0);
 669 }
 670
 671
 672 /*********************************************************************
 673  *  Transmit entry point
 674  *
 675  *  ixgbe_start is called by the stack to initiate a transmit.
 676  *  The driver will remain in this routine as long as there are
 677  *  packets to transmit and transmit resources are available.
 678  *  In case resources are not available stack is notified and
 679  *  the packet is requeued.
 680  **********************************************************************/
 681
 682 static void
 683 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 684 {
 685         struct mbuf    *m_head;
 686         struct adapter *adapter = txr->adapter;
 687
 688         IXGBE_TX_LOCK_ASSERT(txr);
 689
 690         if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
 691                 return;
 692         if (!adapter->link_active)
 693                 return;
 694
 695         while (!ifq_is_empty(&ifp->if_snd)) {
 696                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
 697                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 698                         break;
 699                 }
 700
 701                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
 702                 if (m_head == NULL)
 703                         break;
 704
 705                 if (ixgbe_xmit(txr, &m_head)) {
 706 #if 0 /* XXX: prepend to an ALTQ queue ? */
 707                         if (m_head != NULL)
 708                                 IF_PREPEND(&ifp->if_snd, m_head);
 709 #endif
 710                         if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
 711                                 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 712                         break;
 713                 }
 714                 /* Send a copy of the frame to the BPF listener */
 715                 ETHER_BPF_MTAP(ifp, m_head);
 716
 717                 /* Set watchdog on */
 718                 txr->watchdog_time = ticks;
 719                 txr->queue_status = IXGBE_QUEUE_WORKING;
 720
 721         }
 722         return;
 723 }
 724
 725 /*
 726  * Legacy TX start - called by the stack, this
 727  * always uses the first tx ring, and should
 728  * not be used with multiqueue tx enabled.
 729  */
 730 static void
 731 ixgbe_start(struct ifnet *ifp)
 732 {
 733         struct adapter *adapter = ifp->if_softc;
 734         struct tx_ring  *txr = adapter->tx_rings;
 735
 736         if (ifp->if_flags & IFF_RUNNING) {
 737                 IXGBE_TX_LOCK(txr);
 738                 ixgbe_start_locked(txr, ifp);
 739                 IXGBE_TX_UNLOCK(txr);
 740         }
 741         return;
 742 }
 743
 744 #if 0 /* __FreeBSD_version >= 800000 */
 745 /*
 746 ** Multiqueue Transmit driver
 747 **
 748 */
 749 static int
 750 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
 751 {
 752         struct adapter  *adapter = ifp->if_softc;
 753         struct ix_queue *que;
 754         struct tx_ring  *txr;
 755         int             i = 0, err = 0;
 756
 757         /* Which queue to use */
 758         if ((m->m_flags & M_FLOWID) != 0)
 759                 i = m->m_pkthdr.flowid % adapter->num_queues;
 760         else
 761                 i = curcpu % adapter->num_queues;
 762
 763         txr = &adapter->tx_rings[i];
 764         que = &adapter->queues[i];
 765
 766         if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
 767             IXGBE_TX_TRYLOCK(txr)) {
 768                 err = ixgbe_mq_start_locked(ifp, txr, m);
 769                 IXGBE_TX_UNLOCK(txr);
 770         } else {
 771                 err = drbr_enqueue(ifp, txr->br, m);
 772                 taskqueue_enqueue(que->tq, &que->que_task);
 773         }
 774
 775         return (err);
 776 }
 777
 778 static int
 779 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 780 {
 781         struct adapter  *adapter = txr->adapter;
 782         struct mbuf     *next;
 783         int             enqueued, err = 0;
 784
 785         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
 786             (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
 787             adapter->link_active == 0) {
 788                 if (m != NULL)
 789                         err = drbr_enqueue(ifp, txr->br, m);
 790                 return (err);
 791         }
 792
 793         enqueued = 0;
 794         if (m == NULL) {
 795                 next = drbr_dequeue(ifp, txr->br);
 796         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 797                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 798                         return (err);
 799                 next = drbr_dequeue(ifp, txr->br);
 800         } else
 801                 next = m;
 802
 803         /* Process the queue */
 804         while (next != NULL) {
 805                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 806                         if (next != NULL)
 807                                 err = drbr_enqueue(ifp, txr->br, next);
 808                         break;
 809                 }
 810                 enqueued++;
 811                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 812                 /* Send a copy of the frame to the BPF listener */
 813                 ETHER_BPF_MTAP(ifp, next);
 814                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 815                         break;
 816                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
 817                         ixgbe_txeof(txr);
 818                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
 819                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 820                         break;
 821                 }
 822                 next = drbr_dequeue(ifp, txr->br);
 823         }
 824
 825         if (enqueued > 0) {
 826                 /* Set watchdog on */
 827                 txr->queue_status |= IXGBE_QUEUE_WORKING;
 828                 txr->watchdog_time = ticks;
 829         }
 830
 831         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
 832                 ixgbe_txeof(txr);
 833
 834         return (err);
 835 }
 836
 837 /*
 838 ** Flush all ring buffers
 839 */
 840 static void
 841 ixgbe_qflush(struct ifnet *ifp)
 842 {
 843         struct adapter  *adapter = ifp->if_softc;
 844         struct tx_ring  *txr = adapter->tx_rings;
 845         struct mbuf     *m;
 846
 847         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 848                 IXGBE_TX_LOCK(txr);
 849                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 850                         m_freem(m);
 851                 IXGBE_TX_UNLOCK(txr);
 852         }
 853         if_qflush(ifp);
 854 }
 855 #endif /* __FreeBSD_version >= 800000 */
 856
 857 /*********************************************************************
 858  *  Ioctl entry point
 859  *
 860  *  ixgbe_ioctl is called when the user wants to configure the
 861  *  interface.
 862  *
 863  *  return 0 on success, positive on failure
 864  **********************************************************************/
 865
 866 static int
 867 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
 868 {
 869         struct adapter  *adapter = ifp->if_softc;
 870         struct ifreq    *ifr = (struct ifreq *) data;
 871 #if defined(INET) || defined(INET6)
 872         struct ifaddr *ifa = (struct ifaddr *)data;
 873         bool            avoid_reset = FALSE;
 874 #endif
 875         int             error = 0;
 876
 877         switch (command) {
 878
 879         case SIOCSIFADDR:
 880 #ifdef INET
 881                 if (ifa->ifa_addr->sa_family == AF_INET)
 882                         avoid_reset = TRUE;
 883 #endif
 884 #ifdef INET6
 885                 if (ifa->ifa_addr->sa_family == AF_INET6)
 886                         avoid_reset = TRUE;
 887 #endif
 888 #if defined(INET) || defined(INET6)
 889                 /*
 890                 ** Calling init results in link renegotiation,
 891                 ** so we avoid doing it when possible.
 892                 */
 893                 if (avoid_reset) {
 894                         ifp->if_flags |= IFF_UP;
 895                         if (!(ifp->if_flags & IFF_RUNNING))
 896                                 ixgbe_init(adapter);
 897                         if (!(ifp->if_flags & IFF_NOARP))
 898                                 arp_ifinit(ifp, ifa);
 899                 } else
 900                         error = ether_ioctl(ifp, command, data);
 901 #endif
 902                 break;
 903         case SIOCSIFMTU:
 904                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 905                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 906                         error = EINVAL;
 907                 } else {
 908                         IXGBE_CORE_LOCK(adapter);
 909                         ifp->if_mtu = ifr->ifr_mtu;
 910                         adapter->max_frame_size =
 911                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 912                         ixgbe_init_locked(adapter);
 913                         IXGBE_CORE_UNLOCK(adapter);
 914                 }
 915                 break;
 916         case SIOCSIFFLAGS:
 917                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 918                 IXGBE_CORE_LOCK(adapter);
 919                 if (ifp->if_flags & IFF_UP) {
 920                         if ((ifp->if_flags & IFF_RUNNING)) {
 921                                 if ((ifp->if_flags ^ adapter->if_flags) &
 922                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 923                                         ixgbe_set_promisc(adapter);
 924                                 }
 925                         } else
 926                                 ixgbe_init_locked(adapter);
 927                 } else
 928                         if (ifp->if_flags & IFF_RUNNING)
 929                                 ixgbe_stop(adapter);
 930                 adapter->if_flags = ifp->if_flags;
 931                 IXGBE_CORE_UNLOCK(adapter);
 932                 break;
 933         case SIOCADDMULTI:
 934         case SIOCDELMULTI:
 935                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 936                 if (ifp->if_flags & IFF_RUNNING) {
 937                         IXGBE_CORE_LOCK(adapter);
 938                         ixgbe_disable_intr(adapter);
 939                         ixgbe_set_multi(adapter);
 940                         ixgbe_enable_intr(adapter);
 941                         IXGBE_CORE_UNLOCK(adapter);
 942                 }
 943                 break;
 944         case SIOCSIFMEDIA:
 945         case SIOCGIFMEDIA:
 946                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
 947                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 948                 break;
 949         case SIOCSIFCAP:
 950         {
 951                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 952                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
 953                 if (mask & IFCAP_HWCSUM)
 954                         ifp->if_capenable ^= IFCAP_HWCSUM;
 955                 if (mask & IFCAP_TSO4)
 956                         ifp->if_capenable ^= IFCAP_TSO4;
 957                 if (mask & IFCAP_TSO6)
 958                         ifp->if_capenable ^= IFCAP_TSO6;
 959 #if 0 /* NET_LRO */
 960                 if (mask & IFCAP_LRO)
 961                         ifp->if_capenable ^= IFCAP_LRO;
 962 #endif
 963                 if (mask & IFCAP_VLAN_HWTAGGING)
 964                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 965                 if (mask & IFCAP_VLAN_HWFILTER)
 966                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
 967 #if 0 /* NET_TSO */
 968                 if (mask & IFCAP_VLAN_HWTSO)
 969                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
 970 #endif
 971                 if (ifp->if_flags & IFF_RUNNING) {
 972                         IXGBE_CORE_LOCK(adapter);
 973                         ixgbe_init_locked(adapter);
 974                         IXGBE_CORE_UNLOCK(adapter);
 975                 }
 976 #if 0
 977                 VLAN_CAPABILITIES(ifp);
 978 #endif
 979                 break;
 980         }
 981
 982         default:
 983                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
 984                 error = ether_ioctl(ifp, command, data);
 985                 break;
 986         }
 987
 988         return (error);
 989 }
 990
 991 /*********************************************************************
 992  *  Init entry point
 993  *
 994  *  This routine is used in two ways. It is used by the stack as
 995  *  init entry point in network interface structure. It is also used
 996  *  by the driver as a hw/sw initialization routine to get to a
 997  *  consistent state.
 998  *
 999  *  return 0 on success, positive on failure
1000  **********************************************************************/
1001 #define IXGBE_MHADD_MFS_SHIFT 16
1002
1003 static void
1004 ixgbe_init_locked(struct adapter *adapter)
1005 {
1006         struct ifnet   *ifp = adapter->ifp;
1007         device_t        dev = adapter->dev;
1008         struct ixgbe_hw *hw = &adapter->hw;
1009         u32             k, txdctl, mhadd, gpie;
1010         u32             rxdctl, rxctrl;
1011
1012         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1013         INIT_DEBUGOUT("ixgbe_init: begin");
1014         hw->adapter_stopped = FALSE;
1015         ixgbe_stop_adapter(hw);
1016         callout_stop(&adapter->timer);
1017
1018         /* reprogram the RAR[0] in case user changed it. */
1019         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1020
1021         /* Get the latest mac address, User can use a LAA */
1022         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1023               IXGBE_ETH_LENGTH_OF_ADDRESS);
1024         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1025         hw->addr_ctrl.rar_used_count = 1;
1026
1027         /* Set the various hardware offload abilities */
1028         ifp->if_hwassist = 0;
1029         if (ifp->if_capenable & IFCAP_TSO)
1030                 ifp->if_hwassist |= CSUM_TSO;
1031         if (ifp->if_capenable & IFCAP_TXCSUM) {
1032                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1033 #if 0
1034                 if (hw->mac.type != ixgbe_mac_82598EB)
1035                         ifp->if_hwassist |= CSUM_SCTP;
1036 #endif
1037         }
1038
1039         /* Prepare transmit descriptors and buffers */
1040         if (ixgbe_setup_transmit_structures(adapter)) {
1041                 device_printf(dev,"Could not setup transmit structures\n");
1042                 ixgbe_stop(adapter);
1043                 return;
1044         }
1045
1046         ixgbe_init_hw(hw);
1047         ixgbe_initialize_transmit_units(adapter);
1048
1049         /* Setup Multicast table */
1050         ixgbe_set_multi(adapter);
1051
1052         /*
1053         ** Determine the correct mbuf pool
1054         ** for doing jumbo/headersplit
1055         */
1056         if (adapter->max_frame_size <= 2048)
1057                 adapter->rx_mbuf_sz = MCLBYTES;
1058         else if (adapter->max_frame_size <= 4096)
1059                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1060         else if (adapter->max_frame_size <= 9216)
1061                 adapter->rx_mbuf_sz = MJUM9BYTES;
1062         else
1063                 adapter->rx_mbuf_sz = MJUM16BYTES;
1064
1065         /* Prepare receive descriptors and buffers */
1066         if (ixgbe_setup_receive_structures(adapter)) {
1067                 device_printf(dev,"Could not setup receive structures\n");
1068                 ixgbe_stop(adapter);
1069                 return;
1070         }
1071
1072         /* Configure RX settings */
1073         ixgbe_initialize_receive_units(adapter);
1074
1075         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1076
1077         /* Enable Fan Failure Interrupt */
1078         gpie |= IXGBE_SDP1_GPIEN;
1079
1080         /* Add for Module detection */
1081         if (hw->mac.type == ixgbe_mac_82599EB)
1082                 gpie |= IXGBE_SDP2_GPIEN;
1083
1084         /* Thermal Failure Detection */
1085         if (hw->mac.type == ixgbe_mac_X540)
1086                 gpie |= IXGBE_SDP0_GPIEN;
1087
1088         if (adapter->msix > 1) {
1089                 /* Enable Enhanced MSIX mode */
1090                 gpie |= IXGBE_GPIE_MSIX_MODE;
1091                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1092                     IXGBE_GPIE_OCD;
1093         }
1094         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1095
1096         /* Set MTU size */
1097         if (ifp->if_mtu > ETHERMTU) {
1098                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1099                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1100                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1101                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1102         }
1103
1104         /* Now enable all the queues */
1105
1106         for (int i = 0; i < adapter->num_queues; i++) {
1107                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1108                 txdctl |= IXGBE_TXDCTL_ENABLE;
1109                 /* Set WTHRESH to 8, burst writeback */
1110                 txdctl |= (8 << 16);
1111                 /*
1112                  * When the internal queue falls below PTHRESH (32),
1113                  * start prefetching as long as there are at least
1114                  * HTHRESH (1) buffers ready. The values are taken
1115                  * from the Intel linux driver 3.8.21.
1116                  * Prefetching enables tx line rate even with 1 queue.
1117                  */
1118                 txdctl |= (32 << 0) | (1 << 8);
1119                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1120         }
1121
1122         for (int i = 0; i < adapter->num_queues; i++) {
1123                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1124                 if (hw->mac.type == ixgbe_mac_82598EB) {
1125                         /*
1126                         ** PTHRESH = 21
1127                         ** HTHRESH = 4
1128                         ** WTHRESH = 8
1129                         */
1130                         rxdctl &= ~0x3FFFFF;
1131                         rxdctl |= 0x080420;
1132                 }
1133                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1134                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1135                 for (k = 0; k < 10; k++) {
1136                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1137                             IXGBE_RXDCTL_ENABLE)
1138                                 break;
1139                         else
1140                                 msec_delay(1);
1141                 }
1142                 wmb();
1143 #ifdef DEV_NETMAP
1144                 /*
1145                  * In netmap mode, we must preserve the buffers made
1146                  * available to userspace before the if_init()
1147                  * (this is true by default on the TX side, because
1148                  * init makes all buffers available to userspace).
1149                  *
1150                  * netmap_reset() and the device specific routines
1151                  * (e.g. ixgbe_setup_receive_rings()) map these
1152                  * buffers at the end of the NIC ring, so here we
1153                  * must set the RDT (tail) register to make sure
1154                  * they are not overwritten.
1155                  *
1156                  * In this driver the NIC ring starts at RDH = 0,
1157                  * RDT points to the last slot available for reception (?),
1158                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1159                  */
1160                 if (ifp->if_capenable & IFCAP_NETMAP) {
1161                         struct netmap_adapter *na = NA(adapter->ifp);
1162                         struct netmap_kring *kring = &na->rx_rings[i];
1163                         int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1164
1165                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1166                 } else
1167 #endif /* DEV_NETMAP */
1168                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1169         }
1170
1171         /* Set up VLAN support and filter */
1172         ixgbe_setup_vlan_hw_support(adapter);
1173
1174         /* Enable Receive engine */
1175         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1176         if (hw->mac.type == ixgbe_mac_82598EB)
1177                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1178         rxctrl |= IXGBE_RXCTRL_RXEN;
1179         ixgbe_enable_rx_dma(hw, rxctrl);
1180
1181         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1182
1183         /* Set up MSI/X routing */
1184         if (ixgbe_enable_msix)  {
1185                 ixgbe_configure_ivars(adapter);
1186                 /* Set up auto-mask */
1187                 if (hw->mac.type == ixgbe_mac_82598EB)
1188                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1189                 else {
1190                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1191                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1192                 }
1193         } else {  /* Simple settings for Legacy/MSI */
1194                 ixgbe_set_ivar(adapter, 0, 0, 0);
1195                 ixgbe_set_ivar(adapter, 0, 0, 1);
1196                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1197         }
1198
1199 #ifdef IXGBE_FDIR
1200         /* Init Flow director */
1201         if (hw->mac.type != ixgbe_mac_82598EB) {
1202                 u32 hdrm = 32 << fdir_pballoc;
1203
1204                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1205                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1206         }
1207 #endif
1208
1209         /*
1210         ** Check on any SFP devices that
1211         ** need to be kick-started
1212         */
1213         if (hw->phy.type == ixgbe_phy_none) {
1214                 int err = hw->phy.ops.identify(hw);
1215                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1216                         device_printf(dev,
1217                             "Unsupported SFP+ module type was detected.\n");
1218                         return;
1219                 }
1220         }
1221
1222         /* Set moderation on the Link interrupt */
1223         ixgbe_set_eitr(adapter, adapter->linkvec, IXGBE_LINK_ITR);
1224
1225         /* Config/Enable Link */
1226         ixgbe_config_link(adapter);
1227
1228         /* Hardware Packet Buffer & Flow Control setup */
1229         {
1230                 u32 rxpb, frame, size, tmp;
1231
1232                 frame = adapter->max_frame_size;
1233
1234                 /* Calculate High Water */
1235                 if (hw->mac.type == ixgbe_mac_X540)
1236                         tmp = IXGBE_DV_X540(frame, frame);
1237                 else
1238                         tmp = IXGBE_DV(frame, frame);
1239                 size = IXGBE_BT2KB(tmp);
1240                 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1241                 hw->fc.high_water[0] = rxpb - size;
1242
1243                 /* Now calculate Low Water */
1244                 if (hw->mac.type == ixgbe_mac_X540)
1245                         tmp = IXGBE_LOW_DV_X540(frame);
1246                 else
1247                         tmp = IXGBE_LOW_DV(frame);
1248                 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1249
1250                 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1251                 hw->fc.pause_time = IXGBE_FC_PAUSE;
1252                 hw->fc.send_xon = TRUE;
1253         }
1254         /* Initialize the FC settings */
1255         ixgbe_start_hw(hw);
1256
1257         /* And now turn on interrupts */
1258         ixgbe_enable_intr(adapter);
1259
1260         /* Now inform the stack we're ready */
1261         ifp->if_flags |= IFF_RUNNING;
1262         ifp->if_flags &= ~IFF_OACTIVE;
1263
1264         return;
1265 }
1266
1267 static void
1268 ixgbe_init(void *arg)
1269 {
1270         struct adapter *adapter = arg;
1271
1272         IXGBE_CORE_LOCK(adapter);
1273         ixgbe_init_locked(adapter);
1274         IXGBE_CORE_UNLOCK(adapter);
1275         return;
1276 }
1277
1278
1279 /*
1280 **
1281 ** MSIX Interrupt Handlers and Tasklets
1282 **
1283 */
1284
1285 static inline void
1286 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1287 {
1288         struct ixgbe_hw *hw = &adapter->hw;
1289         u64     queue = (u64)(1 << vector);
1290         u32     mask;
1291
1292         if (hw->mac.type == ixgbe_mac_82598EB) {
1293                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1294                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1295         } else {
1296                 mask = (queue & 0xFFFFFFFF);
1297                 if (mask)
1298                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1299                 mask = (queue >> 32);
1300                 if (mask)
1301                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1302         }
1303 }
1304
1305 static inline void
1306 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1307 {
1308         struct ixgbe_hw *hw = &adapter->hw;
1309         u64     queue = (u64)(1 << vector);
1310         u32     mask;
1311
1312         if (hw->mac.type == ixgbe_mac_82598EB) {
1313                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1314                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1315         } else {
1316                 mask = (queue & 0xFFFFFFFF);
1317                 if (mask)
1318                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1319                 mask = (queue >> 32);
1320                 if (mask)
1321                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1322         }
1323 }
1324
1325 static inline void
1326 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1327 {
1328         u32 mask;
1329
1330         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1331                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1332                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1333         } else {
1334                 mask = (queues & 0xFFFFFFFF);
1335                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1336                 mask = (queues >> 32);
1337                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1338         }
1339 }
1340
1341 /*********************************************************************
1342  *
1343  *  Legacy Interrupt Service routine
1344  *
1345  **********************************************************************/
1346
1347 static void
1348 ixgbe_legacy_irq(void *arg)
1349 {
1350         struct ix_queue *que = arg;
1351         struct adapter  *adapter = que->adapter;
1352         struct ixgbe_hw *hw = &adapter->hw;
1353         struct          tx_ring *txr = adapter->tx_rings;
1354         u32             reg_eicr;
1355
1356
1357         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1358
1359         ++que->irqs;
1360         if (reg_eicr == 0) {
1361                 ixgbe_enable_intr(adapter);
1362                 return;
1363         }
1364
1365         ixgbe_rxeof(que, adapter->rx_process_limit);
1366
1367         IXGBE_TX_LOCK(txr);
1368         ixgbe_txeof(txr);
1369         if (!ifq_is_empty(&adapter->ifp->if_snd))
1370                 ixgbe_start_locked(txr, adapter->ifp);
1371         IXGBE_TX_UNLOCK(txr);
1372
1373         /* Check for fan failure */
1374         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1375             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1376                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1377                     "REPLACE IMMEDIATELY!!\n");
1378                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1379         }
1380
1381         /* Link status change */
1382         if (reg_eicr & IXGBE_EICR_LSC) {
1383                 ixgbe_check_link(&adapter->hw,
1384                     &adapter->link_speed, &adapter->link_up, 0);
1385                 ixgbe_update_link_status(adapter);
1386         }
1387
1388         ixgbe_enable_intr(adapter);
1389 }
1390
1391
1392 /*********************************************************************
1393  *
1394  *  MSIX Queue Interrupt Service routine
1395  *
1396  **********************************************************************/
1397 void
1398 ixgbe_msix_que(void *arg)
1399 {
1400         struct ix_queue *que = arg;
1401         struct adapter  *adapter = que->adapter;
1402         struct tx_ring  *txr = que->txr;
1403
1404         ixgbe_disable_queue(adapter, que->msix);
1405         ++que->irqs;
1406
1407         ixgbe_rxeof(que, adapter->rx_process_limit);
1408
1409         IXGBE_TX_LOCK(txr);
1410         ixgbe_txeof(txr);
1411         if (!ifq_is_empty(&adapter->ifp->if_snd))
1412                 ixgbe_start_locked(txr, adapter->ifp);
1413         IXGBE_TX_UNLOCK(txr);
1414
1415         /* Reenable this interrupt */
1416         ixgbe_enable_queue(adapter, que->msix);
1417 }
1418
1419
1420 static void
1421 ixgbe_msix_link(void *arg)
1422 {
1423         struct adapter  *adapter = arg;
1424         struct ixgbe_hw *hw = &adapter->hw;
1425         u32             reg_eicr;
1426
1427         ++adapter->link_irq;
1428
1429         /* First get the cause */
1430         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1431         /* Clear interrupt with write */
1432         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1433
1434         /* Link status change */
1435         if (reg_eicr & IXGBE_EICR_LSC) {
1436                 ixgbe_check_link(&adapter->hw,
1437                     &adapter->link_speed, &adapter->link_up, 0);
1438                 ixgbe_update_link_status(adapter);
1439         }
1440
1441         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1442 #ifdef IXGBE_FDIR
1443                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1444                         /* This is probably overkill :) */
1445                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1446                                 return;
1447                         /* Disable the interrupt */
1448                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1449                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1450                 } else
1451 #endif
1452                 if (reg_eicr & IXGBE_EICR_ECC) {
1453                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1454                             "Please Reboot!!\n");
1455                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1456                 } else
1457
1458                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1459                         /* Clear the interrupt */
1460                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1461                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1462                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1463                         /* Clear the interrupt */
1464                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1465                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1466                 }
1467         }
1468
1469         /* Check for fan failure */
1470         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1471             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1472                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1473                     "REPLACE IMMEDIATELY!!\n");
1474                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1475         }
1476
1477         /* Check for over temp condition */
1478         if ((hw->mac.type == ixgbe_mac_X540) &&
1479             (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1480                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1481                     "PHY IS SHUT DOWN!!\n");
1482                 device_printf(adapter->dev, "System shutdown required\n");
1483                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1484         }
1485
1486         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1487         return;
1488 }
1489
1490 /*********************************************************************
1491  *
1492  *  Media Ioctl callback
1493  *
1494  *  This routine is called whenever the user queries the status of
1495  *  the interface using ifconfig.
1496  *
1497  **********************************************************************/
1498 static void
1499 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1500 {
1501         struct adapter *adapter = ifp->if_softc;
1502
1503         ASSERT_IFNET_SERIALIZED_ALL(ifp);
1504
1505         INIT_DEBUGOUT("ixgbe_media_status: begin");
1506         ixgbe_update_link_status(adapter);
1507
1508         ifmr->ifm_status = IFM_AVALID;
1509         ifmr->ifm_active = IFM_ETHER;
1510
1511         if (!adapter->link_active)
1512                 return;
1513
1514         ifmr->ifm_status |= IFM_ACTIVE;
1515
1516         switch (adapter->link_speed) {
1517                 case IXGBE_LINK_SPEED_100_FULL:
1518                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1519                         break;
1520                 case IXGBE_LINK_SPEED_1GB_FULL:
1521                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1522                         break;
1523                 case IXGBE_LINK_SPEED_10GB_FULL:
1524                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1525                         break;
1526         }
1527
1528         return;
1529 }
1530
1531 /*********************************************************************
1532  *
1533  *  Media Ioctl callback
1534  *
1535  *  This routine is called when the user changes speed/duplex using
1536  *  media/mediopt option with ifconfig.
1537  *
1538  **********************************************************************/
1539 static int
1540 ixgbe_media_change(struct ifnet * ifp)
1541 {
1542         struct adapter *adapter = ifp->if_softc;
1543         struct ifmedia *ifm = &adapter->media;
1544
1545         INIT_DEBUGOUT("ixgbe_media_change: begin");
1546
1547         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1548                 return (EINVAL);
1549
1550         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1551         case IFM_AUTO:
1552                 adapter->hw.phy.autoneg_advertised =
1553                     IXGBE_LINK_SPEED_100_FULL |
1554                     IXGBE_LINK_SPEED_1GB_FULL |
1555                     IXGBE_LINK_SPEED_10GB_FULL;
1556                 break;
1557         default:
1558                 device_printf(adapter->dev, "Only auto media type\n");
1559                 return (EINVAL);
1560         }
1561
1562         return (0);
1563 }
1564
1565 /*********************************************************************
1566  *
1567  *  This routine maps the mbufs to tx descriptors, allowing the
1568  *  TX engine to transmit the packets.
1569  *      - return 0 on success, positive on failure
1570  *
1571  **********************************************************************/
1572
1573 static int
1574 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1575 {
1576         struct adapter  *adapter = txr->adapter;
1577         u32             olinfo_status = 0, cmd_type_len;
1578         u32             paylen = 0;
1579         int             i, j, error, nsegs, maxsegs;
1580         int             first, last = 0;
1581         struct mbuf     *m_head;
1582         bus_dma_segment_t segs[adapter->num_segs];
1583         bus_dmamap_t    map;
1584         struct ixgbe_tx_buf *txbuf;
1585         union ixgbe_adv_tx_desc *txd = NULL;
1586
1587         m_head = *m_headp;
1588
1589         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1590                 error = ixgbe_tso_pullup(txr, m_headp);
1591                 if (error)
1592                         return error;
1593                 m_head = *m_headp;
1594         }
1595
1596         /* Basic descriptor defines */
1597         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1598             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1599
1600         if (m_head->m_flags & M_VLANTAG)
1601                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1602
1603         /*
1604          * Important to capture the first descriptor
1605          * used because it will contain the index of
1606          * the one we tell the hardware to report back
1607          */
1608         first = txr->next_avail_desc;
1609         txbuf = &txr->tx_buffers[first];
1610         map = txbuf->map;
1611
1612         /*
1613          * Map the packet for DMA.
1614          */
1615         maxsegs = txr->tx_avail - IXGBE_TX_RESERVED;
1616         if (maxsegs > adapter->num_segs)
1617                 maxsegs = adapter->num_segs;
1618
1619         error = bus_dmamap_load_mbuf_defrag(txr->txtag, map, m_headp,
1620             segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1621         if (error) {
1622                 if (error == ENOBUFS)
1623                         adapter->mbuf_defrag_failed++;
1624                 else
1625                         adapter->no_tx_dma_setup++;
1626
1627                 m_freem(*m_headp);
1628                 *m_headp = NULL;
1629                 return (error);
1630         }
1631
1632         /* Make certain there are enough descriptors */
1633         if (nsegs > txr->tx_avail - 2) {
1634                 txr->no_desc_avail++;
1635                 error = ENOBUFS;
1636                 goto xmit_fail;
1637         }
1638         m_head = *m_headp;
1639
1640         /*
1641         ** Set up the appropriate offload context
1642         ** this becomes the first descriptor of
1643         ** a packet.
1644         */
1645         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1646                 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1647                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1648                         olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1649                         olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1650                         ++adapter->tso_tx;
1651                 } else
1652                         return (ENXIO);
1653         } else if (ixgbe_tx_ctx_setup(txr, m_head))
1654                 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1655
1656 #ifdef IXGBE_IEEE1588
1657         /* This is changing soon to an mtag detection */
1658         if (we detect this mbuf has a TSTAMP mtag)
1659                 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1660 #endif
1661
1662 #ifdef IXGBE_FDIR
1663         /* Do the flow director magic */
1664         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1665                 ++txr->atr_count;
1666                 if (txr->atr_count >= atr_sample_rate) {
1667                         ixgbe_atr(txr, m_head);
1668                         txr->atr_count = 0;
1669                 }
1670         }
1671 #endif
1672         /* Record payload length */
1673         if (paylen == 0)
1674                 olinfo_status |= m_head->m_pkthdr.len <<
1675                     IXGBE_ADVTXD_PAYLEN_SHIFT;
1676
1677         i = txr->next_avail_desc;
1678         for (j = 0; j < nsegs; j++) {
1679                 bus_size_t seglen;
1680                 bus_addr_t segaddr;
1681
1682                 txbuf = &txr->tx_buffers[i];
1683                 txd = &txr->tx_base[i];
1684                 seglen = segs[j].ds_len;
1685                 segaddr = htole64(segs[j].ds_addr);
1686
1687                 txd->read.buffer_addr = segaddr;
1688                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1689                     cmd_type_len |seglen);
1690                 txd->read.olinfo_status = htole32(olinfo_status);
1691                 last = i; /* descriptor that will get completion IRQ */
1692
1693                 if (++i == adapter->num_tx_desc)
1694                         i = 0;
1695
1696                 txbuf->m_head = NULL;
1697                 txbuf->eop_index = -1;
1698         }
1699
1700         txd->read.cmd_type_len |=
1701             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1702         txr->tx_avail -= nsegs;
1703         txr->next_avail_desc = i;
1704
1705         txbuf->m_head = m_head;
1706         /* Swap the dma map between the first and last descriptor */
1707         txr->tx_buffers[first].map = txbuf->map;
1708         txbuf->map = map;
1709         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1710
1711         /* Set the index of the descriptor that will be marked done */
1712         txbuf = &txr->tx_buffers[first];
1713         txbuf->eop_index = last;
1714
1715         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1716             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1717         /*
1718          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1719          * hardware that this frame is available to transmit.
1720          */
1721         ++txr->total_packets;
1722         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1723
1724         return (0);
1725
1726 xmit_fail:
1727         bus_dmamap_unload(txr->txtag, txbuf->map);
1728         return (error);
1729
1730 }
1731
1732 static void
1733 ixgbe_set_promisc(struct adapter *adapter)
1734 {
1735         u_int32_t       reg_rctl;
1736         struct ifnet   *ifp = adapter->ifp;
1737
1738         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1739         reg_rctl &= (~IXGBE_FCTRL_UPE);
1740         reg_rctl &= (~IXGBE_FCTRL_MPE);
1741         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1742
1743         if (ifp->if_flags & IFF_PROMISC) {
1744                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1745                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1746         } else if (ifp->if_flags & IFF_ALLMULTI) {
1747                 reg_rctl |= IXGBE_FCTRL_MPE;
1748                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1749                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1750         }
1751         return;
1752 }
1753
1754
1755 /*********************************************************************
1756  *  Multicast Update
1757  *
1758  *  This routine is called whenever multicast address list is updated.
1759  *
1760  **********************************************************************/
1761 #define IXGBE_RAR_ENTRIES 16
1762
1763 static void
1764 ixgbe_set_multi(struct adapter *adapter)
1765 {
1766         u32     fctrl;
1767         u8      *mta;
1768         u8      *update_ptr;
1769         struct  ifmultiaddr *ifma;
1770         int     mcnt = 0;
1771         struct ifnet   *ifp = adapter->ifp;
1772
1773         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1774
1775         mta = adapter->mta;
1776         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1777             MAX_NUM_MULTICAST_ADDRESSES);
1778
1779         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1780         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1781         if (ifp->if_flags & IFF_PROMISC)
1782                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1783         else if (ifp->if_flags & IFF_ALLMULTI) {
1784                 fctrl |= IXGBE_FCTRL_MPE;
1785                 fctrl &= ~IXGBE_FCTRL_UPE;
1786         } else
1787                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1788
1789         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1790
1791         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1792                 if (ifma->ifma_addr->sa_family != AF_LINK)
1793                         continue;
1794                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1795                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1796                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1797                 mcnt++;
1798         }
1799
1800         update_ptr = mta;
1801         ixgbe_update_mc_addr_list(&adapter->hw,
1802             update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1803
1804         return;
1805 }
1806
1807 /*
1808  * This is an iterator function now needed by the multicast
1809  * shared code. It simply feeds the shared code routine the
1810  * addresses in the array of ixgbe_set_multi() one by one.
1811  */
1812 static u8 *
1813 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1814 {
1815         u8 *addr = *update_ptr;
1816         u8 *newptr;
1817         *vmdq = 0;
1818
1819         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1820         *update_ptr = newptr;
1821         return addr;
1822 }
1823
1824
1825 /*********************************************************************
1826  *  Timer routine
1827  *
1828  *  This routine checks for link status,updates statistics,
1829  *  and runs the watchdog check.
1830  *
1831  **********************************************************************/
1832
1833 static void
1834 ixgbe_local_timer(void *arg)
1835 {
1836         struct adapter  *adapter = arg;
1837         device_t        dev = adapter->dev;
1838         struct ifnet    *ifp = adapter->ifp;
1839         struct ix_queue *que = adapter->queues;
1840         struct tx_ring  *txr = adapter->tx_rings;
1841         int             hung, busy, paused;
1842
1843         IXGBE_CORE_LOCK(adapter);
1844         hung = busy = paused = 0;
1845
1846         /* Check for pluggable optics */
1847         if (adapter->sfp_probe)
1848                 if (!ixgbe_sfp_probe(adapter))
1849                         goto out; /* Nothing to do */
1850
1851         ixgbe_update_link_status(adapter);
1852         ixgbe_update_stats_counters(adapter);
1853
1854         /*
1855          * If the interface has been paused
1856          * then don't do the watchdog check
1857          */
1858         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
1859                 paused = 1;
1860
1861         /*
1862         ** Check the TX queues status
1863         **      - central locked handling of OACTIVE
1864         **      - watchdog only if all queues show hung
1865         */
1866         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
1867                 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
1868                     (paused == 0))
1869                         ++hung;
1870                 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
1871                         ++busy;
1872         }
1873         /* Only truely watchdog if all queues show hung */
1874         if (hung == adapter->num_queues)
1875                 goto watchdog;
1876         /* Only turn off the stack flow when ALL are depleted */
1877         if (busy == adapter->num_queues)
1878                 ifp->if_flags |= IFF_OACTIVE;
1879         else if ((ifp->if_flags & IFF_OACTIVE) &&
1880             (busy < adapter->num_queues))
1881                 ifp->if_flags &= ~IFF_OACTIVE;
1882
1883 out:
1884         ixgbe_rearm_queues(adapter, adapter->que_mask);
1885         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1886         IXGBE_CORE_UNLOCK(adapter);
1887         return;
1888
1889 watchdog:
1890         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
1891         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
1892             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
1893             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
1894         device_printf(dev,"TX(%d) desc avail = %d,"
1895             "Next TX to Clean = %d\n",
1896             txr->me, txr->tx_avail, txr->next_to_clean);
1897         adapter->ifp->if_flags &= ~IFF_RUNNING;
1898         adapter->watchdog_events++;
1899         ixgbe_init_locked(adapter);
1900
1901         IXGBE_CORE_UNLOCK(adapter);
1902 }
1903
1904 /*
1905 ** Note: this routine updates the OS on the link state
1906 **      the real check of the hardware only happens with
1907 **      a link interrupt.
1908 */
1909 static void
1910 ixgbe_update_link_status(struct adapter *adapter)
1911 {
1912         struct ifnet    *ifp = adapter->ifp;
1913         struct tx_ring *txr = adapter->tx_rings;
1914         device_t dev = adapter->dev;
1915
1916
1917         if (adapter->link_up){
1918                 if (adapter->link_active == FALSE) {
1919                         if (bootverbose)
1920                                 device_printf(dev,"Link is up %d Gbps %s \n",
1921                                     ((adapter->link_speed == 128)? 10:1),
1922                                     "Full Duplex");
1923                         adapter->link_active = TRUE;
1924                         /* Update any Flow Control changes */
1925                         ixgbe_fc_enable(&adapter->hw);
1926                         ifp->if_link_state = LINK_STATE_UP;
1927                         if_link_state_change(ifp);
1928                 }
1929         } else { /* Link down */
1930                 if (adapter->link_active == TRUE) {
1931                         if (bootverbose)
1932                                 device_printf(dev,"Link is Down\n");
1933                         ifp->if_link_state = LINK_STATE_DOWN;
1934                         if_link_state_change(ifp);
1935                         adapter->link_active = FALSE;
1936                         for (int i = 0; i < adapter->num_queues;
1937                             i++, txr++)
1938                                 txr->queue_status = IXGBE_QUEUE_IDLE;
1939                 }
1940         }
1941
1942         return;
1943 }
1944
1945
1946 /*********************************************************************
1947  *
1948  *  This routine disables all traffic on the adapter by issuing a
1949  *  global reset on the MAC and deallocates TX/RX buffers.
1950  *
1951  **********************************************************************/
1952
1953 static void
1954 ixgbe_stop(void *arg)
1955 {
1956         struct ifnet   *ifp;
1957         struct adapter *adapter = arg;
1958         struct ixgbe_hw *hw = &adapter->hw;
1959         ifp = adapter->ifp;
1960
1961         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1962
1963         INIT_DEBUGOUT("ixgbe_stop: begin\n");
1964         ixgbe_disable_intr(adapter);
1965         callout_stop(&adapter->timer);
1966
1967         /* Let the stack know...*/
1968         ifp->if_flags &= ~IFF_RUNNING;
1969         ifp->if_flags |= IFF_OACTIVE;
1970
1971         ixgbe_reset_hw(hw);
1972         hw->adapter_stopped = FALSE;
1973         ixgbe_stop_adapter(hw);
1974         /* Turn off the laser */
1975         if (hw->phy.multispeed_fiber)
1976                 ixgbe_disable_tx_laser(hw);
1977
1978         /* reprogram the RAR[0] in case user changed it. */
1979         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1980
1981         return;
1982 }
1983
1984
1985 /*********************************************************************
1986  *
1987  *  Determine hardware revision.
1988  *
1989  **********************************************************************/
1990 static void
1991 ixgbe_identify_hardware(struct adapter *adapter)
1992 {
1993         device_t        dev = adapter->dev;
1994         struct ixgbe_hw *hw = &adapter->hw;
1995
1996         /* Save off the information about this board */
1997         hw->vendor_id = pci_get_vendor(dev);
1998         hw->device_id = pci_get_device(dev);
1999         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2000         hw->subsystem_vendor_id =
2001             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2002         hw->subsystem_device_id =
2003             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2004
2005         /* We need this here to set the num_segs below */
2006         ixgbe_set_mac_type(hw);
2007
2008         /* Pick up the 82599 and VF settings */
2009         if (hw->mac.type != ixgbe_mac_82598EB) {
2010                 hw->phy.smart_speed = ixgbe_smart_speed;
2011                 adapter->num_segs = IXGBE_82599_SCATTER;
2012         } else
2013                 adapter->num_segs = IXGBE_82598_SCATTER;
2014
2015         return;
2016 }
2017
2018 /*********************************************************************
2019  *
2020  *  Determine optic type
2021  *
2022  **********************************************************************/
2023 static void
2024 ixgbe_setup_optics(struct adapter *adapter)
2025 {
2026         struct ixgbe_hw *hw = &adapter->hw;
2027         int             layer;
2028
2029         layer = ixgbe_get_supported_physical_layer(hw);
2030
2031         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2032                 adapter->optics = IFM_10G_T;
2033                 return;
2034         }
2035
2036         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2037                 adapter->optics = IFM_1000_T;
2038                 return;
2039         }
2040
2041         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2042             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2043                 adapter->optics = IFM_10G_LR;
2044                 return;
2045         }
2046
2047         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2048                 adapter->optics = IFM_10G_SR;
2049                 return;
2050         }
2051
2052         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2053                 adapter->optics = IFM_10G_TWINAX;
2054                 return;
2055         }
2056
2057         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2058             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2059                 adapter->optics = IFM_10G_CX4;
2060                 return;
2061         }
2062
2063         /* If we get here just set the default */
2064         adapter->optics = IFM_ETHER | IFM_AUTO;
2065         return;
2066 }
2067
2068 /*********************************************************************
2069  *
2070  *  Setup the Legacy or MSI Interrupt handler
2071  *
2072  **********************************************************************/
2073 static int
2074 ixgbe_allocate_legacy(struct adapter *adapter)
2075 {
2076         device_t dev = adapter->dev;
2077         struct          ix_queue *que = adapter->queues;
2078         int error, rid = 0;
2079         unsigned int intr_flags;
2080
2081         /* MSI RID at 1 */
2082         if (adapter->msix == 1)
2083                 rid = 1;
2084
2085         /* Try allocating a MSI interrupt first */
2086         adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2087                 &rid, &intr_flags);
2088
2089         /* We allocate a single interrupt resource */
2090         adapter->res = bus_alloc_resource_any(dev,
2091             SYS_RES_IRQ, &rid, intr_flags);
2092         if (adapter->res == NULL) {
2093                 device_printf(dev, "Unable to allocate bus resource: "
2094                     "interrupt\n");
2095                 return (ENXIO);
2096         }
2097
2098         /* Tasklets for Link, SFP and Multispeed Fiber */
2099         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2100         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2101 #ifdef IXGBE_FDIR
2102         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2103 #endif
2104
2105         if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2106             ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2107                 device_printf(dev, "Failed to register fast interrupt "
2108                     "handler: %d\n", error);
2109                 taskqueue_free(adapter->tq);
2110                 adapter->tq = NULL;
2111                 return (error);
2112         }
2113         /* For simplicity in the handlers */
2114         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2115
2116         return (0);
2117 }
2118
2119
2120 /*********************************************************************
2121  *
2122  *  Setup MSIX Interrupt resources and handlers
2123  *
2124  **********************************************************************/
2125 static int
2126 ixgbe_allocate_msix(struct adapter *adapter)
2127 {
2128         device_t        dev = adapter->dev;
2129         struct          ix_queue *que = adapter->queues;
2130         int             error, rid, vector = 0;
2131         char            desc[16];
2132
2133         error = pci_setup_msix(dev);
2134         if (error) {
2135                 device_printf(dev, "MSI-X setup failed\n");
2136                 return (error);
2137         }
2138
2139         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2140                 rid = vector + 1;
2141
2142                 /*
2143                 ** Bind the msix vector, and thus the
2144                 ** ring to the corresponding cpu.
2145                 */
2146                 error = pci_alloc_msix_vector(dev, vector, &rid, i);
2147                 if (error) {
2148                         device_printf(dev, "pci_alloc_msix_vector failed\n");
2149                         return (error);
2150                 }
2151
2152                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2153                     RF_SHAREABLE | RF_ACTIVE);
2154                 if (que->res == NULL) {
2155                         device_printf(dev,"Unable to allocate"
2156                             " bus resource: que interrupt [%d]\n", vector);
2157                         return (ENXIO);
2158                 }
2159                 /* Set the handler function */
2160                 ksnprintf(desc, sizeof(desc), "%s que %d",
2161                     device_get_nameunit(dev), i);
2162                 error = bus_setup_intr_descr(dev, que->res, INTR_MPSAFE,
2163                     ixgbe_msix_que, que, &que->tag, &que->serializer, desc);
2164                 if (error) {
2165                         que->res = NULL;
2166                         device_printf(dev, "Failed to register QUE handler");
2167                         return (error);
2168                 }
2169                 que->msix = vector;
2170                 adapter->que_mask |= (u64)(1 << que->msix);
2171         }
2172
2173         /* and Link, bind vector to cpu #0 */
2174         rid = vector + 1;
2175         error = pci_alloc_msix_vector(dev, vector, &rid, 0);
2176         if (error) {
2177                 device_printf(dev, "pci_alloc_msix_vector failed\n");
2178                 return (error);
2179         }
2180         adapter->res = bus_alloc_resource_any(dev,
2181             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2182         if (!adapter->res) {
2183                 device_printf(dev,"Unable to allocate"
2184             " bus resource: Link interrupt [%d]\n", rid);
2185                 return (ENXIO);
2186         }
2187         /* Set the link handler function */
2188         error = bus_setup_intr_descr(dev, adapter->res, INTR_MPSAFE,
2189             ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer,
2190             "link");
2191         if (error) {
2192                 adapter->res = NULL;
2193                 device_printf(dev, "Failed to register LINK handler");
2194                 return (error);
2195         }
2196         pci_enable_msix(dev);
2197
2198         adapter->linkvec = vector;
2199         /* Tasklets for Link, SFP and Multispeed Fiber */
2200         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2201         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2202 #ifdef IXGBE_FDIR
2203         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2204 #endif
2205
2206         return (0);
2207 }
2208
2209 /*
2210  * Setup Either MSI/X or MSI
2211  */
2212 static int
2213 ixgbe_setup_msix(struct adapter *adapter)
2214 {
2215         device_t dev = adapter->dev;
2216         int rid, want, queues, msgs;
2217
2218         /* Override by tuneable */
2219         if (ixgbe_enable_msix == 0)
2220                 goto msi;
2221
2222         /* First try MSI/X */
2223         rid = PCIR_BAR(MSIX_82598_BAR);
2224         adapter->msix_mem = bus_alloc_resource_any(dev,
2225             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2226         if (!adapter->msix_mem) {
2227                 rid += 4;       /* 82599 maps in higher BAR */
2228                 adapter->msix_mem = bus_alloc_resource_any(dev,
2229                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2230         }
2231         if (!adapter->msix_mem) {
2232                 /* May not be enabled */
2233                 device_printf(adapter->dev,
2234                     "Unable to map MSIX table \n");
2235                 goto msi;
2236         }
2237
2238         msgs = pci_msix_count(dev);
2239         if (msgs == 0) { /* system has msix disabled */
2240                 bus_release_resource(dev, SYS_RES_MEMORY,
2241                     rid, adapter->msix_mem);
2242                 adapter->msix_mem = NULL;
2243                 goto msi;
2244         }
2245
2246         /* Figure out a reasonable auto config value */
2247         queues = (ncpus > (msgs-1)) ? (msgs-1) : ncpus;
2248
2249         if (ixgbe_num_queues != 0)
2250                 queues = ixgbe_num_queues;
2251         /* Set max queues to 8 when autoconfiguring */
2252         else if ((ixgbe_num_queues == 0) && (queues > 8))
2253                 queues = 8;
2254
2255         /*
2256         ** Want one vector (RX/TX pair) per queue
2257         ** plus an additional for Link.
2258         */
2259         want = queues + 1;
2260         if (msgs >= want)
2261                 msgs = want;
2262         else {
2263                 device_printf(adapter->dev,
2264                     "MSIX Configuration Problem, "
2265                     "%d vectors but %d queues wanted!\n",
2266                     msgs, want);
2267                 return (0); /* Will go to Legacy setup */
2268         }
2269         if (msgs) {
2270                 device_printf(adapter->dev,
2271                     "Using MSIX interrupts with %d vectors\n", msgs);
2272                 adapter->num_queues = queues;
2273                 return (msgs);
2274         }
2275 msi:
2276         msgs = pci_msi_count(dev);
2277         return (msgs);
2278 }
2279
2280
2281 static int
2282 ixgbe_allocate_pci_resources(struct adapter *adapter)
2283 {
2284         int             rid;
2285         device_t        dev = adapter->dev;
2286
2287         rid = PCIR_BAR(0);
2288         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2289             &rid, RF_ACTIVE);
2290
2291         if (!(adapter->pci_mem)) {
2292                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2293                 return (ENXIO);
2294         }
2295
2296         adapter->osdep.mem_bus_space_tag =
2297                 rman_get_bustag(adapter->pci_mem);
2298         adapter->osdep.mem_bus_space_handle =
2299                 rman_get_bushandle(adapter->pci_mem);
2300         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2301
2302         /* Legacy defaults */
2303         adapter->num_queues = 1;
2304         adapter->hw.back = &adapter->osdep;
2305
2306         /*
2307         ** Now setup MSI or MSI/X, should
2308         ** return us the number of supported
2309         ** vectors. (Will be 1 for MSI)
2310         */
2311         adapter->msix = ixgbe_setup_msix(adapter);
2312         return (0);
2313 }
2314
2315 static void
2316 ixgbe_free_pci_resources(struct adapter * adapter)
2317 {
2318         struct          ix_queue *que = adapter->queues;
2319         device_t        dev = adapter->dev;
2320         int             rid, memrid;
2321
2322         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2323                 memrid = PCIR_BAR(MSIX_82598_BAR);
2324         else
2325                 memrid = PCIR_BAR(MSIX_82599_BAR);
2326
2327         /*
2328         ** There is a slight possibility of a failure mode
2329         ** in attach that will result in entering this function
2330         ** before interrupt resources have been initialized, and
2331         ** in that case we do not want to execute the loops below
2332         ** We can detect this reliably by the state of the adapter
2333         ** res pointer.
2334         */
2335         if (adapter->res == NULL)
2336                 goto mem;
2337
2338         /*
2339         **  Release all msix queue resources:
2340         */
2341         for (int i = 0; i < adapter->num_queues; i++, que++) {
2342                 rid = que->msix + 1;
2343                 if (que->tag != NULL) {
2344                         bus_teardown_intr(dev, que->res, que->tag);
2345                         que->tag = NULL;
2346                 }
2347                 if (que->res != NULL)
2348                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2349         }
2350
2351
2352         /* Clean the Legacy or Link interrupt last */
2353         if (adapter->linkvec) /* we are doing MSIX */
2354                 rid = adapter->linkvec + 1;
2355         else
2356                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2357
2358         if (adapter->tag != NULL) {
2359                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2360                 adapter->tag = NULL;
2361         }
2362         if (adapter->res != NULL)
2363                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2364         if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2365                 pci_release_msi(adapter->dev);
2366
2367 mem:
2368         if (adapter->msix)
2369                 pci_release_msi(dev);
2370
2371         if (adapter->msix_mem != NULL)
2372                 bus_release_resource(dev, SYS_RES_MEMORY,
2373                     memrid, adapter->msix_mem);
2374
2375         if (adapter->pci_mem != NULL)
2376                 bus_release_resource(dev, SYS_RES_MEMORY,
2377                     PCIR_BAR(0), adapter->pci_mem);
2378
2379         return;
2380 }
2381
2382 /*********************************************************************
2383  *
2384  *  Setup networking device structure and register an interface.
2385  *
2386  **********************************************************************/
2387 static int
2388 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2389 {
2390         struct ixgbe_hw *hw = &adapter->hw;
2391         struct ifnet   *ifp;
2392
2393         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2394
2395         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2396         if (ifp == NULL) {
2397                 device_printf(dev, "can not allocate ifnet structure\n");
2398                 return (-1);
2399         }
2400         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2401         ifp->if_baudrate = 1000000000;
2402         ifp->if_init = ixgbe_init;
2403         ifp->if_softc = adapter;
2404         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2405         ifp->if_ioctl = ixgbe_ioctl;
2406         ifp->if_start = ixgbe_start;
2407 #if 0 /* __FreeBSD_version >= 800000 */
2408         ifp->if_transmit = ixgbe_mq_start;
2409         ifp->if_qflush = ixgbe_qflush;
2410 #endif
2411         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2412
2413         ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2414
2415         adapter->max_frame_size =
2416             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2417
2418         /*
2419          * Tell the upper layer(s) we support long frames.
2420          */
2421         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2422
2423         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2424         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2425         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2426 #if 0 /* NET_TSO */
2427                              |  IFCAP_VLAN_HWTSO
2428 #endif
2429                              |  IFCAP_VLAN_MTU;
2430         ifp->if_capenable = ifp->if_capabilities;
2431
2432         /* Don't enable LRO by default */
2433 #if 0 /* NET_LRO */
2434         ifp->if_capabilities |= IFCAP_LRO;
2435 #endif
2436
2437         /*
2438         ** Don't turn this on by default, if vlans are
2439         ** created on another pseudo device (eg. lagg)
2440         ** then vlan events are not passed thru, breaking
2441         ** operation, but with HW FILTER off it works. If
2442         ** using vlans directly on the ixgbe driver you can
2443         ** enable this and get full hardware tag filtering.
2444         */
2445         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2446
2447         /*
2448          * Specify the media types supported by this adapter and register
2449          * callbacks to update media and link information
2450          */
2451         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2452                      ixgbe_media_status);
2453         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2454         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2455         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2456                 ifmedia_add(&adapter->media,
2457                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2458                 ifmedia_add(&adapter->media,
2459                     IFM_ETHER | IFM_1000_T, 0, NULL);
2460         }
2461         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2462         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2463
2464         return (0);
2465 }
2466
2467 static void
2468 ixgbe_config_link(struct adapter *adapter)
2469 {
2470         struct ixgbe_hw *hw = &adapter->hw;
2471         u32     autoneg, err = 0;
2472         bool    sfp, negotiate;
2473
2474         sfp = ixgbe_is_sfp(hw);
2475
2476         if (sfp) {
2477                 if (hw->phy.multispeed_fiber) {
2478                         hw->mac.ops.setup_sfp(hw);
2479                         ixgbe_enable_tx_laser(hw);
2480                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2481                 } else
2482                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2483         } else {
2484                 if (hw->mac.ops.check_link)
2485                         err = ixgbe_check_link(hw, &autoneg,
2486                             &adapter->link_up, FALSE);
2487                 if (err)
2488                         goto out;
2489                 autoneg = hw->phy.autoneg_advertised;
2490                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2491                         err  = hw->mac.ops.get_link_capabilities(hw,
2492                             &autoneg, &negotiate);
2493                 if (err)
2494                         goto out;
2495                 if (hw->mac.ops.setup_link)
2496                         err = hw->mac.ops.setup_link(hw, autoneg,
2497                             negotiate, adapter->link_up);
2498         }
2499 out:
2500         return;
2501 }
2502
2503 /********************************************************************
2504  * Manage DMA'able memory.
2505  *******************************************************************/
2506 static void
2507 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2508 {
2509         if (error)
2510                 return;
2511         *(bus_addr_t *) arg = segs->ds_addr;
2512         return;
2513 }
2514
2515 static int
2516 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2517                 struct ixgbe_dma_alloc *dma, int mapflags)
2518 {
2519         device_t dev = adapter->dev;
2520         int             r;
2521
2522         r = bus_dma_tag_create(NULL,    /* parent */
2523                                DBA_ALIGN, 0,    /* alignment, bounds */
2524                                BUS_SPACE_MAXADDR,       /* lowaddr */
2525                                BUS_SPACE_MAXADDR,       /* highaddr */
2526                                NULL, NULL,      /* filter, filterarg */
2527                                size,    /* maxsize */
2528                                1,       /* nsegments */
2529                                size,    /* maxsegsize */
2530                                BUS_DMA_ALLOCNOW,        /* flags */
2531                                &dma->dma_tag);
2532         if (r != 0) {
2533                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2534                        "error %u\n", r);
2535                 goto fail_0;
2536         }
2537         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2538                              BUS_DMA_NOWAIT, &dma->dma_map);
2539         if (r != 0) {
2540                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2541                        "error %u\n", r);
2542                 goto fail_1;
2543         }
2544         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2545                             size,
2546                             ixgbe_dmamap_cb,
2547                             &dma->dma_paddr,
2548                             mapflags | BUS_DMA_NOWAIT);
2549         if (r != 0) {
2550                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2551                        "error %u\n", r);
2552                 goto fail_2;
2553         }
2554         dma->dma_size = size;
2555         return (0);
2556 fail_2:
2557         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2558 fail_1:
2559         bus_dma_tag_destroy(dma->dma_tag);
2560 fail_0:
2561         dma->dma_map = NULL;
2562         dma->dma_tag = NULL;
2563         return (r);
2564 }
2565
2566 static void
2567 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2568 {
2569         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2570             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2571         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2572         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2573         bus_dma_tag_destroy(dma->dma_tag);
2574 }
2575
2576
2577 /*********************************************************************
2578  *
2579  *  Allocate memory for the transmit and receive rings, and then
2580  *  the descriptors associated with each, called only once at attach.
2581  *
2582  **********************************************************************/
2583 static int
2584 ixgbe_allocate_queues(struct adapter *adapter)
2585 {
2586         device_t        dev = adapter->dev;
2587         struct ix_queue *que;
2588         struct tx_ring  *txr;
2589         struct rx_ring  *rxr;
2590         int rsize, tsize, error = IXGBE_SUCCESS;
2591         int txconf = 0, rxconf = 0;
2592
2593         /* First allocate the top level queue structs */
2594         if (!(adapter->queues =
2595             (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2596             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2597                 device_printf(dev, "Unable to allocate queue memory\n");
2598                 error = ENOMEM;
2599                 goto fail;
2600         }
2601
2602         /* First allocate the TX ring struct memory */
2603         if (!(adapter->tx_rings =
2604             (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2605             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2606                 device_printf(dev, "Unable to allocate TX ring memory\n");
2607                 error = ENOMEM;
2608                 goto tx_fail;
2609         }
2610
2611         /* Next allocate the RX */
2612         if (!(adapter->rx_rings =
2613             (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2614             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2615                 device_printf(dev, "Unable to allocate RX ring memory\n");
2616                 error = ENOMEM;
2617                 goto rx_fail;
2618         }
2619
2620         /* For the ring itself */
2621         tsize = roundup2(adapter->num_tx_desc *
2622             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2623
2624         /*
2625          * Now set up the TX queues, txconf is needed to handle the
2626          * possibility that things fail midcourse and we need to
2627          * undo memory gracefully
2628          */
2629         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2630                 /* Set up some basics */
2631                 txr = &adapter->tx_rings[i];
2632                 txr->adapter = adapter;
2633                 txr->me = i;
2634
2635                 /* Initialize the TX side lock */
2636                 IXGBE_TX_LOCK_INIT(txr);
2637
2638                 if (ixgbe_dma_malloc(adapter, tsize,
2639                         &txr->txdma, BUS_DMA_NOWAIT)) {
2640                         device_printf(dev,
2641                             "Unable to allocate TX Descriptor memory\n");
2642                         error = ENOMEM;
2643                         goto err_tx_desc;
2644                 }
2645                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2646                 bzero((void *)txr->tx_base, tsize);
2647
2648                 /* Now allocate transmit buffers for the ring */
2649                 if (ixgbe_allocate_transmit_buffers(txr)) {
2650                         device_printf(dev,
2651                             "Critical Failure setting up transmit buffers\n");
2652                         error = ENOMEM;
2653                         goto err_tx_desc;
2654                 }
2655 #if 0 /* __FreeBSD_version >= 800000 */
2656                 /* Allocate a buf ring */
2657                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2658                     M_WAITOK, &txr->tx_mtx);
2659                 if (txr->br == NULL) {
2660                         device_printf(dev,
2661                             "Critical Failure setting up buf ring\n");
2662                         error = ENOMEM;
2663                         goto err_tx_desc;
2664                 }
2665 #endif
2666         }
2667
2668         /*
2669          * Next the RX queues...
2670          */
2671         rsize = roundup2(adapter->num_rx_desc *
2672             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2673         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2674                 rxr = &adapter->rx_rings[i];
2675                 /* Set up some basics */
2676                 rxr->adapter = adapter;
2677                 rxr->me = i;
2678
2679                 /* Initialize the RX side lock */
2680                 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2681                     device_get_nameunit(dev), rxr->me);
2682                 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2683
2684                 if (ixgbe_dma_malloc(adapter, rsize,
2685                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2686                         device_printf(dev,
2687                             "Unable to allocate RxDescriptor memory\n");
2688                         error = ENOMEM;
2689                         goto err_rx_desc;
2690                 }
2691                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2692                 bzero((void *)rxr->rx_base, rsize);
2693
2694                 /* Allocate receive buffers for the ring*/
2695                 if (ixgbe_allocate_receive_buffers(rxr)) {
2696                         device_printf(dev,
2697                             "Critical Failure setting up receive buffers\n");
2698                         error = ENOMEM;
2699                         goto err_rx_desc;
2700                 }
2701         }
2702
2703         /*
2704         ** Finally set up the queue holding structs
2705         */
2706         for (int i = 0; i < adapter->num_queues; i++) {
2707                 que = &adapter->queues[i];
2708                 que->adapter = adapter;
2709                 que->txr = &adapter->tx_rings[i];
2710                 que->rxr = &adapter->rx_rings[i];
2711         }
2712
2713         return (0);
2714
2715 err_rx_desc:
2716         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2717                 ixgbe_dma_free(adapter, &rxr->rxdma);
2718 err_tx_desc:
2719         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2720                 ixgbe_dma_free(adapter, &txr->txdma);
2721         kfree(adapter->rx_rings, M_DEVBUF);
2722 rx_fail:
2723         kfree(adapter->tx_rings, M_DEVBUF);
2724 tx_fail:
2725         kfree(adapter->queues, M_DEVBUF);
2726 fail:
2727         return (error);
2728 }
2729
2730 /*********************************************************************
2731  *
2732  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2733  *  the information needed to transmit a packet on the wire. This is
2734  *  called only once at attach, setup is done every reset.
2735  *
2736  **********************************************************************/
2737 static int
2738 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2739 {
2740         struct adapter *adapter = txr->adapter;
2741         device_t dev = adapter->dev;
2742         struct ixgbe_tx_buf *txbuf;
2743         int error, i;
2744
2745         /*
2746          * Setup DMA descriptor areas.
2747          */
2748         if ((error = bus_dma_tag_create(
2749                                NULL,    /* parent */
2750                                1, 0,            /* alignment, bounds */
2751                                BUS_SPACE_MAXADDR,       /* lowaddr */
2752                                BUS_SPACE_MAXADDR,       /* highaddr */
2753                                NULL, NULL,              /* filter, filterarg */
2754                                IXGBE_TSO_SIZE,          /* maxsize */
2755                                adapter->num_segs,       /* nsegments */
2756                                PAGE_SIZE,               /* maxsegsize */
2757                                0,                       /* flags */
2758                                &txr->txtag))) {
2759                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2760                 goto fail;
2761         }
2762
2763         if (!(txr->tx_buffers =
2764             (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2765             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2766                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2767                 error = ENOMEM;
2768                 goto fail;
2769         }
2770
2771         /* Create the descriptor buffer dma maps */
2772         txbuf = txr->tx_buffers;
2773         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2774                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2775                 if (error != 0) {
2776                         device_printf(dev, "Unable to create TX DMA map\n");
2777                         goto fail;
2778                 }
2779         }
2780
2781         return 0;
2782 fail:
2783         /* We free all, it handles case where we are in the middle */
2784         ixgbe_free_transmit_structures(adapter);
2785         return (error);
2786 }
2787
2788 /*********************************************************************
2789  *
2790  *  Initialize a transmit ring.
2791  *
2792  **********************************************************************/
2793 static void
2794 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2795 {
2796         struct adapter *adapter = txr->adapter;
2797         struct ixgbe_tx_buf *txbuf;
2798         int i;
2799 #ifdef DEV_NETMAP
2800         struct netmap_adapter *na = NA(adapter->ifp);
2801         struct netmap_slot *slot;
2802 #endif /* DEV_NETMAP */
2803
2804         /* Clear the old ring contents */
2805         IXGBE_TX_LOCK(txr);
2806 #ifdef DEV_NETMAP
2807         /*
2808          * (under lock): if in netmap mode, do some consistency
2809          * checks and set slot to entry 0 of the netmap ring.
2810          */
2811         slot = netmap_reset(na, NR_TX, txr->me, 0);
2812 #endif /* DEV_NETMAP */
2813         bzero((void *)txr->tx_base,
2814               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
2815         /* Reset indices */
2816         txr->next_avail_desc = 0;
2817         txr->next_to_clean = 0;
2818
2819         /* Free any existing tx buffers. */
2820         txbuf = txr->tx_buffers;
2821         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2822                 if (txbuf->m_head != NULL) {
2823                         bus_dmamap_sync(txr->txtag, txbuf->map,
2824                             BUS_DMASYNC_POSTWRITE);
2825                         bus_dmamap_unload(txr->txtag, txbuf->map);
2826                         m_freem(txbuf->m_head);
2827                         txbuf->m_head = NULL;
2828                 }
2829 #ifdef DEV_NETMAP
2830                 /*
2831                  * In netmap mode, set the map for the packet buffer.
2832                  * NOTE: Some drivers (not this one) also need to set
2833                  * the physical buffer address in the NIC ring.
2834                  * Slots in the netmap ring (indexed by "si") are
2835                  * kring->nkr_hwofs positions "ahead" wrt the
2836                  * corresponding slot in the NIC ring. In some drivers
2837                  * (not here) nkr_hwofs can be negative. Function
2838                  * netmap_idx_n2k() handles wraparounds properly.
2839                  */
2840                 if (slot) {
2841                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
2842                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
2843                 }
2844 #endif /* DEV_NETMAP */
2845                 /* Clear the EOP index */
2846                 txbuf->eop_index = -1;
2847         }
2848
2849 #ifdef IXGBE_FDIR
2850         /* Set the rate at which we sample packets */
2851         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
2852                 txr->atr_sample = atr_sample_rate;
2853 #endif
2854
2855         /* Set number of descriptors available */
2856         txr->tx_avail = adapter->num_tx_desc;
2857
2858         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2859             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2860         IXGBE_TX_UNLOCK(txr);
2861 }
2862
2863 /*********************************************************************
2864  *
2865  *  Initialize all transmit rings.
2866  *
2867  **********************************************************************/
2868 static int
2869 ixgbe_setup_transmit_structures(struct adapter *adapter)
2870 {
2871         struct tx_ring *txr = adapter->tx_rings;
2872
2873         for (int i = 0; i < adapter->num_queues; i++, txr++)
2874                 ixgbe_setup_transmit_ring(txr);
2875
2876         return (0);
2877 }
2878
2879 /*********************************************************************
2880  *
2881  *  Enable transmit unit.
2882  *
2883  **********************************************************************/
2884 static void
2885 ixgbe_initialize_transmit_units(struct adapter *adapter)
2886 {
2887         struct tx_ring  *txr = adapter->tx_rings;
2888         struct ixgbe_hw *hw = &adapter->hw;
2889
2890         /* Setup the Base and Length of the Tx Descriptor Ring */
2891
2892         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2893                 u64     tdba = txr->txdma.dma_paddr;
2894                 u32     txctrl;
2895
2896                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
2897                        (tdba & 0x00000000ffffffffULL));
2898                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
2899                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
2900                     adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
2901
2902                 /* Setup the HW Tx Head and Tail descriptor pointers */
2903                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
2904                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
2905
2906                 /* Setup Transmit Descriptor Cmd Settings */
2907                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
2908                 txr->queue_status = IXGBE_QUEUE_IDLE;
2909
2910                 /* Disable Head Writeback */
2911                 switch (hw->mac.type) {
2912                 case ixgbe_mac_82598EB:
2913                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
2914                         break;
2915                 case ixgbe_mac_82599EB:
2916                 case ixgbe_mac_X540:
2917                 default:
2918                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
2919                         break;
2920                 }
2921                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2922                 switch (hw->mac.type) {
2923                 case ixgbe_mac_82598EB:
2924                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
2925                         break;
2926                 case ixgbe_mac_82599EB:
2927                 case ixgbe_mac_X540:
2928                 default:
2929                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
2930                         break;
2931                 }
2932
2933         }
2934
2935         if (hw->mac.type != ixgbe_mac_82598EB) {
2936                 u32 dmatxctl, rttdcs;
2937                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2938                 dmatxctl |= IXGBE_DMATXCTL_TE;
2939                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2940                 /* Disable arbiter to set MTQC */
2941                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2942                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
2943                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2944                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2945                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2946                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2947         }
2948
2949         return;
2950 }
2951
2952 /*********************************************************************
2953  *
2954  *  Free all transmit rings.
2955  *
2956  **********************************************************************/
2957 static void
2958 ixgbe_free_transmit_structures(struct adapter *adapter)
2959 {
2960         struct tx_ring *txr = adapter->tx_rings;
2961
2962         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2963                 IXGBE_TX_LOCK(txr);
2964                 ixgbe_free_transmit_buffers(txr);
2965                 ixgbe_dma_free(adapter, &txr->txdma);
2966                 IXGBE_TX_UNLOCK(txr);
2967                 IXGBE_TX_LOCK_DESTROY(txr);
2968         }
2969         kfree(adapter->tx_rings, M_DEVBUF);
2970 }
2971
2972 /*********************************************************************
2973  *
2974  *  Free transmit ring related data structures.
2975  *
2976  **********************************************************************/
2977 static void
2978 ixgbe_free_transmit_buffers(struct tx_ring *txr)
2979 {
2980         struct adapter *adapter = txr->adapter;
2981         struct ixgbe_tx_buf *tx_buffer;
2982         int             i;
2983
2984         INIT_DEBUGOUT("free_transmit_ring: begin");
2985
2986         if (txr->tx_buffers == NULL)
2987                 return;
2988
2989         tx_buffer = txr->tx_buffers;
2990         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2991                 if (tx_buffer->m_head != NULL) {
2992                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
2993                             BUS_DMASYNC_POSTWRITE);
2994                         bus_dmamap_unload(txr->txtag,
2995                             tx_buffer->map);
2996                         m_freem(tx_buffer->m_head);
2997                         tx_buffer->m_head = NULL;
2998                         if (tx_buffer->map != NULL) {
2999                                 bus_dmamap_destroy(txr->txtag,
3000                                     tx_buffer->map);
3001                                 tx_buffer->map = NULL;
3002                         }
3003                 } else if (tx_buffer->map != NULL) {
3004                         bus_dmamap_unload(txr->txtag,
3005                             tx_buffer->map);
3006                         bus_dmamap_destroy(txr->txtag,
3007                             tx_buffer->map);
3008                         tx_buffer->map = NULL;
3009                 }
3010         }
3011 #if 0 /* __FreeBSD_version >= 800000 */
3012         if (txr->br != NULL)
3013                 buf_ring_free(txr->br, M_DEVBUF);
3014 #endif
3015         if (txr->tx_buffers != NULL) {
3016                 kfree(txr->tx_buffers, M_DEVBUF);
3017                 txr->tx_buffers = NULL;
3018         }
3019         if (txr->txtag != NULL) {
3020                 bus_dma_tag_destroy(txr->txtag);
3021                 txr->txtag = NULL;
3022         }
3023         return;
3024 }
3025
3026 /*********************************************************************
3027  *
3028  *  Advanced Context Descriptor setup for VLAN or CSUM
3029  *
3030  **********************************************************************/
3031
3032 static bool
3033 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3034 {
3035         struct adapter *adapter = txr->adapter;
3036         struct ixgbe_adv_tx_context_desc *TXD;
3037         struct ixgbe_tx_buf        *tx_buffer;
3038         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3039         struct ether_vlan_header *eh;
3040         struct ip *ip;
3041         struct ip6_hdr *ip6;
3042         int  ehdrlen, ip_hlen = 0;
3043         u16     etype;
3044         u8      ipproto = 0;
3045         bool    offload = TRUE;
3046         int ctxd = txr->next_avail_desc;
3047         u16 vtag = 0;
3048
3049
3050         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3051                 offload = FALSE;
3052
3053         tx_buffer = &txr->tx_buffers[ctxd];
3054         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3055
3056         /*
3057         ** In advanced descriptors the vlan tag must
3058         ** be placed into the descriptor itself.
3059         */
3060         if (mp->m_flags & M_VLANTAG) {
3061                 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3062                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3063         } else if (offload == FALSE)
3064                 return FALSE;
3065
3066         /*
3067          * Determine where frame payload starts.
3068          * Jump over vlan headers if already present,
3069          * helpful for QinQ too.
3070          */
3071         eh = mtod(mp, struct ether_vlan_header *);
3072         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3073                 etype = ntohs(eh->evl_proto);
3074                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3075         } else {
3076                 etype = ntohs(eh->evl_encap_proto);
3077                 ehdrlen = ETHER_HDR_LEN;
3078         }
3079
3080         /* Set the ether header length */
3081         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3082
3083         switch (etype) {
3084                 case ETHERTYPE_IP:
3085                         ip = (struct ip *)(mp->m_data + ehdrlen);
3086                         ip_hlen = ip->ip_hl << 2;
3087                         ipproto = ip->ip_p;
3088                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3089                         break;
3090                 case ETHERTYPE_IPV6:
3091                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3092                         ip_hlen = sizeof(struct ip6_hdr);
3093                         /* XXX-BZ this will go badly in case of ext hdrs. */
3094                         ipproto = ip6->ip6_nxt;
3095                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3096                         break;
3097                 default:
3098                         offload = FALSE;
3099                         break;
3100         }
3101
3102         vlan_macip_lens |= ip_hlen;
3103         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3104
3105         switch (ipproto) {
3106                 case IPPROTO_TCP:
3107                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3108                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3109                         break;
3110
3111                 case IPPROTO_UDP:
3112                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3113                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3114                         break;
3115
3116 #if 0
3117                 case IPPROTO_SCTP:
3118                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3119                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3120                         break;
3121 #endif
3122                 default:
3123                         offload = FALSE;
3124                         break;
3125         }
3126
3127         /* Now copy bits into descriptor */
3128         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3129         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3130         TXD->seqnum_seed = htole32(0);
3131         TXD->mss_l4len_idx = htole32(0);
3132
3133         tx_buffer->m_head = NULL;
3134         tx_buffer->eop_index = -1;
3135
3136         /* We've consumed the first desc, adjust counters */
3137         if (++ctxd == adapter->num_tx_desc)
3138                 ctxd = 0;
3139         txr->next_avail_desc = ctxd;
3140         --txr->tx_avail;
3141
3142         return (offload);
3143 }
3144
3145 /**********************************************************************
3146  *
3147  *  Setup work for hardware segmentation offload (TSO) on
3148  *  adapters using advanced tx descriptors
3149  *
3150  **********************************************************************/
3151 static bool
3152 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3153     u32 *olinfo_status)
3154 {
3155         struct adapter *adapter = txr->adapter;
3156         struct ixgbe_adv_tx_context_desc *TXD;
3157         struct ixgbe_tx_buf        *tx_buffer;
3158         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3159         u16 vtag = 0, eh_type;
3160         u32 mss_l4len_idx = 0, len;
3161         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3162         struct ether_vlan_header *eh;
3163 #if 0 /* IPv6 TSO */
3164 #ifdef INET6
3165         struct ip6_hdr *ip6;
3166 #endif
3167 #endif
3168 #ifdef INET
3169         struct ip *ip;
3170 #endif
3171         struct tcphdr *th;
3172
3173
3174         /*
3175          * Determine where frame payload starts.
3176          * Jump over vlan headers if already present
3177          */
3178         eh = mtod(mp, struct ether_vlan_header *);
3179         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3180                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3181                 eh_type = eh->evl_proto;
3182         } else {
3183                 ehdrlen = ETHER_HDR_LEN;
3184                 eh_type = eh->evl_encap_proto;
3185         }
3186
3187         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3188         len = ehdrlen + sizeof(struct tcphdr);
3189         switch (ntohs(eh_type)) {
3190 #if 0 /* IPv6 TSO */
3191 #ifdef INET6
3192         case ETHERTYPE_IPV6:
3193                 if (mp->m_len < len + sizeof(struct ip6_hdr))
3194                         return FALSE;
3195                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3196                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3197                 if (ip6->ip6_nxt != IPPROTO_TCP)
3198                         return FALSE;
3199                 ip_hlen = sizeof(struct ip6_hdr);
3200                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3201                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3202                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3203                 break;
3204 #endif
3205 #endif
3206 #ifdef INET
3207         case ETHERTYPE_IP:
3208                 if (mp->m_len < len + sizeof(struct ip))
3209                         return FALSE;
3210                 ip = (struct ip *)(mp->m_data + ehdrlen);
3211                 if (ip->ip_p != IPPROTO_TCP)
3212                         return FALSE;
3213                 ip->ip_sum = 0;
3214                 ip_hlen = ip->ip_hl << 2;
3215                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3216                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3217                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3218                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3219                 /* Tell transmit desc to also do IPv4 checksum. */
3220                 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3221                 break;
3222 #endif
3223         default:
3224                 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3225                     __func__, ntohs(eh_type));
3226                 break;
3227         }
3228
3229         ctxd = txr->next_avail_desc;
3230         tx_buffer = &txr->tx_buffers[ctxd];
3231         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3232
3233         tcp_hlen = th->th_off << 2;
3234
3235         /* This is used in the transmit desc in encap */
3236         *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3237
3238         /* VLAN MACLEN IPLEN */
3239         if (mp->m_flags & M_VLANTAG) {
3240                 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3241                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3242         }
3243
3244         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3245         vlan_macip_lens |= ip_hlen;
3246         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3247
3248         /* ADV DTYPE TUCMD */
3249         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3250         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3251         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3252
3253         /* MSS L4LEN IDX */
3254         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3255         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3256         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3257
3258         TXD->seqnum_seed = htole32(0);
3259         tx_buffer->m_head = NULL;
3260         tx_buffer->eop_index = -1;
3261
3262         if (++ctxd == adapter->num_tx_desc)
3263                 ctxd = 0;
3264
3265         txr->tx_avail--;
3266         txr->next_avail_desc = ctxd;
3267         return TRUE;
3268 }
3269
3270 #ifdef IXGBE_FDIR
3271 /*
3272 ** This routine parses packet headers so that Flow
3273 ** Director can make a hashed filter table entry
3274 ** allowing traffic flows to be identified and kept
3275 ** on the same cpu.  This would be a performance
3276 ** hit, but we only do it at IXGBE_FDIR_RATE of
3277 ** packets.
3278 */
3279 static void
3280 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3281 {
3282         struct adapter                  *adapter = txr->adapter;
3283         struct ix_queue                 *que;
3284         struct ip                       *ip;
3285         struct tcphdr                   *th;
3286         struct udphdr                   *uh;
3287         struct ether_vlan_header        *eh;
3288         union ixgbe_atr_hash_dword      input = {.dword = 0};
3289         union ixgbe_atr_hash_dword      common = {.dword = 0};
3290         int                             ehdrlen, ip_hlen;
3291         u16                             etype;
3292
3293         eh = mtod(mp, struct ether_vlan_header *);
3294         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3295                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3296                 etype = eh->evl_proto;
3297         } else {
3298                 ehdrlen = ETHER_HDR_LEN;
3299                 etype = eh->evl_encap_proto;
3300         }
3301
3302         /* Only handling IPv4 */
3303         if (etype != htons(ETHERTYPE_IP))
3304                 return;
3305
3306         ip = (struct ip *)(mp->m_data + ehdrlen);
3307         ip_hlen = ip->ip_hl << 2;
3308
3309         /* check if we're UDP or TCP */
3310         switch (ip->ip_p) {
3311         case IPPROTO_TCP:
3312                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3313                 /* src and dst are inverted */
3314                 common.port.dst ^= th->th_sport;
3315                 common.port.src ^= th->th_dport;
3316                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3317                 break;
3318         case IPPROTO_UDP:
3319                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3320                 /* src and dst are inverted */
3321                 common.port.dst ^= uh->uh_sport;
3322                 common.port.src ^= uh->uh_dport;
3323                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3324                 break;
3325         default:
3326                 return;
3327         }
3328
3329         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3330         if (mp->m_pkthdr.ether_vtag)
3331                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3332         else
3333                 common.flex_bytes ^= etype;
3334         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3335
3336         que = &adapter->queues[txr->me];
3337         /*
3338         ** This assumes the Rx queue and Tx
3339         ** queue are bound to the same CPU
3340         */
3341         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3342             input, common, que->msix);
3343 }
3344 #endif /* IXGBE_FDIR */
3345
3346 /**********************************************************************
3347  *
3348  *  Examine each tx_buffer in the used queue. If the hardware is done
3349  *  processing the packet then free associated resources. The
3350  *  tx_buffer is put back on the free queue.
3351  *
3352  **********************************************************************/
3353 static void
3354 ixgbe_txeof(struct tx_ring *txr)
3355 {
3356         struct adapter  *adapter = txr->adapter;
3357         struct ifnet    *ifp = adapter->ifp;
3358         u32     first, last, done, processed;
3359         struct ixgbe_tx_buf *tx_buffer;
3360         struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3361
3362         IXGBE_TX_LOCK_ASSERT(txr);
3363
3364 #ifdef DEV_NETMAP
3365         if (ifp->if_capenable & IFCAP_NETMAP) {
3366                 struct netmap_adapter *na = NA(ifp);
3367                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3368
3369                 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3370
3371                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3372                     BUS_DMASYNC_POSTREAD);
3373                 /*
3374                  * In netmap mode, all the work is done in the context
3375                  * of the client thread. Interrupt handlers only wake up
3376                  * clients, which may be sleeping on individual rings
3377                  * or on a global resource for all rings.
3378                  * To implement tx interrupt mitigation, we wake up the client
3379                  * thread roughly every half ring, even if the NIC interrupts
3380                  * more frequently. This is implemented as follows:
3381                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3382                  *   the slot that should wake up the thread (nkr_num_slots
3383                  *   means the user thread should not be woken up);
3384                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3385                  *   or the slot has the DD bit set.
3386                  *
3387                  * When the driver has separate locks, we need to
3388                  * release and re-acquire txlock to avoid deadlocks.
3389                  * XXX see if we can find a better way.
3390                  */
3391                 if (!netmap_mitigate ||
3392                     (kring->nr_kflags < kring->nkr_num_slots &&
3393                      tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3394                         kring->nr_kflags = kring->nkr_num_slots;
3395                         selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3396                         IXGBE_TX_UNLOCK(txr);
3397                         IXGBE_CORE_LOCK(adapter);
3398                         selwakeuppri(&na->tx_si, PI_NET);
3399                         IXGBE_CORE_UNLOCK(adapter);
3400                         IXGBE_TX_LOCK(txr);
3401                 }
3402                 return FALSE;
3403         }
3404 #endif /* DEV_NETMAP */
3405
3406         if (txr->tx_avail == adapter->num_tx_desc) {
3407                 txr->queue_status = IXGBE_QUEUE_IDLE;
3408                 return;
3409         }
3410
3411         processed = 0;
3412         first = txr->next_to_clean;
3413         tx_buffer = &txr->tx_buffers[first];
3414         /* For cleanup we just use legacy struct */
3415         tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3416         last = tx_buffer->eop_index;
3417         if (last == -1)
3418                 return;
3419         eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3420
3421         /*
3422         ** Get the index of the first descriptor
3423         ** BEYOND the EOP and call that 'done'.
3424         ** I do this so the comparison in the
3425         ** inner while loop below can be simple
3426         */
3427         if (++last == adapter->num_tx_desc) last = 0;
3428         done = last;
3429
3430         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3431             BUS_DMASYNC_POSTREAD);
3432         /*
3433         ** Only the EOP descriptor of a packet now has the DD
3434         ** bit set, this is what we look for...
3435         */
3436         while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3437                 /* We clean the range of the packet */
3438                 while (first != done) {
3439                         tx_desc->upper.data = 0;
3440                         tx_desc->lower.data = 0;
3441                         tx_desc->buffer_addr = 0;
3442                         ++txr->tx_avail;
3443                         ++processed;
3444
3445                         if (tx_buffer->m_head) {
3446                                 txr->bytes +=
3447                                     tx_buffer->m_head->m_pkthdr.len;
3448                                 bus_dmamap_sync(txr->txtag,
3449                                     tx_buffer->map,
3450                                     BUS_DMASYNC_POSTWRITE);
3451                                 bus_dmamap_unload(txr->txtag,
3452                                     tx_buffer->map);
3453                                 m_freem(tx_buffer->m_head);
3454                                 tx_buffer->m_head = NULL;
3455                                 tx_buffer->map = NULL;
3456                         }
3457                         tx_buffer->eop_index = -1;
3458                         txr->watchdog_time = ticks;
3459
3460                         if (++first == adapter->num_tx_desc)
3461                                 first = 0;
3462
3463                         tx_buffer = &txr->tx_buffers[first];
3464                         tx_desc =
3465                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3466                 }
3467                 ++txr->packets;
3468                 ++ifp->if_opackets;
3469                 /* See if there is more work now */
3470                 last = tx_buffer->eop_index;
3471                 if (last != -1) {
3472                         eop_desc =
3473                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3474                         /* Get next done point */
3475                         if (++last == adapter->num_tx_desc) last = 0;
3476                         done = last;
3477                 } else
3478                         break;
3479         }
3480         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3481             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3482
3483         txr->next_to_clean = first;
3484
3485         /*
3486         ** Watchdog calculation, we know there's
3487         ** work outstanding or the first return
3488         ** would have been taken, so none processed
3489         ** for too long indicates a hang.
3490         */
3491         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3492                 txr->queue_status = IXGBE_QUEUE_HUNG;
3493
3494         /* With a minimum free clear the depleted state bit.  */
3495         if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3496                 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3497
3498         if (txr->tx_avail == adapter->num_tx_desc) {
3499                 txr->queue_status = IXGBE_QUEUE_IDLE;
3500         }
3501 }
3502
3503 /*********************************************************************
3504  *
3505  *  Refresh mbuf buffers for RX descriptor rings
3506  *   - now keeps its own state so discards due to resource
3507  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3508  *     it just returns, keeping its placeholder, thus it can simply
3509  *     be recalled to try again.
3510  *
3511  **********************************************************************/
3512 static void
3513 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3514 {
3515         struct adapter          *adapter = rxr->adapter;
3516         bus_dma_segment_t       hseg[1];
3517         bus_dma_segment_t       pseg[1];
3518         struct ixgbe_rx_buf     *rxbuf;
3519         struct mbuf             *mh, *mp;
3520         int                     i, j, nsegs, error;
3521         bool                    refreshed = FALSE;
3522
3523         i = j = rxr->next_to_refresh;
3524         /* Control the loop with one beyond */
3525         if (++j == adapter->num_rx_desc)
3526                 j = 0;
3527
3528         while (j != limit) {
3529                 rxbuf = &rxr->rx_buffers[i];
3530                 if (rxr->hdr_split == FALSE)
3531                         goto no_split;
3532
3533                 if (rxbuf->m_head == NULL) {
3534                         mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3535                         if (mh == NULL)
3536                                 goto update;
3537                 } else
3538                         mh = rxbuf->m_head;
3539
3540                 mh->m_pkthdr.len = mh->m_len = MHLEN;
3541                 mh->m_len = MHLEN;
3542                 mh->m_flags |= M_PKTHDR;
3543                 /* Get the memory mapping */
3544                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3545                     rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3546                 if (error != 0) {
3547                         kprintf("Refresh mbufs: hdr dmamap load"
3548                             " failure - %d\n", error);
3549                         m_free(mh);
3550                         rxbuf->m_head = NULL;
3551                         goto update;
3552                 }
3553                 rxbuf->m_head = mh;
3554                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3555                     BUS_DMASYNC_PREREAD);
3556                 rxr->rx_base[i].read.hdr_addr =
3557                     htole64(hseg[0].ds_addr);
3558
3559 no_split:
3560                 if (rxbuf->m_pack == NULL) {
3561                         mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3562                             M_PKTHDR, adapter->rx_mbuf_sz);
3563                         if (mp == NULL)
3564                                 goto update;
3565                 } else
3566                         mp = rxbuf->m_pack;
3567
3568                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3569                 /* Get the memory mapping */
3570                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3571                     rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3572                 if (error != 0) {
3573                         kprintf("Refresh mbufs: payload dmamap load"
3574                             " failure - %d\n", error);
3575                         m_free(mp);
3576                         rxbuf->m_pack = NULL;
3577                         goto update;
3578                 }
3579                 rxbuf->m_pack = mp;
3580                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3581                     BUS_DMASYNC_PREREAD);
3582                 rxr->rx_base[i].read.pkt_addr =
3583                     htole64(pseg[0].ds_addr);
3584
3585                 refreshed = TRUE;
3586                 /* Next is precalculated */
3587                 i = j;
3588                 rxr->next_to_refresh = i;
3589                 if (++j == adapter->num_rx_desc)
3590                         j = 0;
3591         }
3592 update:
3593         if (refreshed) /* Update hardware tail index */
3594                 IXGBE_WRITE_REG(&adapter->hw,
3595                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3596         return;
3597 }
3598
3599 /*********************************************************************
3600  *
3601  *  Allocate memory for rx_buffer structures. Since we use one
3602  *  rx_buffer per received packet, the maximum number of rx_buffer's
3603  *  that we'll need is equal to the number of receive descriptors
3604  *  that we've allocated.
3605  *
3606  **********************************************************************/
3607 static int
3608 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3609 {
3610         struct  adapter         *adapter = rxr->adapter;
3611         device_t                dev = adapter->dev;
3612         struct ixgbe_rx_buf     *rxbuf;
3613         int                     i, bsize, error;
3614
3615         bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3616         if (!(rxr->rx_buffers =
3617             (struct ixgbe_rx_buf *) kmalloc(bsize,
3618             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3619                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3620                 error = ENOMEM;
3621                 goto fail;
3622         }
3623
3624         if ((error = bus_dma_tag_create(NULL,   /* parent */
3625                                    1, 0,        /* alignment, bounds */
3626                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3627                                    BUS_SPACE_MAXADDR,   /* highaddr */
3628                                    NULL, NULL,          /* filter, filterarg */
3629                                    MSIZE,               /* maxsize */
3630                                    1,                   /* nsegments */
3631                                    MSIZE,               /* maxsegsize */
3632                                    0,                   /* flags */
3633                                    &rxr->htag))) {
3634                 device_printf(dev, "Unable to create RX DMA tag\n");
3635                 goto fail;
3636         }
3637
3638         if ((error = bus_dma_tag_create(NULL,   /* parent */
3639                                    1, 0,        /* alignment, bounds */
3640                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3641                                    BUS_SPACE_MAXADDR,   /* highaddr */
3642                                    NULL, NULL,          /* filter, filterarg */
3643                                    MJUM16BYTES,         /* maxsize */
3644                                    1,                   /* nsegments */
3645                                    MJUM16BYTES,         /* maxsegsize */
3646                                    0,                   /* flags */
3647                                    &rxr->ptag))) {
3648                 device_printf(dev, "Unable to create RX DMA tag\n");
3649                 goto fail;
3650         }
3651
3652         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3653                 rxbuf = &rxr->rx_buffers[i];
3654                 error = bus_dmamap_create(rxr->htag,
3655                     BUS_DMA_NOWAIT, &rxbuf->hmap);
3656                 if (error) {
3657                         device_printf(dev, "Unable to create RX head map\n");
3658                         goto fail;
3659                 }
3660                 error = bus_dmamap_create(rxr->ptag,
3661                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3662                 if (error) {
3663                         device_printf(dev, "Unable to create RX pkt map\n");
3664                         goto fail;
3665                 }
3666         }
3667
3668         return (0);
3669
3670 fail:
3671         /* Frees all, but can handle partial completion */
3672         ixgbe_free_receive_structures(adapter);
3673         return (error);
3674 }
3675
3676 /*
3677 ** Used to detect a descriptor that has
3678 ** been merged by Hardware RSC.
3679 */
3680 static inline u32
3681 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3682 {
3683         return (le32toh(rx->wb.lower.lo_dword.data) &
3684             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3685 }
3686
3687 /*********************************************************************
3688  *
3689  *  Initialize Hardware RSC (LRO) feature on 82599
3690  *  for an RX ring, this is toggled by the LRO capability
3691  *  even though it is transparent to the stack.
3692  *
3693  **********************************************************************/
3694 #if 0   /* NET_LRO */
3695 static void
3696 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3697 {
3698         struct  adapter         *adapter = rxr->adapter;
3699         struct  ixgbe_hw        *hw = &adapter->hw;
3700         u32                     rscctrl, rdrxctl;
3701
3702         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3703         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3704 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3705         if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3706 #endif /* DEV_NETMAP */
3707         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3708         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3709         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3710
3711         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3712         rscctrl |= IXGBE_RSCCTL_RSCEN;
3713         /*
3714         ** Limit the total number of descriptors that
3715         ** can be combined, so it does not exceed 64K
3716         */
3717         if (adapter->rx_mbuf_sz == MCLBYTES)
3718                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3719         else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3720                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3721         else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3722                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3723         else  /* Using 16K cluster */
3724                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3725
3726         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3727
3728         /* Enable TCP header recognition */
3729         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3730             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3731             IXGBE_PSRTYPE_TCPHDR));
3732
3733         /* Disable RSC for ACK packets */
3734         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3735             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3736
3737         rxr->hw_rsc = TRUE;
3738 }
3739 #endif
3740
3741 static void
3742 ixgbe_free_receive_ring(struct rx_ring *rxr)
3743 {
3744         struct  adapter         *adapter;
3745         struct ixgbe_rx_buf       *rxbuf;
3746         int i;
3747
3748         adapter = rxr->adapter;
3749         for (i = 0; i < adapter->num_rx_desc; i++) {
3750                 rxbuf = &rxr->rx_buffers[i];
3751                 if (rxbuf->m_head != NULL) {
3752                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3753                             BUS_DMASYNC_POSTREAD);
3754                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3755                         rxbuf->m_head->m_flags |= M_PKTHDR;
3756                         m_freem(rxbuf->m_head);
3757                 }
3758                 if (rxbuf->m_pack != NULL) {
3759                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3760                             BUS_DMASYNC_POSTREAD);
3761                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3762                         rxbuf->m_pack->m_flags |= M_PKTHDR;
3763                         m_freem(rxbuf->m_pack);
3764                 }
3765                 rxbuf->m_head = NULL;
3766                 rxbuf->m_pack = NULL;
3767         }
3768 }
3769
3770
3771 /*********************************************************************
3772  *
3773  *  Initialize a receive ring and its buffers.
3774  *
3775  **********************************************************************/
3776 static int
3777 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3778 {
3779         struct  adapter         *adapter;
3780         struct ifnet            *ifp;
3781         device_t                dev;
3782         struct ixgbe_rx_buf     *rxbuf;
3783         bus_dma_segment_t       pseg[1], hseg[1];
3784 #if 0   /* NET_LRO */
3785         struct lro_ctrl         *lro = &rxr->lro;
3786 #endif
3787         int                     rsize, nsegs, error = 0;
3788 #ifdef DEV_NETMAP
3789         struct netmap_adapter *na = NA(rxr->adapter->ifp);
3790         struct netmap_slot *slot;
3791 #endif /* DEV_NETMAP */
3792
3793         adapter = rxr->adapter;
3794         ifp = adapter->ifp;
3795         dev = adapter->dev;
3796
3797         /* Clear the ring contents */
3798         IXGBE_RX_LOCK(rxr);
3799 #ifdef DEV_NETMAP
3800         /* same as in ixgbe_setup_transmit_ring() */
3801         slot = netmap_reset(na, NR_RX, rxr->me, 0);
3802 #endif /* DEV_NETMAP */
3803         rsize = roundup2(adapter->num_rx_desc *
3804             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3805         bzero((void *)rxr->rx_base, rsize);
3806
3807         /* Free current RX buffer structs and their mbufs */
3808         ixgbe_free_receive_ring(rxr);
3809
3810         /* Configure header split? */
3811         if (ixgbe_header_split)
3812                 rxr->hdr_split = TRUE;
3813
3814         /* Now replenish the mbufs */
3815         for (int j = 0; j != adapter->num_rx_desc; ++j) {
3816                 struct mbuf     *mh, *mp;
3817
3818                 rxbuf = &rxr->rx_buffers[j];
3819 #ifdef DEV_NETMAP
3820                 /*
3821                  * In netmap mode, fill the map and set the buffer
3822                  * address in the NIC ring, considering the offset
3823                  * between the netmap and NIC rings (see comment in
3824                  * ixgbe_setup_transmit_ring() ). No need to allocate
3825                  * an mbuf, so end the block with a continue;
3826                  */
3827                 if (slot) {
3828                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
3829                         uint64_t paddr;
3830                         void *addr;
3831
3832                         addr = PNMB(slot + sj, &paddr);
3833                         netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
3834                         /* Update descriptor */
3835                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
3836                         continue;
3837                 }
3838 #endif /* DEV_NETMAP */
3839                 /*
3840                 ** Don't allocate mbufs if not
3841                 ** doing header split, its wasteful
3842                 */
3843                 if (rxr->hdr_split == FALSE)
3844                         goto skip_head;
3845
3846                 /* First the header */
3847                 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
3848                 if (rxbuf->m_head == NULL) {
3849                         error = ENOBUFS;
3850                         goto fail;
3851                 }
3852                 m_adj(rxbuf->m_head, ETHER_ALIGN);
3853                 mh = rxbuf->m_head;
3854                 mh->m_len = mh->m_pkthdr.len = MHLEN;
3855                 mh->m_flags |= M_PKTHDR;
3856                 /* Get the memory mapping */
3857                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3858                     rxbuf->hmap, rxbuf->m_head, hseg, 1,
3859                     &nsegs, BUS_DMA_NOWAIT);
3860
3861                 if (error != 0) /* Nothing elegant to do here */
3862                         goto fail;
3863                 bus_dmamap_sync(rxr->htag,
3864                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
3865                 /* Update descriptor */
3866                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
3867
3868 skip_head:
3869                 /* Now the payload cluster */
3870                 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
3871                     M_PKTHDR, adapter->rx_mbuf_sz);
3872                 if (rxbuf->m_pack == NULL) {
3873                         error = ENOBUFS;
3874                         goto fail;
3875                 }
3876                 mp = rxbuf->m_pack;
3877                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3878                 /* Get the memory mapping */
3879                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3880                     rxbuf->pmap, mp, hseg, 1,
3881                     &nsegs, BUS_DMA_NOWAIT);
3882                 if (error != 0)
3883                         goto fail;
3884                 bus_dmamap_sync(rxr->ptag,
3885                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
3886                 /* Update descriptor */
3887                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
3888         }
3889
3890
3891         /* Setup our descriptor indices */
3892         rxr->next_to_check = 0;
3893         rxr->next_to_refresh = 0;
3894         rxr->lro_enabled = FALSE;
3895         rxr->rx_split_packets = 0;
3896         rxr->rx_bytes = 0;
3897         rxr->discard = FALSE;
3898         rxr->vtag_strip = FALSE;
3899
3900         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3901             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3902
3903         /*
3904         ** Now set up the LRO interface:
3905         ** 82598 uses software LRO, the
3906         ** 82599 and X540 use a hardware assist.
3907         */
3908 #if 0 /* NET_LRO */
3909         if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
3910             (ifp->if_capenable & IFCAP_RXCSUM) &&
3911             (ifp->if_capenable & IFCAP_LRO))
3912                 ixgbe_setup_hw_rsc(rxr);
3913         else if (ifp->if_capenable & IFCAP_LRO) {
3914                 int err = tcp_lro_init(lro);
3915                 if (err) {
3916                         device_printf(dev, "LRO Initialization failed!\n");
3917                         goto fail;
3918                 }
3919                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
3920                 rxr->lro_enabled = TRUE;
3921                 lro->ifp = adapter->ifp;
3922         }
3923 #endif
3924
3925         IXGBE_RX_UNLOCK(rxr);
3926         return (0);
3927
3928 fail:
3929         ixgbe_free_receive_ring(rxr);
3930         IXGBE_RX_UNLOCK(rxr);
3931         return (error);
3932 }
3933
3934 /*********************************************************************
3935  *
3936  *  Initialize all receive rings.
3937  *
3938  **********************************************************************/
3939 static int
3940 ixgbe_setup_receive_structures(struct adapter *adapter)
3941 {
3942         struct rx_ring *rxr = adapter->rx_rings;
3943         int j;
3944
3945         for (j = 0; j < adapter->num_queues; j++, rxr++)
3946                 if (ixgbe_setup_receive_ring(rxr))
3947                         goto fail;
3948
3949         return (0);
3950 fail:
3951         /*
3952          * Free RX buffers allocated so far, we will only handle
3953          * the rings that completed, the failing case will have
3954          * cleaned up for itself. 'j' failed, so its the terminus.
3955          */
3956         for (int i = 0; i < j; ++i) {
3957                 rxr = &adapter->rx_rings[i];
3958                 ixgbe_free_receive_ring(rxr);
3959         }
3960
3961         return (ENOBUFS);
3962 }
3963
3964 /*********************************************************************
3965  *
3966  *  Setup receive registers and features.
3967  *
3968  **********************************************************************/
3969 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
3970
3971 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
3972
3973 static void
3974 ixgbe_initialize_receive_units(struct adapter *adapter)
3975 {
3976         struct  rx_ring *rxr = adapter->rx_rings;
3977         struct ixgbe_hw *hw = &adapter->hw;
3978         struct ifnet   *ifp = adapter->ifp;
3979         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
3980         u32             reta, mrqc = 0, hlreg, random[10];
3981
3982
3983         /*
3984          * Make sure receives are disabled while
3985          * setting up the descriptor ring
3986          */
3987         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
3988         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
3989             rxctrl & ~IXGBE_RXCTRL_RXEN);
3990
3991         /* Enable broadcasts */
3992         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3993         fctrl |= IXGBE_FCTRL_BAM;
3994         fctrl |= IXGBE_FCTRL_DPF;
3995         fctrl |= IXGBE_FCTRL_PMCF;
3996         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
3997
3998         /* Set for Jumbo Frames? */
3999         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4000         if (ifp->if_mtu > ETHERMTU)
4001                 hlreg |= IXGBE_HLREG0_JUMBOEN;
4002         else
4003                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4004 #ifdef DEV_NETMAP
4005         /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4006         if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4007                 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4008         else
4009                 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4010 #endif /* DEV_NETMAP */
4011         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4012
4013         bufsz = (adapter->rx_mbuf_sz +
4014             BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4015
4016         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4017                 u64 rdba = rxr->rxdma.dma_paddr;
4018
4019                 /* Setup the Base and Length of the Rx Descriptor Ring */
4020                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4021                                (rdba & 0x00000000ffffffffULL));
4022                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4023                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4024                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4025
4026                 /* Set up the SRRCTL register */
4027                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4028                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4029                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4030                 srrctl |= bufsz;
4031                 if (rxr->hdr_split) {
4032                         /* Use a standard mbuf for the header */
4033                         srrctl |= ((IXGBE_RX_HDR <<
4034                             IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
4035                             & IXGBE_SRRCTL_BSIZEHDR_MASK);
4036                         srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4037                 } else
4038                         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4039                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4040
4041                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4042                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4043                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4044         }
4045
4046         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4047                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4048                               IXGBE_PSRTYPE_UDPHDR |
4049                               IXGBE_PSRTYPE_IPV4HDR |
4050                               IXGBE_PSRTYPE_IPV6HDR;
4051                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4052         }
4053
4054         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4055
4056         /* Setup RSS */
4057         if (adapter->num_queues > 1) {
4058                 int i, j;
4059                 reta = 0;
4060
4061                 /* set up random bits */
4062                 karc4rand(&random, sizeof(random));
4063
4064                 /* Set up the redirection table */
4065                 for (i = 0, j = 0; i < 128; i++, j++) {
4066                         if (j == adapter->num_queues) j = 0;
4067                         reta = (reta << 8) | (j * 0x11);
4068                         if ((i & 3) == 3)
4069                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4070                 }
4071
4072                 /* Now fill our hash function seeds */
4073                 for (int i = 0; i < 10; i++)
4074                         IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4075
4076                 /* Perform hash on these packet types */
4077                 mrqc = IXGBE_MRQC_RSSEN
4078                      | IXGBE_MRQC_RSS_FIELD_IPV4
4079                      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4080                      | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4081                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4082                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4083                      | IXGBE_MRQC_RSS_FIELD_IPV6
4084                      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4085                      | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4086                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4087                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4088
4089                 /* RSS and RX IPP Checksum are mutually exclusive */
4090                 rxcsum |= IXGBE_RXCSUM_PCSD;
4091         }
4092
4093         if (ifp->if_capenable & IFCAP_RXCSUM)
4094                 rxcsum |= IXGBE_RXCSUM_PCSD;
4095
4096         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4097                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4098
4099         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4100
4101         return;
4102 }
4103
4104 /*********************************************************************
4105  *
4106  *  Free all receive rings.
4107  *
4108  **********************************************************************/
4109 static void
4110 ixgbe_free_receive_structures(struct adapter *adapter)
4111 {
4112         struct rx_ring *rxr = adapter->rx_rings;
4113
4114         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4115 #if 0   /* NET_LRO */
4116                 struct lro_ctrl         *lro = &rxr->lro;
4117 #endif
4118                 ixgbe_free_receive_buffers(rxr);
4119                 /* Free LRO memory */
4120 #if 0   /* NET_LRO */
4121                 tcp_lro_free(lro);
4122 #endif
4123                 /* Free the ring memory as well */
4124                 ixgbe_dma_free(adapter, &rxr->rxdma);
4125         }
4126
4127         kfree(adapter->rx_rings, M_DEVBUF);
4128 }
4129
4130
4131 /*********************************************************************
4132  *
4133  *  Free receive ring data structures
4134  *
4135  **********************************************************************/
4136 static void
4137 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4138 {
4139         struct adapter          *adapter = rxr->adapter;
4140         struct ixgbe_rx_buf     *rxbuf;
4141
4142         INIT_DEBUGOUT("free_receive_structures: begin");
4143
4144         /* Cleanup any existing buffers */
4145         if (rxr->rx_buffers != NULL) {
4146                 for (int i = 0; i < adapter->num_rx_desc; i++) {
4147                         rxbuf = &rxr->rx_buffers[i];
4148                         if (rxbuf->m_head != NULL) {
4149                                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4150                                     BUS_DMASYNC_POSTREAD);
4151                                 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4152                                 rxbuf->m_head->m_flags |= M_PKTHDR;
4153                                 m_freem(rxbuf->m_head);
4154                         }
4155                         if (rxbuf->m_pack != NULL) {
4156                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4157                                     BUS_DMASYNC_POSTREAD);
4158                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4159                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
4160                                 m_freem(rxbuf->m_pack);
4161                         }
4162                         rxbuf->m_head = NULL;
4163                         rxbuf->m_pack = NULL;
4164                         if (rxbuf->hmap != NULL) {
4165                                 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4166                                 rxbuf->hmap = NULL;
4167                         }
4168                         if (rxbuf->pmap != NULL) {
4169                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4170                                 rxbuf->pmap = NULL;
4171                         }
4172                 }
4173                 if (rxr->rx_buffers != NULL) {
4174                         kfree(rxr->rx_buffers, M_DEVBUF);
4175                         rxr->rx_buffers = NULL;
4176                 }
4177         }
4178
4179         if (rxr->htag != NULL) {
4180                 bus_dma_tag_destroy(rxr->htag);
4181                 rxr->htag = NULL;
4182         }
4183         if (rxr->ptag != NULL) {
4184                 bus_dma_tag_destroy(rxr->ptag);
4185                 rxr->ptag = NULL;
4186         }
4187
4188         return;
4189 }
4190
4191 static __inline void
4192 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4193 {
4194
4195         /*
4196          * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4197          * should be computed by hardware. Also it should not have VLAN tag in
4198          * ethernet header.  In case of IPv6 we do not yet support ext. hdrs.
4199          */
4200 #if 0   /* NET_LRO */
4201         if (rxr->lro_enabled &&
4202             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4203             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4204             ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4205             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4206             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4207             (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4208             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4209             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4210                 /*
4211                  * Send to the stack if:
4212                  **  - LRO not enabled, or
4213                  **  - no LRO resources, or
4214                  **  - lro enqueue fails
4215                  */
4216                 if (rxr->lro.lro_cnt != 0)
4217                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4218                                 return;
4219         }
4220 #endif
4221         IXGBE_RX_UNLOCK(rxr);
4222         (*ifp->if_input)(ifp, m);
4223         IXGBE_RX_LOCK(rxr);
4224 }
4225
4226 static __inline void
4227 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4228 {
4229         struct ixgbe_rx_buf     *rbuf;
4230
4231         rbuf = &rxr->rx_buffers[i];
4232
4233         if (rbuf->fmp != NULL) {/* Partial chain ? */
4234                 rbuf->fmp->m_flags |= M_PKTHDR;
4235                 m_freem(rbuf->fmp);
4236                 rbuf->fmp = NULL;
4237         }
4238
4239         /*
4240         ** With advanced descriptors the writeback
4241         ** clobbers the buffer addrs, so its easier
4242         ** to just free the existing mbufs and take
4243         ** the normal refresh path to get new buffers
4244         ** and mapping.
4245         */
4246         if (rbuf->m_head) {
4247                 m_free(rbuf->m_head);
4248                 rbuf->m_head = NULL;
4249         }
4250
4251         if (rbuf->m_pack) {
4252                 m_free(rbuf->m_pack);
4253                 rbuf->m_pack = NULL;
4254         }
4255
4256         return;
4257 }
4258
4259 static void
4260 ixgbe_add_sysctl(struct adapter *adapter)
4261 {
4262         sysctl_ctx_init(&adapter->sysctl_ctx);
4263         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
4264             SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
4265             device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
4266         if (adapter->sysctl_tree == NULL) {
4267                 device_printf(adapter->dev, "can't add sysctl node\n");
4268                 return;
4269         }
4270         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4271                         SYSCTL_CHILDREN(adapter->sysctl_tree),
4272                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
4273                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
4274
4275         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4276             SYSCTL_CHILDREN(adapter->sysctl_tree),
4277             OID_AUTO, "intr_rate", CTLTYPE_INT | CTLFLAG_RW,
4278             adapter, 0, ixgbe_sysctl_intr_rate, "I", "interrupt rate");
4279
4280         /*
4281         ** Allow a kind of speed control by forcing the autoneg
4282         ** advertised speed list to only a certain value, this
4283         ** supports 1G on 82599 devices, and 100Mb on x540.
4284         */
4285         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4286                         SYSCTL_CHILDREN(adapter->sysctl_tree),
4287                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
4288                         adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
4289
4290         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4291                         SYSCTL_CHILDREN(adapter->sysctl_tree),
4292                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
4293                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
4294
4295         /* Sysctl for limiting the amount of work done in the taskqueue */
4296         ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
4297             "max number of rx packets to process", &adapter->rx_process_limit,
4298             ixgbe_rx_process_limit);
4299 }
4300
4301 /*********************************************************************
4302  *
4303  *  This routine executes in interrupt context. It replenishes
4304  *  the mbufs in the descriptor and sends data which has been
4305  *  dma'ed into host memory to upper layer.
4306  *
4307  *  We loop at most count times if count is > 0, or until done if
4308  *  count < 0.
4309  *
4310  *  Return TRUE for more work, FALSE for all clean.
4311  *********************************************************************/
4312 static void
4313 ixgbe_rxeof(struct ix_queue *que, int count)
4314 {
4315         struct adapter          *adapter = que->adapter;
4316         struct rx_ring          *rxr = que->rxr;
4317         struct ifnet            *ifp = adapter->ifp;
4318 #if 0   /* NET_LRO */
4319         struct lro_ctrl         *lro = &rxr->lro;
4320         struct lro_entry        *queued;
4321 #endif
4322         int                     i, nextp, processed = 0;
4323         u32                     staterr = 0;
4324         union ixgbe_adv_rx_desc *cur;
4325         struct ixgbe_rx_buf     *rbuf, *nbuf;
4326
4327         IXGBE_RX_LOCK(rxr);
4328
4329 #ifdef DEV_NETMAP
4330         if (ifp->if_capenable & IFCAP_NETMAP) {
4331                 /*
4332                  * Same as the txeof routine: only wakeup clients on intr.
4333                  * NKR_PENDINTR in nr_kflags is used to implement interrupt
4334                  * mitigation (ixgbe_rxsync() will not look for new packets
4335                  * unless NKR_PENDINTR is set).
4336                  */
4337                 struct netmap_adapter *na = NA(ifp);
4338
4339                 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4340                 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4341                 IXGBE_RX_UNLOCK(rxr);
4342                 IXGBE_CORE_LOCK(adapter);
4343                 selwakeuppri(&na->rx_si, PI_NET);
4344                 IXGBE_CORE_UNLOCK(adapter);
4345                 return (FALSE);
4346         }
4347 #endif /* DEV_NETMAP */
4348         for (i = rxr->next_to_check; count != 0;) {
4349                 struct mbuf     *sendmp, *mh, *mp;
4350                 u32             rsc, ptype;
4351                 u16             hlen, plen, hdr;
4352                 u16             vtag = 0;
4353                 bool            eop;
4354
4355                 /* Sync the ring. */
4356                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4357                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4358
4359                 cur = &rxr->rx_base[i];
4360                 staterr = le32toh(cur->wb.upper.status_error);
4361
4362                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4363                         break;
4364                 if ((ifp->if_flags & IFF_RUNNING) == 0)
4365                         break;
4366
4367                 count--;
4368                 sendmp = NULL;
4369                 nbuf = NULL;
4370                 rsc = 0;
4371                 cur->wb.upper.status_error = 0;
4372                 rbuf = &rxr->rx_buffers[i];
4373                 mh = rbuf->m_head;
4374                 mp = rbuf->m_pack;
4375
4376                 plen = le16toh(cur->wb.upper.length);
4377                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4378                     IXGBE_RXDADV_PKTTYPE_MASK;
4379                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4380                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4381
4382                 /* Process vlan info */
4383                 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
4384                         vtag = le16toh(cur->wb.upper.vlan);
4385
4386                 /* Make sure bad packets are discarded */
4387                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4388                     (rxr->discard)) {
4389                         ifp->if_ierrors++;
4390                         rxr->rx_discarded++;
4391                         if (eop)
4392                                 rxr->discard = FALSE;
4393                         else
4394                                 rxr->discard = TRUE;
4395                         ixgbe_rx_discard(rxr, i);
4396                         goto next_desc;
4397                 }
4398
4399                 /*
4400                 ** On 82599 which supports a hardware
4401                 ** LRO (called HW RSC), packets need
4402                 ** not be fragmented across sequential
4403                 ** descriptors, rather the next descriptor
4404                 ** is indicated in bits of the descriptor.
4405                 ** This also means that we might proceses
4406                 ** more than one packet at a time, something
4407                 ** that has never been true before, it
4408                 ** required eliminating global chain pointers
4409                 ** in favor of what we are doing here.  -jfv
4410                 */
4411                 if (!eop) {
4412                         /*
4413                         ** Figure out the next descriptor
4414                         ** of this frame.
4415                         */
4416                         if (rxr->hw_rsc == TRUE) {
4417                                 rsc = ixgbe_rsc_count(cur);
4418                                 rxr->rsc_num += (rsc - 1);
4419                         }
4420                         if (rsc) { /* Get hardware index */
4421                                 nextp = ((staterr &
4422                                     IXGBE_RXDADV_NEXTP_MASK) >>
4423                                     IXGBE_RXDADV_NEXTP_SHIFT);
4424                         } else { /* Just sequential */
4425                                 nextp = i + 1;
4426                                 if (nextp == adapter->num_rx_desc)
4427                                         nextp = 0;
4428                         }
4429                         nbuf = &rxr->rx_buffers[nextp];
4430                         prefetch(nbuf);
4431                 }
4432                 /*
4433                 ** The header mbuf is ONLY used when header
4434                 ** split is enabled, otherwise we get normal
4435                 ** behavior, ie, both header and payload
4436                 ** are DMA'd into the payload buffer.
4437                 **
4438                 ** Rather than using the fmp/lmp global pointers
4439                 ** we now keep the head of a packet chain in the
4440                 ** buffer struct and pass this along from one
4441                 ** descriptor to the next, until we get EOP.
4442                 */
4443                 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4444                         /* This must be an initial descriptor */
4445                         hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4446                             IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4447                         if (hlen > IXGBE_RX_HDR)
4448                                 hlen = IXGBE_RX_HDR;
4449                         mh->m_len = hlen;
4450                         mh->m_flags |= M_PKTHDR;
4451                         mh->m_next = NULL;
4452                         mh->m_pkthdr.len = mh->m_len;
4453                         /* Null buf pointer so it is refreshed */
4454                         rbuf->m_head = NULL;
4455                         /*
4456                         ** Check the payload length, this
4457                         ** could be zero if its a small
4458                         ** packet.
4459                         */
4460                         if (plen > 0) {
4461                                 mp->m_len = plen;
4462                                 mp->m_next = NULL;
4463                                 mp->m_flags &= ~M_PKTHDR;
4464                                 mh->m_next = mp;
4465                                 mh->m_pkthdr.len += mp->m_len;
4466                                 /* Null buf pointer so it is refreshed */
4467                                 rbuf->m_pack = NULL;
4468                                 rxr->rx_split_packets++;
4469                         }
4470                         /*
4471                         ** Now create the forward
4472                         ** chain so when complete
4473                         ** we wont have to.
4474                         */
4475                         if (eop == 0) {
4476                                 /* stash the chain head */
4477                                 nbuf->fmp = mh;
4478                                 /* Make forward chain */
4479                                 if (plen)
4480                                         mp->m_next = nbuf->m_pack;
4481                                 else
4482                                         mh->m_next = nbuf->m_pack;
4483                         } else {
4484                                 /* Singlet, prepare to send */
4485                                 sendmp = mh;
4486                                 /* If hardware handled vtag */
4487                                 if (vtag) {
4488                                         sendmp->m_pkthdr.ether_vlantag = vtag;
4489                                         sendmp->m_flags |= M_VLANTAG;
4490                                 }
4491                         }
4492                 } else {
4493                         /*
4494                         ** Either no header split, or a
4495                         ** secondary piece of a fragmented
4496                         ** split packet.
4497                         */
4498                         mp->m_len = plen;
4499                         /*
4500                         ** See if there is a stored head
4501                         ** that determines what we are
4502                         */
4503                         sendmp = rbuf->fmp;
4504                         rbuf->m_pack = rbuf->fmp = NULL;
4505
4506                         if (sendmp != NULL) {  /* secondary frag */
4507                                 mp->m_flags &= ~M_PKTHDR;
4508                                 sendmp->m_pkthdr.len += mp->m_len;
4509                         } else {
4510                                 /* first desc of a non-ps chain */
4511                                 sendmp = mp;
4512                                 sendmp->m_flags |= M_PKTHDR;
4513                                 sendmp->m_pkthdr.len = mp->m_len;
4514                                 if (staterr & IXGBE_RXD_STAT_VP) {
4515                                         sendmp->m_pkthdr.ether_vlantag = vtag;
4516                                         sendmp->m_flags |= M_VLANTAG;
4517                                 }
4518                         }
4519                         /* Pass the head pointer on */
4520                         if (eop == 0) {
4521                                 nbuf->fmp = sendmp;
4522                                 sendmp = NULL;
4523                                 mp->m_next = nbuf->m_pack;
4524                         }
4525                 }
4526                 ++processed;
4527                 /* Sending this frame? */
4528                 if (eop) {
4529                         sendmp->m_pkthdr.rcvif = ifp;
4530                         ifp->if_ipackets++;
4531                         rxr->rx_packets++;
4532                         /* capture data for AIM */
4533                         rxr->bytes += sendmp->m_pkthdr.len;
4534                         rxr->rx_bytes += sendmp->m_pkthdr.len;
4535                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4536                                 ixgbe_rx_checksum(staterr, sendmp, ptype);
4537 #if 0 /* __FreeBSD_version >= 800000 */
4538                         sendmp->m_pkthdr.flowid = que->msix;
4539                         sendmp->m_flags |= M_FLOWID;
4540 #endif
4541                 }
4542 next_desc:
4543                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4544                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4545
4546                 /* Advance our pointers to the next descriptor. */
4547                 if (++i == adapter->num_rx_desc)
4548                         i = 0;
4549
4550                 /* Now send to the stack or do LRO */
4551                 if (sendmp != NULL) {
4552                         rxr->next_to_check = i;
4553                         ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4554                         i = rxr->next_to_check;
4555                 }
4556
4557                /* Every 8 descriptors we go to refresh mbufs */
4558                 if (processed == 8) {
4559                         ixgbe_refresh_mbufs(rxr, i);
4560                         processed = 0;
4561                 }
4562         }
4563
4564         /* Refresh any remaining buf structs */
4565         if (ixgbe_rx_unrefreshed(rxr))
4566                 ixgbe_refresh_mbufs(rxr, i);
4567
4568         rxr->next_to_check = i;
4569
4570         /*
4571          * Flush any outstanding LRO work
4572          */
4573 #if 0   /* NET_LRO */
4574         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4575                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4576                 tcp_lro_flush(lro, queued);
4577         }
4578 #endif
4579
4580         IXGBE_RX_UNLOCK(rxr);
4581
4582         /*
4583         ** We still have cleaning to do?
4584         ** Schedule another interrupt if so.
4585         */
4586         if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4587                 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4588         }
4589 }
4590
4591
4592 /*********************************************************************
4593  *
4594  *  Verify that the hardware indicated that the checksum is valid.
4595  *  Inform the stack about the status of checksum so that stack
4596  *  doesn't spend time verifying the checksum.
4597  *
4598  *********************************************************************/
4599 static void
4600 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4601 {
4602         u16     status = (u16) staterr;
4603         u8      errors = (u8) (staterr >> 24);
4604         bool    sctp = FALSE;
4605
4606         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4607             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4608                 sctp = TRUE;
4609
4610         if (status & IXGBE_RXD_STAT_IPCS) {
4611                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4612                         /* IP Checksum Good */
4613                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4614                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4615
4616                 } else
4617                         mp->m_pkthdr.csum_flags = 0;
4618         }
4619         if (status & IXGBE_RXD_STAT_L4CS) {
4620                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4621 #if 0
4622                 if (sctp)
4623                         type = CSUM_SCTP_VALID;
4624 #endif
4625                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4626                         mp->m_pkthdr.csum_flags |= type;
4627                         if (!sctp)
4628                                 mp->m_pkthdr.csum_data = htons(0xffff);
4629                 }
4630         }
4631         return;
4632 }
4633
4634
4635 /*
4636 ** This routine is run via an vlan config EVENT,
4637 ** it enables us to use the HW Filter table since
4638 ** we can get the vlan id. This just creates the
4639 ** entry in the soft version of the VFTA, init will
4640 ** repopulate the real table.
4641 */
4642 static void
4643 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4644 {
4645         struct adapter  *adapter = ifp->if_softc;
4646         u16             index, bit;
4647
4648         if (ifp->if_softc !=  arg)   /* Not our event */
4649                 return;
4650
4651         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4652                 return;
4653
4654         IXGBE_CORE_LOCK(adapter);
4655         index = (vtag >> 5) & 0x7F;
4656         bit = vtag & 0x1F;
4657         adapter->shadow_vfta[index] |= (1 << bit);
4658         ++adapter->num_vlans;
4659         ixgbe_init_locked(adapter);
4660         IXGBE_CORE_UNLOCK(adapter);
4661 }
4662
4663 /*
4664 ** This routine is run via an vlan
4665 ** unconfig EVENT, remove our entry
4666 ** in the soft vfta.
4667 */
4668 static void
4669 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4670 {
4671         struct adapter  *adapter = ifp->if_softc;
4672         u16             index, bit;
4673
4674         if (ifp->if_softc !=  arg)
4675                 return;
4676
4677         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4678                 return;
4679
4680         IXGBE_CORE_LOCK(adapter);
4681         index = (vtag >> 5) & 0x7F;
4682         bit = vtag & 0x1F;
4683         adapter->shadow_vfta[index] &= ~(1 << bit);
4684         --adapter->num_vlans;
4685         /* Re-init to load the changes */
4686         ixgbe_init_locked(adapter);
4687         IXGBE_CORE_UNLOCK(adapter);
4688 }
4689
4690 static void
4691 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4692 {
4693         struct ifnet    *ifp = adapter->ifp;
4694         struct ixgbe_hw *hw = &adapter->hw;
4695         struct rx_ring  *rxr;
4696         u32             ctrl;
4697
4698         /*
4699         ** We get here thru init_locked, meaning
4700         ** a soft reset, this has already cleared
4701         ** the VFTA and other state, so if there
4702         ** have been no vlan's registered do nothing.
4703         */
4704         if (adapter->num_vlans == 0)
4705                 return;
4706
4707         /*
4708         ** A soft reset zero's out the VFTA, so
4709         ** we need to repopulate it now.
4710         */
4711         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4712                 if (adapter->shadow_vfta[i] != 0)
4713                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4714                             adapter->shadow_vfta[i]);
4715
4716         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4717         /* Enable the Filter Table if enabled */
4718         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4719                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4720                 ctrl |= IXGBE_VLNCTRL_VFE;
4721         }
4722         if (hw->mac.type == ixgbe_mac_82598EB)
4723                 ctrl |= IXGBE_VLNCTRL_VME;
4724         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4725
4726         /* Setup the queues for vlans */
4727         for (int i = 0; i < adapter->num_queues; i++) {
4728                 rxr = &adapter->rx_rings[i];
4729                 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4730                 if (hw->mac.type != ixgbe_mac_82598EB) {
4731                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4732                         ctrl |= IXGBE_RXDCTL_VME;
4733                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4734                 }
4735                 rxr->vtag_strip = TRUE;
4736         }
4737 }
4738
4739 static void
4740 ixgbe_enable_intr(struct adapter *adapter)
4741 {
4742         struct ixgbe_hw *hw = &adapter->hw;
4743         struct ix_queue *que = adapter->queues;
4744         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4745
4746
4747         /* Enable Fan Failure detection */
4748         if (hw->device_id == IXGBE_DEV_ID_82598AT)
4749                     mask |= IXGBE_EIMS_GPI_SDP1;
4750         else {
4751                     mask |= IXGBE_EIMS_ECC;
4752                     mask |= IXGBE_EIMS_GPI_SDP0;
4753                     mask |= IXGBE_EIMS_GPI_SDP1;
4754                     mask |= IXGBE_EIMS_GPI_SDP2;
4755 #ifdef IXGBE_FDIR
4756                     mask |= IXGBE_EIMS_FLOW_DIR;
4757 #endif
4758         }
4759
4760         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4761
4762         /* With RSS we use auto clear */
4763         if (adapter->msix_mem) {
4764                 mask = IXGBE_EIMS_ENABLE_MASK;
4765                 /* Don't autoclear Link */
4766                 mask &= ~IXGBE_EIMS_OTHER;
4767                 mask &= ~IXGBE_EIMS_LSC;
4768                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4769         }
4770
4771         /*
4772         ** Now enable all queues, this is done separately to
4773         ** allow for handling the extended (beyond 32) MSIX
4774         ** vectors that can be used by 82599
4775         */
4776         for (int i = 0; i < adapter->num_queues; i++, que++)
4777                 ixgbe_enable_queue(adapter, que->msix);
4778
4779         IXGBE_WRITE_FLUSH(hw);
4780
4781         return;
4782 }
4783
4784 static void
4785 ixgbe_disable_intr(struct adapter *adapter)
4786 {
4787         if (adapter->msix_mem)
4788                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4789         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4790                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4791         } else {
4792                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4793                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4794                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4795         }
4796         IXGBE_WRITE_FLUSH(&adapter->hw);
4797         return;
4798 }
4799
4800 u16
4801 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4802 {
4803         u16 value;
4804
4805         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4806             reg, 2);
4807
4808         return (value);
4809 }
4810
4811 void
4812 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4813 {
4814         pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4815             reg, value, 2);
4816
4817         return;
4818 }
4819
4820 /*
4821 ** Setup the correct IVAR register for a particular MSIX interrupt
4822 **   (yes this is all very magic and confusing :)
4823 **  - entry is the register array entry
4824 **  - vector is the MSIX vector for this queue
4825 **  - type is RX/TX/MISC
4826 */
4827 static void
4828 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4829 {
4830         struct ixgbe_hw *hw = &adapter->hw;
4831         u32 ivar, index;
4832
4833         vector |= IXGBE_IVAR_ALLOC_VAL;
4834
4835         switch (hw->mac.type) {
4836
4837         case ixgbe_mac_82598EB:
4838                 if (type == -1)
4839                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4840                 else
4841                         entry += (type * 64);
4842                 index = (entry >> 2) & 0x1F;
4843                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4844                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4845                 ivar |= (vector << (8 * (entry & 0x3)));
4846                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
4847                 break;
4848
4849         case ixgbe_mac_82599EB:
4850         case ixgbe_mac_X540:
4851                 if (type == -1) { /* MISC IVAR */
4852                         index = (entry & 1) * 8;
4853                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4854                         ivar &= ~(0xFF << index);
4855                         ivar |= (vector << index);
4856                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4857                 } else {        /* RX/TX IVARS */
4858                         index = (16 * (entry & 1)) + (8 * type);
4859                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4860                         ivar &= ~(0xFF << index);
4861                         ivar |= (vector << index);
4862                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4863                 }
4864
4865         default:
4866                 break;
4867         }
4868 }
4869
4870 static void
4871 ixgbe_configure_ivars(struct adapter *adapter)
4872 {
4873         struct  ix_queue *que = adapter->queues;
4874
4875         for (int i = 0; i < adapter->num_queues; i++, que++) {
4876                 /* First the RX queue entry */
4877                 ixgbe_set_ivar(adapter, i, que->msix, 0);
4878                 /* ... and the TX */
4879                 ixgbe_set_ivar(adapter, i, que->msix, 1);
4880                 /* Set an Initial EITR value */
4881                 ixgbe_set_eitr(adapter, que->msix, IXGBE_INTR_RATE);
4882         }
4883
4884         /* For the Link interrupt */
4885         ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
4886 }
4887
4888 /*
4889 ** ixgbe_sfp_probe - called in the local timer to
4890 ** determine if a port had optics inserted.
4891 */
4892 static bool ixgbe_sfp_probe(struct adapter *adapter)
4893 {
4894         struct ixgbe_hw *hw = &adapter->hw;
4895         device_t        dev = adapter->dev;
4896         bool            result = FALSE;
4897
4898         if ((hw->phy.type == ixgbe_phy_nl) &&
4899             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
4900                 s32 ret = hw->phy.ops.identify_sfp(hw);
4901                 if (ret)
4902                         goto out;
4903                 ret = hw->phy.ops.reset(hw);
4904                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4905                         device_printf(dev,"Unsupported SFP+ module detected!");
4906                         kprintf(" Reload driver with supported module.\n");
4907                         adapter->sfp_probe = FALSE;
4908                         goto out;
4909                 } else
4910                         device_printf(dev,"SFP+ module detected!\n");
4911                 /* We now have supported optics */
4912                 adapter->sfp_probe = FALSE;
4913                 /* Set the optics type so system reports correctly */
4914                 ixgbe_setup_optics(adapter);
4915                 result = TRUE;
4916         }
4917 out:
4918         return (result);
4919 }
4920
4921 /*
4922 ** Tasklet for handling SFP module interrupts
4923 */
4924 static void
4925 ixgbe_handle_mod(void *context, int pending)
4926 {
4927         struct adapter  *adapter = context;
4928         struct ixgbe_hw *hw = &adapter->hw;
4929         device_t        dev = adapter->dev;
4930         u32 err;
4931
4932         err = hw->phy.ops.identify_sfp(hw);
4933         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4934                 device_printf(dev,
4935                     "Unsupported SFP+ module type was detected.\n");
4936                 return;
4937         }
4938         err = hw->mac.ops.setup_sfp(hw);
4939         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4940                 device_printf(dev,
4941                     "Setup failure - unsupported SFP+ module type.\n");
4942                 return;
4943         }
4944         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
4945         return;
4946 }
4947
4948
4949 /*
4950 ** Tasklet for handling MSF (multispeed fiber) interrupts
4951 */
4952 static void
4953 ixgbe_handle_msf(void *context, int pending)
4954 {
4955         struct adapter  *adapter = context;
4956         struct ixgbe_hw *hw = &adapter->hw;
4957         u32 autoneg;
4958         bool negotiate;
4959
4960         autoneg = hw->phy.autoneg_advertised;
4961         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
4962                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
4963         if (hw->mac.ops.setup_link)
4964                 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
4965         return;
4966 }
4967
4968 #ifdef IXGBE_FDIR
4969 /*
4970 ** Tasklet for reinitializing the Flow Director filter table
4971 */
4972 static void
4973 ixgbe_reinit_fdir(void *context, int pending)
4974 {
4975         struct adapter  *adapter = context;
4976         struct ifnet   *ifp = adapter->ifp;
4977
4978         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
4979                 return;
4980         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
4981         adapter->fdir_reinit = 0;
4982         /* re-enable flow director interrupts */
4983         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
4984         /* Restart the interface */
4985         ifp->if_drv_flags |= IFF_DRV_RUNNING;
4986         return;
4987 }
4988 #endif
4989
4990 /**********************************************************************
4991  *
4992  *  Update the board statistics counters.
4993  *
4994  **********************************************************************/
4995 static void
4996 ixgbe_update_stats_counters(struct adapter *adapter)
4997 {
4998         struct ifnet   *ifp = adapter->ifp;
4999         struct ixgbe_hw *hw = &adapter->hw;
5000         u32  missed_rx = 0, bprc, lxon, lxoff, total;
5001         u64  total_missed_rx = 0;
5002
5003         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5004         adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5005         adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5006         adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5007
5008         for (int i = 0; i < 8; i++) {
5009                 u32 mp;
5010                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5011                 /* missed_rx tallies misses for the gprc workaround */
5012                 missed_rx += mp;
5013                 /* global total per queue */
5014                 adapter->stats.mpc[i] += mp;
5015                 /* Running comprehensive total for stats display */
5016                 total_missed_rx += adapter->stats.mpc[i];
5017                 if (hw->mac.type == ixgbe_mac_82598EB)
5018                         adapter->stats.rnbc[i] +=
5019                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5020                 adapter->stats.pxontxc[i] +=
5021                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5022                 adapter->stats.pxonrxc[i] +=
5023                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5024                 adapter->stats.pxofftxc[i] +=
5025                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5026                 adapter->stats.pxoffrxc[i] +=
5027                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5028                 adapter->stats.pxon2offc[i] +=
5029                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5030         }
5031         for (int i = 0; i < 16; i++) {
5032                 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5033                 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5034                 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5035                 adapter->stats.qbrc[i] +=
5036                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5037                 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5038                 adapter->stats.qbtc[i] +=
5039                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5040                 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5041         }
5042         adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5043         adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5044         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5045
5046         /* Hardware workaround, gprc counts missed packets */
5047         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5048         adapter->stats.gprc -= missed_rx;
5049
5050         if (hw->mac.type != ixgbe_mac_82598EB) {
5051                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5052                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5053                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5054                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5055                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5056                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5057                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5058                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5059         } else {
5060                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5061                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5062                 /* 82598 only has a counter in the high register */
5063                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5064                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5065                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5066         }
5067
5068         /*
5069          * Workaround: mprc hardware is incorrectly counting
5070          * broadcasts, so for now we subtract those.
5071          */
5072         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5073         adapter->stats.bprc += bprc;
5074         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5075         if (hw->mac.type == ixgbe_mac_82598EB)
5076                 adapter->stats.mprc -= bprc;
5077
5078         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5079         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5080         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5081         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5082         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5083         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5084
5085         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5086         adapter->stats.lxontxc += lxon;
5087         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5088         adapter->stats.lxofftxc += lxoff;
5089         total = lxon + lxoff;
5090
5091         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5092         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5093         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5094         adapter->stats.gptc -= total;
5095         adapter->stats.mptc -= total;
5096         adapter->stats.ptc64 -= total;
5097         adapter->stats.gotc -= total * ETHER_MIN_LEN;
5098
5099         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5100         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5101         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5102         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5103         adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5104         adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5105         adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5106         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5107         adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5108         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5109         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5110         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5111         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5112         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5113         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5114         adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5115         adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5116         adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5117         /* Only read FCOE on 82599 */
5118         if (hw->mac.type != ixgbe_mac_82598EB) {
5119                 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5120                 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5121                 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5122                 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5123                 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5124         }
5125
5126         /* Fill out the OS statistics structure */
5127         ifp->if_ipackets = adapter->stats.gprc;
5128         ifp->if_opackets = adapter->stats.gptc;
5129         ifp->if_ibytes = adapter->stats.gorc;
5130         ifp->if_obytes = adapter->stats.gotc;
5131         ifp->if_imcasts = adapter->stats.mprc;
5132         ifp->if_collisions = 0;
5133
5134         /* Rx Errors */
5135         ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs +
5136                 adapter->stats.rlec;
5137 }
5138
5139 /** ixgbe_sysctl_tdh_handler - Handler function
5140  *  Retrieves the TDH value from the hardware
5141  */
5142 static int
5143 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5144 {
5145         int error;
5146
5147         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5148         if (!txr) return 0;
5149
5150         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5151         error = sysctl_handle_int(oidp, &val, 0, req);
5152         if (error || !req->newptr)
5153                 return error;
5154         return 0;
5155 }
5156
5157 /** ixgbe_sysctl_tdt_handler - Handler function
5158  *  Retrieves the TDT value from the hardware
5159  */
5160 static int
5161 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5162 {
5163         int error;
5164
5165         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5166         if (!txr) return 0;
5167
5168         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5169         error = sysctl_handle_int(oidp, &val, 0, req);
5170         if (error || !req->newptr)
5171                 return error;
5172         return 0;
5173 }
5174
5175 /** ixgbe_sysctl_rdh_handler - Handler function
5176  *  Retrieves the RDH value from the hardware
5177  */
5178 static int
5179 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5180 {
5181         int error;
5182
5183         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5184         if (!rxr) return 0;
5185
5186         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5187         error = sysctl_handle_int(oidp, &val, 0, req);
5188         if (error || !req->newptr)
5189                 return error;
5190         return 0;
5191 }
5192
5193 /** ixgbe_sysctl_rdt_handler - Handler function
5194  *  Retrieves the RDT value from the hardware
5195  */
5196 static int
5197 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5198 {
5199         int error;
5200
5201         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5202         if (!rxr) return 0;
5203
5204         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5205         error = sysctl_handle_int(oidp, &val, 0, req);
5206         if (error || !req->newptr)
5207                 return error;
5208         return 0;
5209 }
5210
5211 /*
5212  * Add sysctl variables, one per statistic, to the system.
5213  */
5214 static void
5215 ixgbe_add_hw_stats(struct adapter *adapter)
5216 {
5217         struct tx_ring *txr = adapter->tx_rings;
5218         struct rx_ring *rxr = adapter->rx_rings;
5219
5220         struct sysctl_ctx_list *ctx = &adapter->sysctl_ctx;
5221         struct sysctl_oid *tree = adapter->sysctl_tree;
5222         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5223         struct ixgbe_hw_stats *stats = &adapter->stats;
5224
5225         struct sysctl_oid *stat_node, *queue_node;
5226         struct sysctl_oid_list *stat_list, *queue_list;
5227
5228 #define QUEUE_NAME_LEN 32
5229         char namebuf[QUEUE_NAME_LEN];
5230
5231         /* Driver Statistics */
5232         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5233                         CTLFLAG_RD, &adapter->dropped_pkts,
5234                         "Driver dropped packets");
5235         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5236                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5237                         "m_defrag() failed");
5238         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup",
5239                         CTLFLAG_RD, &adapter->no_tx_dma_setup,
5240                         "Driver tx dma failure in xmit");
5241         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5242                         CTLFLAG_RD, &adapter->watchdog_events,
5243                         "Watchdog timeouts");
5244         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx",
5245                         CTLFLAG_RD, &adapter->tso_tx,
5246                         "TSO");
5247         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5248                         CTLFLAG_RD, &adapter->link_irq,
5249                         "Link MSIX IRQ Handled");
5250
5251         for (int i = 0; i < adapter->num_queues; i++, txr++) {
5252         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5253                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5254                                             CTLFLAG_RD, NULL, "Queue Name");
5255                 queue_list = SYSCTL_CHILDREN(queue_node);
5256                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5257                                 CTLFLAG_RD, &(adapter->queues[i].irqs), 0,
5258                                 "irqs on this queue");
5259                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5260                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5261                                 ixgbe_sysctl_tdh_handler, "IU",
5262                                 "Transmit Descriptor Head");
5263                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5264                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5265                                 ixgbe_sysctl_tdt_handler, "IU",
5266                                 "Transmit Descriptor Tail");
5267                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5268                                 CTLFLAG_RD, &txr->no_desc_avail, 0,
5269                                 "Queue No Descriptor Available");
5270                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5271                                 CTLFLAG_RD, &txr->total_packets, 0,
5272                                 "Queue Packets Transmitted");
5273         }
5274
5275         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5276         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5277                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5278                                             CTLFLAG_RD, NULL, "Queue Name");
5279                 queue_list = SYSCTL_CHILDREN(queue_node);
5280
5281 #if 0   /* NET_LRO */
5282                 struct lro_ctrl *lro = &rxr->lro;
5283 #endif
5284
5285         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5286                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5287                                             CTLFLAG_RD, NULL, "Queue Name");
5288                 queue_list = SYSCTL_CHILDREN(queue_node);
5289
5290                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5291                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5292                                 ixgbe_sysctl_rdh_handler, "IU",
5293                                 "Receive Descriptor Head");
5294                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5295                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5296                                 ixgbe_sysctl_rdt_handler, "IU",
5297                                 "Receive Descriptor Tail");
5298                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5299                                 CTLFLAG_RD, &rxr->rx_packets, 0,
5300                                 "Queue Packets Received");
5301                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5302                                 CTLFLAG_RD, &rxr->rx_bytes, 0,
5303                                 "Queue Bytes Received");
5304 #if 0   /* NET_LRO */
5305                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5306                                 CTLFLAG_RD, &lro->lro_queued, 0,
5307                                 "LRO Queued");
5308                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5309                                 CTLFLAG_RD, &lro->lro_flushed, 0,
5310                                 "LRO Flushed");
5311 #endif
5312         }
5313
5314         /* MAC stats get the own sub node */
5315
5316         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5317                                     CTLFLAG_RD, NULL, "MAC Statistics");
5318         stat_list = SYSCTL_CHILDREN(stat_node);
5319
5320         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5321                         CTLFLAG_RD, &stats->crcerrs, 0,
5322                         "CRC Errors");
5323         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5324                         CTLFLAG_RD, &stats->illerrc, 0,
5325                         "Illegal Byte Errors");
5326         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5327                         CTLFLAG_RD, &stats->errbc, 0,
5328                         "Byte Errors");
5329         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5330                         CTLFLAG_RD, &stats->mspdc, 0,
5331                         "MAC Short Packets Discarded");
5332         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5333                         CTLFLAG_RD, &stats->mlfc, 0,
5334                         "MAC Local Faults");
5335         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5336                         CTLFLAG_RD, &stats->mrfc, 0,
5337                         "MAC Remote Faults");
5338         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5339                         CTLFLAG_RD, &stats->rlec, 0,
5340                         "Receive Length Errors");
5341         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd",
5342                         CTLFLAG_RD, &stats->lxontxc, 0,
5343                         "Link XON Transmitted");
5344         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd",
5345                         CTLFLAG_RD, &stats->lxonrxc, 0,
5346                         "Link XON Received");
5347         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd",
5348                         CTLFLAG_RD, &stats->lxofftxc, 0,
5349                         "Link XOFF Transmitted");
5350         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd",
5351                         CTLFLAG_RD, &stats->lxoffrxc, 0,
5352                         "Link XOFF Received");
5353
5354         /* Packet Reception Stats */
5355         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5356                         CTLFLAG_RD, &stats->tor, 0,
5357                         "Total Octets Received");
5358         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5359                         CTLFLAG_RD, &stats->gorc, 0,
5360                         "Good Octets Received");
5361         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5362                         CTLFLAG_RD, &stats->tpr, 0,
5363                         "Total Packets Received");
5364         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5365                         CTLFLAG_RD, &stats->gprc, 0,
5366                         "Good Packets Received");
5367         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5368                         CTLFLAG_RD, &stats->mprc, 0,
5369                         "Multicast Packets Received");
5370         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5371                         CTLFLAG_RD, &stats->bprc, 0,
5372                         "Broadcast Packets Received");
5373         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5374                         CTLFLAG_RD, &stats->prc64, 0,
5375                         "64 byte frames received ");
5376         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5377                         CTLFLAG_RD, &stats->prc127, 0,
5378                         "65-127 byte frames received");
5379         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5380                         CTLFLAG_RD, &stats->prc255, 0,
5381                         "128-255 byte frames received");
5382         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5383                         CTLFLAG_RD, &stats->prc511, 0,
5384                         "256-511 byte frames received");
5385         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5386                         CTLFLAG_RD, &stats->prc1023, 0,
5387                         "512-1023 byte frames received");
5388         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5389                         CTLFLAG_RD, &stats->prc1522, 0,
5390                         "1023-1522 byte frames received");
5391         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5392                         CTLFLAG_RD, &stats->ruc, 0,
5393                         "Receive Undersized");
5394         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5395                         CTLFLAG_RD, &stats->rfc, 0,
5396                         "Fragmented Packets Received ");
5397         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5398                         CTLFLAG_RD, &stats->roc, 0,
5399                         "Oversized Packets Received");
5400         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5401                         CTLFLAG_RD, &stats->rjc, 0,
5402                         "Received Jabber");
5403         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5404                         CTLFLAG_RD, &stats->mngprc, 0,
5405                         "Management Packets Received");
5406         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5407                         CTLFLAG_RD, &stats->mngptc, 0,
5408                         "Management Packets Dropped");
5409         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5410                         CTLFLAG_RD, &stats->xec, 0,
5411                         "Checksum Errors");
5412
5413         /* Packet Transmission Stats */
5414         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5415                         CTLFLAG_RD, &stats->gotc, 0,
5416                         "Good Octets Transmitted");
5417         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5418                         CTLFLAG_RD, &stats->tpt, 0,
5419                         "Total Packets Transmitted");
5420         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5421                         CTLFLAG_RD, &stats->gptc, 0,
5422                         "Good Packets Transmitted");
5423         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5424                         CTLFLAG_RD, &stats->bptc, 0,
5425                         "Broadcast Packets Transmitted");
5426         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5427                         CTLFLAG_RD, &stats->mptc, 0,
5428                         "Multicast Packets Transmitted");
5429         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5430                         CTLFLAG_RD, &stats->mngptc, 0,
5431                         "Management Packets Transmitted");
5432         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5433                         CTLFLAG_RD, &stats->ptc64, 0,
5434                         "64 byte frames transmitted ");
5435         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5436                         CTLFLAG_RD, &stats->ptc127, 0,
5437                         "65-127 byte frames transmitted");
5438         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5439                         CTLFLAG_RD, &stats->ptc255, 0,
5440                         "128-255 byte frames transmitted");
5441         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5442                         CTLFLAG_RD, &stats->ptc511, 0,
5443                         "256-511 byte frames transmitted");
5444         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5445                         CTLFLAG_RD, &stats->ptc1023, 0,
5446                         "512-1023 byte frames transmitted");
5447         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5448                         CTLFLAG_RD, &stats->ptc1522, 0,
5449                         "1024-1522 byte frames transmitted");
5450
5451         /* FC Stats */
5452         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc",
5453                 CTLFLAG_RD, &stats->fccrc, 0,
5454                 "FC CRC Errors");
5455         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last",
5456                 CTLFLAG_RD, &stats->fclast, 0,
5457                 "FC Last Error");
5458         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd",
5459                 CTLFLAG_RD, &stats->fcoerpdc, 0,
5460                 "FCoE Packets Dropped");
5461         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd",
5462                 CTLFLAG_RD, &stats->fcoeprc, 0,
5463                 "FCoE Packets Received");
5464         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd",
5465                 CTLFLAG_RD, &stats->fcoeptc, 0,
5466                 "FCoE Packets Transmitted");
5467         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd",
5468                 CTLFLAG_RD, &stats->fcoedwrc, 0,
5469                 "FCoE DWords Received");
5470         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd",
5471                 CTLFLAG_RD, &stats->fcoedwtc, 0,
5472                 "FCoE DWords Transmitted");
5473 }
5474
5475 /*
5476 ** Set flow control using sysctl:
5477 ** Flow control values:
5478 **      0 - off
5479 **      1 - rx pause
5480 **      2 - tx pause
5481 **      3 - full
5482 */
5483 static int
5484 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5485 {
5486         int error, last;
5487         struct adapter *adapter = (struct adapter *) arg1;
5488
5489         last = adapter->fc;
5490         error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5491         if ((error) || (req->newptr == NULL))
5492                 return (error);
5493
5494         /* Don't bother if it's not changed */
5495         if (adapter->fc == last)
5496                 return (0);
5497
5498         switch (adapter->fc) {
5499                 case ixgbe_fc_rx_pause:
5500                 case ixgbe_fc_tx_pause:
5501                 case ixgbe_fc_full:
5502                         adapter->hw.fc.requested_mode = adapter->fc;
5503                         break;
5504                 case ixgbe_fc_none:
5505                 default:
5506                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
5507         }
5508         /* Don't autoneg if forcing a value */
5509         adapter->hw.fc.disable_fc_autoneg = TRUE;
5510         ixgbe_fc_enable(&adapter->hw);
5511         return error;
5512 }
5513
5514 static void
5515 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5516         const char *description, int *limit, int value)
5517 {
5518         *limit = value;
5519         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
5520             SYSCTL_CHILDREN(adapter->sysctl_tree),
5521             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5522 }
5523
5524 /*
5525 ** Control link advertise speed:
5526 **      1 - advertise only 1G
5527 **      2 - advertise 100Mb
5528 **      3 - advertise normal
5529 */
5530 static int
5531 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5532 {
5533         int                     error = 0;
5534         struct adapter          *adapter;
5535         device_t                dev;
5536         struct ixgbe_hw         *hw;
5537         ixgbe_link_speed        speed, last;
5538
5539         adapter = (struct adapter *) arg1;
5540         dev = adapter->dev;
5541         hw = &adapter->hw;
5542         last = adapter->advertise;
5543
5544         error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5545         if ((error) || (adapter->advertise == -1))
5546                 return (error);
5547
5548         if (adapter->advertise == last) /* no change */
5549                 return (0);
5550
5551         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5552             (hw->phy.multispeed_fiber)))
5553                 return (error);
5554
5555         if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5556                 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5557                 return (error);
5558         }
5559
5560         if (adapter->advertise == 1)
5561                 speed = IXGBE_LINK_SPEED_1GB_FULL;
5562         else if (adapter->advertise == 2)
5563                 speed = IXGBE_LINK_SPEED_100_FULL;
5564         else if (adapter->advertise == 3)
5565                 speed = IXGBE_LINK_SPEED_1GB_FULL |
5566                         IXGBE_LINK_SPEED_10GB_FULL;
5567         else /* bogus value */
5568                 return (error);
5569
5570         hw->mac.autotry_restart = TRUE;
5571         hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5572
5573         return (error);
5574 }
5575
5576 /*
5577 ** Thermal Shutdown Trigger
5578 **   - cause a Thermal Overtemp IRQ
5579 */
5580 static int
5581 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5582 {
5583         int             error, fire = 0;
5584         struct adapter  *adapter = (struct adapter *) arg1;
5585         struct ixgbe_hw *hw = &adapter->hw;
5586
5587
5588         if (hw->mac.type != ixgbe_mac_X540)
5589                 return (0);
5590
5591         error = sysctl_handle_int(oidp, &fire, 0, req);
5592         if ((error) || (req->newptr == NULL))
5593                 return (error);
5594
5595         if (fire) {
5596                 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5597                 reg |= IXGBE_EICR_TS;
5598                 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5599         }
5600
5601         return (0);
5602 }
5603
5604 static void
5605 ixgbe_set_eitr(struct adapter *sc, int idx, int rate)
5606 {
5607         uint32_t eitr = 0;
5608
5609         /* convert rate in intr/s to hw representation */
5610         if (rate > 0) {
5611                 eitr = 1000000 / rate;
5612                 eitr <<= IXGBE_EITR_INTVL_SHIFT;
5613
5614                 if (eitr == 0) {
5615                         /* Don't disable it */
5616                         eitr = 1 << IXGBE_EITR_INTVL_SHIFT;
5617                 } else if (eitr > IXGBE_EITR_INTVL_MASK) {
5618                         /* Don't allow it to be too large */
5619                         eitr = IXGBE_EITR_INTVL_MASK;
5620                 }
5621         }
5622         IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(idx), eitr);
5623 }
5624
5625 static int
5626 ixgbe_sysctl_intr_rate(SYSCTL_HANDLER_ARGS)
5627 {
5628         struct adapter *sc = (void *)arg1;
5629         struct ifnet *ifp = sc->ifp;
5630         int error, intr_rate, running;
5631         struct ix_queue *que = sc->queues;
5632
5633         intr_rate = sc->intr_rate;
5634         error = sysctl_handle_int(oidp, &intr_rate, 0, req);
5635         if (error || req->newptr == NULL)
5636                 return error;
5637         if (intr_rate < 0)
5638                 return EINVAL;
5639
5640         ifnet_serialize_all(ifp);
5641
5642         sc->intr_rate = intr_rate;
5643         running = ifp->if_flags & IFF_RUNNING;
5644         if (running)
5645                 ixgbe_set_eitr(sc, 0, sc->intr_rate);
5646
5647         if (running && (sc->intr_type == PCI_INTR_TYPE_MSIX)) {
5648                 for (int i = 0; i < sc->num_queues; i++, que++)
5649                         ixgbe_set_eitr(sc, que->msix, sc->intr_rate);
5650         }
5651
5652         if (bootverbose)
5653                 if_printf(ifp, "interrupt rate set to %d/sec\n", sc->intr_rate);
5654
5655         ifnet_deserialize_all(ifp);
5656
5657         return 0;
5658 }
5659
5660 /* rearrange mbuf chain to get contiguous bytes */
5661 static int
5662 ixgbe_tso_pullup(struct tx_ring *txr, struct mbuf **mp)
5663 {
5664         int hoff, iphlen, thoff;
5665         struct mbuf *m;
5666
5667         m = *mp;
5668         KASSERT(M_WRITABLE(m), ("TSO mbuf not writable"));
5669
5670         iphlen = m->m_pkthdr.csum_iphlen;
5671         thoff = m->m_pkthdr.csum_thlen;
5672         hoff = m->m_pkthdr.csum_lhlen;
5673
5674         KASSERT(iphlen > 0, ("invalid ip hlen"));
5675         KASSERT(thoff > 0, ("invalid tcp hlen"));
5676         KASSERT(hoff > 0, ("invalid ether hlen"));
5677
5678         if (__predict_false(m->m_len < hoff + iphlen + thoff)) {
5679                 m = m_pullup(m, hoff + iphlen + thoff);
5680                 if (m == NULL) {
5681                         *mp = NULL;
5682                         return ENOBUFS;
5683                 }
5684                 *mp = m;
5685         }
5686
5687         return 0;
5688 }