sys/dev/netif/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2012, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
  34
  35 #include "opt_inet.h"
  36 #include "opt_inet6.h"
  37
  38 #include "ixgbe.h"
  39
  40 /*********************************************************************
  41  *  Set this to one to display debug statistics
  42  *********************************************************************/
  43 int             ixgbe_display_debug_stats = 0;
  44
  45 /*********************************************************************
  46  *  Driver version
  47  *********************************************************************/
  48 char ixgbe_driver_version[] = "2.4.8";
  49
  50 /*********************************************************************
  51  *  PCI Device ID Table
  52  *
  53  *  Used by probe to select devices to load on
  54  *  Last field stores an index into ixgbe_strings
  55  *  Last entry must be all 0s
  56  *
  57  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  58  *********************************************************************/
  59
  60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  61 {
  62         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
  81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
  82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
  83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
  84         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
  85         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
  86         /* required last entry */
  87         {0, 0, 0, 0, 0}
  88 };
  89
  90 /*********************************************************************
  91  *  Table of branding strings
  92  *********************************************************************/
  93
  94 static char    *ixgbe_strings[] = {
  95         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  96 };
  97
  98 /*********************************************************************
  99  *  Function prototypes
 100  *********************************************************************/
 101 static int      ixgbe_probe(device_t);
 102 static int      ixgbe_attach(device_t);
 103 static int      ixgbe_detach(device_t);
 104 static int      ixgbe_shutdown(device_t);
 105 static void     ixgbe_start(struct ifnet *);
 106 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
 107 #if 0 /* __FreeBSD_version >= 800000 */
 108 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
 109 static int      ixgbe_mq_start_locked(struct ifnet *,
 110                     struct tx_ring *, struct mbuf *);
 111 static void     ixgbe_qflush(struct ifnet *);
 112 #endif
 113 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 114 static void     ixgbe_init(void *);
 115 static void     ixgbe_init_locked(struct adapter *);
 116 static void     ixgbe_stop(void *);
 117 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 118 static int      ixgbe_media_change(struct ifnet *);
 119 static void     ixgbe_identify_hardware(struct adapter *);
 120 static int      ixgbe_allocate_pci_resources(struct adapter *);
 121 static int      ixgbe_allocate_msix(struct adapter *);
 122 static int      ixgbe_allocate_legacy(struct adapter *);
 123 static int      ixgbe_allocate_queues(struct adapter *);
 124 static int      ixgbe_setup_msix(struct adapter *);
 125 static void     ixgbe_free_pci_resources(struct adapter *);
 126 static void     ixgbe_local_timer(void *);
 127 static int      ixgbe_setup_interface(device_t, struct adapter *);
 128 static void     ixgbe_config_link(struct adapter *);
 129
 130 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 131 static int      ixgbe_setup_transmit_structures(struct adapter *);
 132 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 133 static void     ixgbe_initialize_transmit_units(struct adapter *);
 134 static void     ixgbe_free_transmit_structures(struct adapter *);
 135 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 136
 137 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 138 static int      ixgbe_setup_receive_structures(struct adapter *);
 139 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 140 static void     ixgbe_initialize_receive_units(struct adapter *);
 141 static void     ixgbe_free_receive_structures(struct adapter *);
 142 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 143 #if 0   /* NET_LRO */
 144 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
 145 #endif
 146
 147 static void     ixgbe_enable_intr(struct adapter *);
 148 static void     ixgbe_disable_intr(struct adapter *);
 149 static void     ixgbe_update_stats_counters(struct adapter *);
 150 static void     ixgbe_txeof(struct tx_ring *);
 151 static void     ixgbe_rxeof(struct ix_queue *, int);
 152 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
 153 static void     ixgbe_set_promisc(struct adapter *);
 154 static void     ixgbe_set_multi(struct adapter *);
 155 static void     ixgbe_update_link_status(struct adapter *);
 156 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
 157 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 158 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 159 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
 160 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
 161 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 162                     struct ixgbe_dma_alloc *, int);
 163 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 164 static void     ixgbe_add_rx_process_limit(struct adapter *, const char *,
 165                     const char *, int *, int);
 166 static bool     ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 167 static bool     ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
 168 static int      ixgbe_tso_pullup(struct tx_ring *, struct mbuf **);
 169 static void     ixgbe_add_sysctl(struct adapter *);
 170 static void     ixgbe_set_eitr(struct adapter *, int, int);
 171 static int      ixgbe_sysctl_intr_rate(SYSCTL_HANDLER_ARGS);
 172 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 173 static void     ixgbe_configure_ivars(struct adapter *);
 174 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 175
 176 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 177 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 178 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 179
 180 static void     ixgbe_add_hw_stats(struct adapter *adapter);
 181
 182 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
 183 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
 184                     struct mbuf *, u32);
 185
 186 /* Support for pluggable optic modules */
 187 static bool     ixgbe_sfp_probe(struct adapter *);
 188 static void     ixgbe_setup_optics(struct adapter *);
 189
 190 /* Legacy (single vector interrupt handler */
 191 static void     ixgbe_legacy_irq(void *);
 192
 193 /* The MSI/X Interrupt handlers */
 194 static void     ixgbe_msix_que(void *);
 195 static void     ixgbe_msix_link(void *);
 196
 197 /* Deferred interrupt tasklets */
 198 static void     ixgbe_handle_msf(void *, int);
 199 static void     ixgbe_handle_mod(void *, int);
 200
 201 #ifdef IXGBE_FDIR
 202 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
 203 static void     ixgbe_reinit_fdir(void *, int);
 204 #endif
 205
 206 /*********************************************************************
 207  *  FreeBSD Device Interface Entry Points
 208  *********************************************************************/
 209
 210 static device_method_t ixgbe_methods[] = {
 211         /* Device interface */
 212         DEVMETHOD(device_probe, ixgbe_probe),
 213         DEVMETHOD(device_attach, ixgbe_attach),
 214         DEVMETHOD(device_detach, ixgbe_detach),
 215         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 216         {0, 0}
 217 };
 218
 219 static driver_t ixgbe_driver = {
 220         "ix", ixgbe_methods, sizeof(struct adapter),
 221 };
 222
 223 devclass_t ixgbe_devclass;
 224 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
 225
 226 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 227 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 228
 229 /*
 230 ** TUNEABLE PARAMETERS:
 231 */
 232
 233 /* How many packets rxeof tries to clean at a time */
 234 static int ixgbe_rx_process_limit = 128;
 235 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 236
 237 /*
 238 ** Smart speed setting, default to on
 239 ** this only works as a compile option
 240 ** right now as its during attach, set
 241 ** this to 'ixgbe_smart_speed_off' to
 242 ** disable.
 243 */
 244 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 245
 246 static int ixgbe_msi_enable = 1;
 247 TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
 248
 249 /*
 250  * MSIX should be the default for best performance,
 251  * but this allows it to be forced off for testing.
 252  */
 253 static int ixgbe_enable_msix = 1;
 254 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 255
 256 /*
 257  * Header split: this causes the hardware to DMA
 258  * the header into a separate mbuf from the payload,
 259  * it can be a performance win in some workloads, but
 260  * in others it actually hurts, its off by default.
 261  */
 262 static int ixgbe_header_split = FALSE;
 263 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
 264
 265 /*
 266  * Number of Queues, can be set to 0,
 267  * it then autoconfigures based on the
 268  * number of cpus with a max of 8. This
 269  * can be overriden manually here.
 270  */
 271 static int ixgbe_num_queues = 0;
 272 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
 273
 274 /*
 275 ** Number of TX descriptors per ring,
 276 ** setting higher than RX as this seems
 277 ** the better performing choice.
 278 */
 279 static int ixgbe_txd = PERFORM_TXD;
 280 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 281
 282 /* Number of RX descriptors per ring */
 283 static int ixgbe_rxd = PERFORM_RXD;
 284 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 285
 286 /* Keep running tab on them for sanity check */
 287 static int ixgbe_total_ports;
 288
 289 #ifdef IXGBE_FDIR
 290 /*
 291 ** For Flow Director: this is the
 292 ** number of TX packets we sample
 293 ** for the filter pool, this means
 294 ** every 20th packet will be probed.
 295 **
 296 ** This feature can be disabled by
 297 ** setting this to 0.
 298 */
 299 static int atr_sample_rate = 20;
 300 /*
 301 ** Flow Director actually 'steals'
 302 ** part of the packet buffer as its
 303 ** filter pool, this variable controls
 304 ** how much it uses:
 305 **  0 = 64K, 1 = 128K, 2 = 256K
 306 */
 307 static int fdir_pballoc = 1;
 308 #endif
 309
 310 #ifdef DEV_NETMAP
 311 /*
 312  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
 313  * be a reference on how to implement netmap support in a driver.
 314  * Additional comments are in ixgbe_netmap.h .
 315  *
 316  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
 317  * that extend the standard driver.
 318  */
 319 #include <dev/netmap/ixgbe_netmap.h>
 320 #endif /* DEV_NETMAP */
 321
 322 /*********************************************************************
 323  *  Device identification routine
 324  *
 325  *  ixgbe_probe determines if the driver should be loaded on
 326  *  adapter based on PCI vendor/device id of the adapter.
 327  *
 328  *  return BUS_PROBE_DEFAULT on success, positive on failure
 329  *********************************************************************/
 330
 331 static int
 332 ixgbe_probe(device_t dev)
 333 {
 334         ixgbe_vendor_info_t *ent;
 335
 336         u16     pci_vendor_id = 0;
 337         u16     pci_device_id = 0;
 338         u16     pci_subvendor_id = 0;
 339         u16     pci_subdevice_id = 0;
 340         char    adapter_name[256];
 341
 342         INIT_DEBUGOUT("ixgbe_probe: begin");
 343
 344         pci_vendor_id = pci_get_vendor(dev);
 345         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 346                 return (ENXIO);
 347
 348         pci_device_id = pci_get_device(dev);
 349         pci_subvendor_id = pci_get_subvendor(dev);
 350         pci_subdevice_id = pci_get_subdevice(dev);
 351
 352         ent = ixgbe_vendor_info_array;
 353         while (ent->vendor_id != 0) {
 354                 if ((pci_vendor_id == ent->vendor_id) &&
 355                     (pci_device_id == ent->device_id) &&
 356
 357                     ((pci_subvendor_id == ent->subvendor_id) ||
 358                      (ent->subvendor_id == 0)) &&
 359
 360                     ((pci_subdevice_id == ent->subdevice_id) ||
 361                      (ent->subdevice_id == 0))) {
 362                         ksprintf(adapter_name, "%s, Version - %s",
 363                                 ixgbe_strings[ent->index],
 364                                 ixgbe_driver_version);
 365                         device_set_desc_copy(dev, adapter_name);
 366                         ++ixgbe_total_ports;
 367                         return (BUS_PROBE_DEFAULT);
 368                 }
 369                 ent++;
 370         }
 371         return (ENXIO);
 372 }
 373
 374 /*********************************************************************
 375  *  Device initialization routine
 376  *
 377  *  The attach entry point is called when the driver is being loaded.
 378  *  This routine identifies the type of hardware, allocates all resources
 379  *  and initializes the hardware.
 380  *
 381  *  return 0 on success, positive on failure
 382  *********************************************************************/
 383
 384 static int
 385 ixgbe_attach(device_t dev)
 386 {
 387         struct adapter *adapter;
 388         struct ixgbe_hw *hw;
 389         int             error = 0;
 390         u16             csum;
 391         u32             ctrl_ext;
 392
 393         INIT_DEBUGOUT("ixgbe_attach: begin");
 394
 395         if (resource_disabled("ixgbe", device_get_unit(dev))) {
 396                 device_printf(dev, "Disabled by device hint\n");
 397                 return (ENXIO);
 398         }
 399
 400         /* Allocate, clear, and link in our adapter structure */
 401         adapter = device_get_softc(dev);
 402         adapter->dev = adapter->osdep.dev = dev;
 403         hw = &adapter->hw;
 404
 405         /* Core Lock Init*/
 406         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 407
 408         /* Set up the timer callout */
 409         callout_init_mp(&adapter->timer);
 410
 411         /* Determine hardware revision */
 412         ixgbe_identify_hardware(adapter);
 413
 414         /* Enable bus mastering */
 415         pci_enable_busmaster(dev);
 416
 417         /* Do base PCI setup - map BAR0 */
 418         if (ixgbe_allocate_pci_resources(adapter)) {
 419                 device_printf(dev, "Allocation of PCI resources failed\n");
 420                 error = ENXIO;
 421                 goto err_out;
 422         }
 423
 424         /* Do descriptor calc and sanity checks */
 425         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 426             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 427                 device_printf(dev, "TXD config issue, using default!\n");
 428                 adapter->num_tx_desc = DEFAULT_TXD;
 429         } else
 430                 adapter->num_tx_desc = ixgbe_txd;
 431
 432         /*
 433         ** With many RX rings it is easy to exceed the
 434         ** system mbuf allocation. Tuning nmbclusters
 435         ** can alleviate this.
 436         */
 437         if (nmbclusters > 0 ) {
 438                 int s;
 439                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 440                 if (s > nmbclusters) {
 441                         device_printf(dev, "RX Descriptors exceed "
 442                             "system mbuf max, using default instead!\n");
 443                         ixgbe_rxd = DEFAULT_RXD;
 444                 }
 445         }
 446
 447         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 448             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 449                 device_printf(dev, "RXD config issue, using default!\n");
 450                 adapter->num_rx_desc = DEFAULT_RXD;
 451         } else
 452                 adapter->num_rx_desc = ixgbe_rxd;
 453
 454         /* Allocate our TX/RX Queues */
 455         if (ixgbe_allocate_queues(adapter)) {
 456                 error = ENOMEM;
 457                 goto err_out;
 458         }
 459
 460         /* Allocate multicast array memory. */
 461         adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
 462             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 463         if (adapter->mta == NULL) {
 464                 device_printf(dev, "Can not allocate multicast setup array\n");
 465                 error = ENOMEM;
 466                 goto err_late;
 467         }
 468
 469         /* Initialize the shared code */
 470         error = ixgbe_init_shared_code(hw);
 471         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 472                 /*
 473                 ** No optics in this port, set up
 474                 ** so the timer routine will probe
 475                 ** for later insertion.
 476                 */
 477                 adapter->sfp_probe = TRUE;
 478                 error = 0;
 479         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 480                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 481                 error = EIO;
 482                 goto err_late;
 483         } else if (error) {
 484                 device_printf(dev,"Unable to initialize the shared code\n");
 485                 error = EIO;
 486                 goto err_late;
 487         }
 488
 489         /* Make sure we have a good EEPROM before we read from it */
 490         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 491                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 492                 error = EIO;
 493                 goto err_late;
 494         }
 495
 496         error = ixgbe_init_hw(hw);
 497         switch (error) {
 498         case IXGBE_ERR_EEPROM_VERSION:
 499                 device_printf(dev, "This device is a pre-production adapter/"
 500                     "LOM.  Please be aware there may be issues associated "
 501                     "with your hardware.\n If you are experiencing problems "
 502                     "please contact your Intel or hardware representative "
 503                     "who provided you with this hardware.\n");
 504                 break;
 505         case IXGBE_ERR_SFP_NOT_SUPPORTED:
 506                 device_printf(dev,"Unsupported SFP+ Module\n");
 507                 error = EIO;
 508                 device_printf(dev,"Hardware Initialization Failure\n");
 509                 goto err_late;
 510         case IXGBE_ERR_SFP_NOT_PRESENT:
 511                 device_printf(dev,"No SFP+ Module found\n");
 512                 /* falls thru */
 513         default:
 514                 break;
 515         }
 516
 517         /* Detect and set physical type */
 518         ixgbe_setup_optics(adapter);
 519
 520         if ((adapter->msix > 1) && (ixgbe_enable_msix)) {
 521                 adapter->intr_type = PCI_INTR_TYPE_MSIX;
 522                 error = ixgbe_allocate_msix(adapter);
 523         } else {
 524                 error = ixgbe_allocate_legacy(adapter);
 525         }
 526         if (error)
 527                 goto err_late;
 528
 529         /* Setup OS specific network interface */
 530         if (ixgbe_setup_interface(dev, adapter) != 0)
 531                 goto err_late;
 532
 533         /* Add sysctl tree */
 534         ixgbe_add_sysctl(adapter);
 535
 536         /* Initialize statistics */
 537         ixgbe_update_stats_counters(adapter);
 538
 539         /* Register for VLAN events */
 540         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 541             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 542         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 543             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 544
 545         /* Print PCIE bus type/speed/width info */
 546         ixgbe_get_bus_info(hw);
 547         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
 548             ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
 549             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
 550             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
 551             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
 552             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
 553             ("Unknown"));
 554
 555         if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
 556             (hw->bus.speed == ixgbe_bus_speed_2500)) {
 557                 device_printf(dev, "PCI-Express bandwidth available"
 558                     " for this card\n     is not sufficient for"
 559                     " optimal performance.\n");
 560                 device_printf(dev, "For optimal performance a x8 "
 561                     "PCIE, or x4 PCIE 2 slot is required.\n");
 562         }
 563
 564         /* let hardware know driver is loaded */
 565         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 566         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 567         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 568
 569         ixgbe_add_hw_stats(adapter);
 570
 571 #ifdef DEV_NETMAP
 572         ixgbe_netmap_attach(adapter);
 573 #endif /* DEV_NETMAP */
 574         INIT_DEBUGOUT("ixgbe_attach: end");
 575         return (0);
 576 err_late:
 577         ixgbe_free_transmit_structures(adapter);
 578         ixgbe_free_receive_structures(adapter);
 579 err_out:
 580         if (adapter->ifp != NULL)
 581                 if_free(adapter->ifp);
 582         ixgbe_free_pci_resources(adapter);
 583         kfree(adapter->mta, M_DEVBUF);
 584         return (error);
 585
 586 }
 587
 588 /*********************************************************************
 589  *  Device removal routine
 590  *
 591  *  The detach entry point is called when the driver is being removed.
 592  *  This routine stops the adapter and deallocates all the resources
 593  *  that were allocated for driver operation.
 594  *
 595  *  return 0 on success, positive on failure
 596  *********************************************************************/
 597
 598 static int
 599 ixgbe_detach(device_t dev)
 600 {
 601         struct adapter *adapter = device_get_softc(dev);
 602         u32     ctrl_ext;
 603
 604         INIT_DEBUGOUT("ixgbe_detach: begin");
 605
 606         /* Make sure VLANS are not using driver */
 607         if (adapter->ifp->if_vlantrunks != NULL) {
 608                 device_printf(dev,"Vlan in use, detach first\n");
 609                 return (EBUSY);
 610         }
 611
 612         IXGBE_CORE_LOCK(adapter);
 613         ixgbe_stop(adapter);
 614         IXGBE_CORE_UNLOCK(adapter);
 615
 616         /* Drain the Link queue */
 617         if (adapter->tq) {
 618                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 619                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 620 #ifdef IXGBE_FDIR
 621                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 622 #endif
 623                 taskqueue_free(adapter->tq);
 624         }
 625
 626         /* let hardware know driver is unloading */
 627         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 628         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 629         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 630
 631         /* Unregister VLAN events */
 632         if (adapter->vlan_attach != NULL)
 633                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 634         if (adapter->vlan_detach != NULL)
 635                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 636
 637         ether_ifdetach(adapter->ifp);
 638         callout_stop(&adapter->timer);
 639 #ifdef DEV_NETMAP
 640         netmap_detach(adapter->ifp);
 641 #endif /* DEV_NETMAP */
 642         ixgbe_free_pci_resources(adapter);
 643         bus_generic_detach(dev);
 644         if_free(adapter->ifp);
 645
 646         ixgbe_free_transmit_structures(adapter);
 647         ixgbe_free_receive_structures(adapter);
 648         kfree(adapter->mta, M_DEVBUF);
 649         sysctl_ctx_free(&adapter->sysctl_ctx);
 650
 651         IXGBE_CORE_LOCK_DESTROY(adapter);
 652         return (0);
 653 }
 654
 655 /*********************************************************************
 656  *
 657  *  Shutdown entry point
 658  *
 659  **********************************************************************/
 660
 661 static int
 662 ixgbe_shutdown(device_t dev)
 663 {
 664         struct adapter *adapter = device_get_softc(dev);
 665         IXGBE_CORE_LOCK(adapter);
 666         ixgbe_stop(adapter);
 667         IXGBE_CORE_UNLOCK(adapter);
 668         return (0);
 669 }
 670
 671
 672 /*********************************************************************
 673  *  Transmit entry point
 674  *
 675  *  ixgbe_start is called by the stack to initiate a transmit.
 676  *  The driver will remain in this routine as long as there are
 677  *  packets to transmit and transmit resources are available.
 678  *  In case resources are not available stack is notified and
 679  *  the packet is requeued.
 680  **********************************************************************/
 681
 682 static void
 683 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 684 {
 685         struct mbuf    *m_head;
 686         struct adapter *adapter = txr->adapter;
 687
 688         IXGBE_TX_LOCK_ASSERT(txr);
 689
 690         if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
 691                 return;
 692         if (!adapter->link_active)
 693                 return;
 694
 695         while (!ifq_is_empty(&ifp->if_snd)) {
 696                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
 697                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 698                         break;
 699                 }
 700
 701                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
 702                 if (m_head == NULL)
 703                         break;
 704
 705                 if (ixgbe_xmit(txr, &m_head)) {
 706 #if 0 /* XXX: prepend to an ALTQ queue ? */
 707                         if (m_head != NULL)
 708                                 IF_PREPEND(&ifp->if_snd, m_head);
 709 #endif
 710                         if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
 711                                 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 712                         break;
 713                 }
 714                 /* Send a copy of the frame to the BPF listener */
 715                 ETHER_BPF_MTAP(ifp, m_head);
 716
 717                 /* Set watchdog on */
 718                 txr->watchdog_time = ticks;
 719                 txr->queue_status = IXGBE_QUEUE_WORKING;
 720
 721         }
 722         return;
 723 }
 724
 725 /*
 726  * Legacy TX start - called by the stack, this
 727  * always uses the first tx ring, and should
 728  * not be used with multiqueue tx enabled.
 729  */
 730 static void
 731 ixgbe_start(struct ifnet *ifp)
 732 {
 733         struct adapter *adapter = ifp->if_softc;
 734         struct tx_ring  *txr = adapter->tx_rings;
 735
 736         if (ifp->if_flags & IFF_RUNNING) {
 737                 IXGBE_TX_LOCK(txr);
 738                 ixgbe_start_locked(txr, ifp);
 739                 IXGBE_TX_UNLOCK(txr);
 740         }
 741         return;
 742 }
 743
 744 #if 0 /* __FreeBSD_version >= 800000 */
 745 /*
 746 ** Multiqueue Transmit driver
 747 **
 748 */
 749 static int
 750 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
 751 {
 752         struct adapter  *adapter = ifp->if_softc;
 753         struct ix_queue *que;
 754         struct tx_ring  *txr;
 755         int             i = 0, err = 0;
 756
 757         /* Which queue to use */
 758         if ((m->m_flags & M_FLOWID) != 0)
 759                 i = m->m_pkthdr.flowid % adapter->num_queues;
 760         else
 761                 i = curcpu % adapter->num_queues;
 762
 763         txr = &adapter->tx_rings[i];
 764         que = &adapter->queues[i];
 765
 766         if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
 767             IXGBE_TX_TRYLOCK(txr)) {
 768                 err = ixgbe_mq_start_locked(ifp, txr, m);
 769                 IXGBE_TX_UNLOCK(txr);
 770         } else {
 771                 err = drbr_enqueue(ifp, txr->br, m);
 772                 taskqueue_enqueue(que->tq, &que->que_task);
 773         }
 774
 775         return (err);
 776 }
 777
 778 static int
 779 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 780 {
 781         struct adapter  *adapter = txr->adapter;
 782         struct mbuf     *next;
 783         int             enqueued, err = 0;
 784
 785         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
 786             (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
 787             adapter->link_active == 0) {
 788                 if (m != NULL)
 789                         err = drbr_enqueue(ifp, txr->br, m);
 790                 return (err);
 791         }
 792
 793         enqueued = 0;
 794         if (m == NULL) {
 795                 next = drbr_dequeue(ifp, txr->br);
 796         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 797                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 798                         return (err);
 799                 next = drbr_dequeue(ifp, txr->br);
 800         } else
 801                 next = m;
 802
 803         /* Process the queue */
 804         while (next != NULL) {
 805                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 806                         if (next != NULL)
 807                                 err = drbr_enqueue(ifp, txr->br, next);
 808                         break;
 809                 }
 810                 enqueued++;
 811                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 812                 /* Send a copy of the frame to the BPF listener */
 813                 ETHER_BPF_MTAP(ifp, next);
 814                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 815                         break;
 816                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
 817                         ixgbe_txeof(txr);
 818                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
 819                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 820                         break;
 821                 }
 822                 next = drbr_dequeue(ifp, txr->br);
 823         }
 824
 825         if (enqueued > 0) {
 826                 /* Set watchdog on */
 827                 txr->queue_status |= IXGBE_QUEUE_WORKING;
 828                 txr->watchdog_time = ticks;
 829         }
 830
 831         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
 832                 ixgbe_txeof(txr);
 833
 834         return (err);
 835 }
 836
 837 /*
 838 ** Flush all ring buffers
 839 */
 840 static void
 841 ixgbe_qflush(struct ifnet *ifp)
 842 {
 843         struct adapter  *adapter = ifp->if_softc;
 844         struct tx_ring  *txr = adapter->tx_rings;
 845         struct mbuf     *m;
 846
 847         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 848                 IXGBE_TX_LOCK(txr);
 849                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 850                         m_freem(m);
 851                 IXGBE_TX_UNLOCK(txr);
 852         }
 853         if_qflush(ifp);
 854 }
 855 #endif /* __FreeBSD_version >= 800000 */
 856
 857 /*********************************************************************
 858  *  Ioctl entry point
 859  *
 860  *  ixgbe_ioctl is called when the user wants to configure the
 861  *  interface.
 862  *
 863  *  return 0 on success, positive on failure
 864  **********************************************************************/
 865
 866 static int
 867 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
 868 {
 869         struct adapter  *adapter = ifp->if_softc;
 870         struct ifreq    *ifr = (struct ifreq *) data;
 871 #if defined(INET) || defined(INET6)
 872         struct ifaddr *ifa = (struct ifaddr *)data;
 873         bool            avoid_reset = FALSE;
 874 #endif
 875         int             error = 0;
 876
 877         switch (command) {
 878
 879         case SIOCSIFADDR:
 880 #ifdef INET
 881                 if (ifa->ifa_addr->sa_family == AF_INET)
 882                         avoid_reset = TRUE;
 883 #endif
 884 #ifdef INET6
 885                 if (ifa->ifa_addr->sa_family == AF_INET6)
 886                         avoid_reset = TRUE;
 887 #endif
 888 #if defined(INET) || defined(INET6)
 889                 /*
 890                 ** Calling init results in link renegotiation,
 891                 ** so we avoid doing it when possible.
 892                 */
 893                 if (avoid_reset) {
 894                         ifp->if_flags |= IFF_UP;
 895                         if (!(ifp->if_flags & IFF_RUNNING))
 896                                 ixgbe_init(adapter);
 897                         if (!(ifp->if_flags & IFF_NOARP))
 898                                 arp_ifinit(ifp, ifa);
 899                 } else
 900                         error = ether_ioctl(ifp, command, data);
 901 #endif
 902                 break;
 903         case SIOCSIFMTU:
 904                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 905                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 906                         error = EINVAL;
 907                 } else {
 908                         IXGBE_CORE_LOCK(adapter);
 909                         ifp->if_mtu = ifr->ifr_mtu;
 910                         adapter->max_frame_size =
 911                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 912                         ixgbe_init_locked(adapter);
 913                         IXGBE_CORE_UNLOCK(adapter);
 914                 }
 915                 break;
 916         case SIOCSIFFLAGS:
 917                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 918                 IXGBE_CORE_LOCK(adapter);
 919                 if (ifp->if_flags & IFF_UP) {
 920                         if ((ifp->if_flags & IFF_RUNNING)) {
 921                                 if ((ifp->if_flags ^ adapter->if_flags) &
 922                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 923                                         ixgbe_set_promisc(adapter);
 924                                 }
 925                         } else
 926                                 ixgbe_init_locked(adapter);
 927                 } else
 928                         if (ifp->if_flags & IFF_RUNNING)
 929                                 ixgbe_stop(adapter);
 930                 adapter->if_flags = ifp->if_flags;
 931                 IXGBE_CORE_UNLOCK(adapter);
 932                 break;
 933         case SIOCADDMULTI:
 934         case SIOCDELMULTI:
 935                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 936                 if (ifp->if_flags & IFF_RUNNING) {
 937                         IXGBE_CORE_LOCK(adapter);
 938                         ixgbe_disable_intr(adapter);
 939                         ixgbe_set_multi(adapter);
 940                         ixgbe_enable_intr(adapter);
 941                         IXGBE_CORE_UNLOCK(adapter);
 942                 }
 943                 break;
 944         case SIOCSIFMEDIA:
 945         case SIOCGIFMEDIA:
 946                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
 947                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 948                 break;
 949         case SIOCSIFCAP:
 950         {
 951                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 952                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
 953                 if (mask & IFCAP_HWCSUM)
 954                         ifp->if_capenable ^= IFCAP_HWCSUM;
 955                 if (mask & IFCAP_TSO4)
 956                         ifp->if_capenable ^= IFCAP_TSO4;
 957                 if (mask & IFCAP_TSO6)
 958                         ifp->if_capenable ^= IFCAP_TSO6;
 959 #if 0 /* NET_LRO */
 960                 if (mask & IFCAP_LRO)
 961                         ifp->if_capenable ^= IFCAP_LRO;
 962 #endif
 963                 if (mask & IFCAP_VLAN_HWTAGGING)
 964                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 965                 if (mask & IFCAP_VLAN_HWFILTER)
 966                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
 967 #if 0 /* NET_TSO */
 968                 if (mask & IFCAP_VLAN_HWTSO)
 969                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
 970 #endif
 971                 if (ifp->if_flags & IFF_RUNNING) {
 972                         IXGBE_CORE_LOCK(adapter);
 973                         ixgbe_init_locked(adapter);
 974                         IXGBE_CORE_UNLOCK(adapter);
 975                 }
 976 #if 0
 977                 VLAN_CAPABILITIES(ifp);
 978 #endif
 979                 break;
 980         }
 981
 982         default:
 983                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
 984                 error = ether_ioctl(ifp, command, data);
 985                 break;
 986         }
 987
 988         return (error);
 989 }
 990
 991 /*********************************************************************
 992  *  Init entry point
 993  *
 994  *  This routine is used in two ways. It is used by the stack as
 995  *  init entry point in network interface structure. It is also used
 996  *  by the driver as a hw/sw initialization routine to get to a
 997  *  consistent state.
 998  *
 999  *  return 0 on success, positive on failure
1000  **********************************************************************/
1001 #define IXGBE_MHADD_MFS_SHIFT 16
1002
1003 static void
1004 ixgbe_init_locked(struct adapter *adapter)
1005 {
1006         struct ifnet   *ifp = adapter->ifp;
1007         device_t        dev = adapter->dev;
1008         struct ixgbe_hw *hw = &adapter->hw;
1009         u32             k, txdctl, mhadd, gpie;
1010         u32             rxdctl, rxctrl;
1011
1012         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1013         INIT_DEBUGOUT("ixgbe_init: begin");
1014         hw->adapter_stopped = FALSE;
1015         ixgbe_stop_adapter(hw);
1016         callout_stop(&adapter->timer);
1017
1018         /* reprogram the RAR[0] in case user changed it. */
1019         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1020
1021         /* Get the latest mac address, User can use a LAA */
1022         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1023               IXGBE_ETH_LENGTH_OF_ADDRESS);
1024         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1025         hw->addr_ctrl.rar_used_count = 1;
1026
1027         /* Set the various hardware offload abilities */
1028         ifp->if_hwassist = 0;
1029         if (ifp->if_capenable & IFCAP_TSO)
1030                 ifp->if_hwassist |= CSUM_TSO;
1031         if (ifp->if_capenable & IFCAP_TXCSUM) {
1032                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1033 #if 0
1034                 if (hw->mac.type != ixgbe_mac_82598EB)
1035                         ifp->if_hwassist |= CSUM_SCTP;
1036 #endif
1037         }
1038
1039         /* Prepare transmit descriptors and buffers */
1040         if (ixgbe_setup_transmit_structures(adapter)) {
1041                 device_printf(dev,"Could not setup transmit structures\n");
1042                 ixgbe_stop(adapter);
1043                 return;
1044         }
1045
1046         ixgbe_init_hw(hw);
1047         ixgbe_initialize_transmit_units(adapter);
1048
1049         /* Setup Multicast table */
1050         ixgbe_set_multi(adapter);
1051
1052         /*
1053         ** Determine the correct mbuf pool
1054         ** for doing jumbo/headersplit
1055         */
1056         if (adapter->max_frame_size <= 2048)
1057                 adapter->rx_mbuf_sz = MCLBYTES;
1058         else if (adapter->max_frame_size <= 4096)
1059                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1060         else if (adapter->max_frame_size <= 9216)
1061                 adapter->rx_mbuf_sz = MJUM9BYTES;
1062         else
1063                 adapter->rx_mbuf_sz = MJUM16BYTES;
1064
1065         /* Prepare receive descriptors and buffers */
1066         if (ixgbe_setup_receive_structures(adapter)) {
1067                 device_printf(dev,"Could not setup receive structures\n");
1068                 ixgbe_stop(adapter);
1069                 return;
1070         }
1071
1072         /* Configure RX settings */
1073         ixgbe_initialize_receive_units(adapter);
1074
1075         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1076
1077         /* Enable Fan Failure Interrupt */
1078         gpie |= IXGBE_SDP1_GPIEN;
1079
1080         /* Add for Module detection */
1081         if (hw->mac.type == ixgbe_mac_82599EB)
1082                 gpie |= IXGBE_SDP2_GPIEN;
1083
1084         /* Thermal Failure Detection */
1085         if (hw->mac.type == ixgbe_mac_X540)
1086                 gpie |= IXGBE_SDP0_GPIEN;
1087
1088         if (adapter->msix > 1) {
1089                 /* Enable Enhanced MSIX mode */
1090                 gpie |= IXGBE_GPIE_MSIX_MODE;
1091                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1092                     IXGBE_GPIE_OCD;
1093         }
1094         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1095
1096         /* Set MTU size */
1097         if (ifp->if_mtu > ETHERMTU) {
1098                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1099                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1100                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1101                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1102         }
1103
1104         /* Now enable all the queues */
1105
1106         for (int i = 0; i < adapter->num_queues; i++) {
1107                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1108                 txdctl |= IXGBE_TXDCTL_ENABLE;
1109                 /* Set WTHRESH to 8, burst writeback */
1110                 txdctl |= (8 << 16);
1111                 /*
1112                  * When the internal queue falls below PTHRESH (32),
1113                  * start prefetching as long as there are at least
1114                  * HTHRESH (1) buffers ready. The values are taken
1115                  * from the Intel linux driver 3.8.21.
1116                  * Prefetching enables tx line rate even with 1 queue.
1117                  */
1118                 txdctl |= (32 << 0) | (1 << 8);
1119                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1120         }
1121
1122         for (int i = 0; i < adapter->num_queues; i++) {
1123                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1124                 if (hw->mac.type == ixgbe_mac_82598EB) {
1125                         /*
1126                         ** PTHRESH = 21
1127                         ** HTHRESH = 4
1128                         ** WTHRESH = 8
1129                         */
1130                         rxdctl &= ~0x3FFFFF;
1131                         rxdctl |= 0x080420;
1132                 }
1133                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1134                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1135                 for (k = 0; k < 10; k++) {
1136                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1137                             IXGBE_RXDCTL_ENABLE)
1138                                 break;
1139                         else
1140                                 msec_delay(1);
1141                 }
1142                 wmb();
1143 #ifdef DEV_NETMAP
1144                 /*
1145                  * In netmap mode, we must preserve the buffers made
1146                  * available to userspace before the if_init()
1147                  * (this is true by default on the TX side, because
1148                  * init makes all buffers available to userspace).
1149                  *
1150                  * netmap_reset() and the device specific routines
1151                  * (e.g. ixgbe_setup_receive_rings()) map these
1152                  * buffers at the end of the NIC ring, so here we
1153                  * must set the RDT (tail) register to make sure
1154                  * they are not overwritten.
1155                  *
1156                  * In this driver the NIC ring starts at RDH = 0,
1157                  * RDT points to the last slot available for reception (?),
1158                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1159                  */
1160                 if (ifp->if_capenable & IFCAP_NETMAP) {
1161                         struct netmap_adapter *na = NA(adapter->ifp);
1162                         struct netmap_kring *kring = &na->rx_rings[i];
1163                         int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1164
1165                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1166                 } else
1167 #endif /* DEV_NETMAP */
1168                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1169         }
1170
1171         /* Set up VLAN support and filter */
1172         ixgbe_setup_vlan_hw_support(adapter);
1173
1174         /* Enable Receive engine */
1175         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1176         if (hw->mac.type == ixgbe_mac_82598EB)
1177                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1178         rxctrl |= IXGBE_RXCTRL_RXEN;
1179         ixgbe_enable_rx_dma(hw, rxctrl);
1180
1181         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1182
1183         /* Set up MSI/X routing */
1184         if (ixgbe_enable_msix)  {
1185                 ixgbe_configure_ivars(adapter);
1186                 /* Set up auto-mask */
1187                 if (hw->mac.type == ixgbe_mac_82598EB)
1188                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1189                 else {
1190                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1191                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1192                 }
1193         } else {  /* Simple settings for Legacy/MSI */
1194                 ixgbe_set_ivar(adapter, 0, 0, 0);
1195                 ixgbe_set_ivar(adapter, 0, 0, 1);
1196                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1197         }
1198
1199 #ifdef IXGBE_FDIR
1200         /* Init Flow director */
1201         if (hw->mac.type != ixgbe_mac_82598EB) {
1202                 u32 hdrm = 32 << fdir_pballoc;
1203
1204                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1205                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1206         }
1207 #endif
1208
1209         /*
1210         ** Check on any SFP devices that
1211         ** need to be kick-started
1212         */
1213         if (hw->phy.type == ixgbe_phy_none) {
1214                 int err = hw->phy.ops.identify(hw);
1215                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1216                         device_printf(dev,
1217                             "Unsupported SFP+ module type was detected.\n");
1218                         return;
1219                 }
1220         }
1221
1222         /* Set moderation on the Link interrupt */
1223         ixgbe_set_eitr(adapter, adapter->linkvec, IXGBE_LINK_ITR);
1224
1225         /* Config/Enable Link */
1226         ixgbe_config_link(adapter);
1227
1228         /* Hardware Packet Buffer & Flow Control setup */
1229         {
1230                 u32 rxpb, frame, size, tmp;
1231
1232                 frame = adapter->max_frame_size;
1233
1234                 /* Calculate High Water */
1235                 if (hw->mac.type == ixgbe_mac_X540)
1236                         tmp = IXGBE_DV_X540(frame, frame);
1237                 else
1238                         tmp = IXGBE_DV(frame, frame);
1239                 size = IXGBE_BT2KB(tmp);
1240                 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1241                 hw->fc.high_water[0] = rxpb - size;
1242
1243                 /* Now calculate Low Water */
1244                 if (hw->mac.type == ixgbe_mac_X540)
1245                         tmp = IXGBE_LOW_DV_X540(frame);
1246                 else
1247                         tmp = IXGBE_LOW_DV(frame);
1248                 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1249
1250                 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1251                 hw->fc.pause_time = IXGBE_FC_PAUSE;
1252                 hw->fc.send_xon = TRUE;
1253         }
1254         /* Initialize the FC settings */
1255         ixgbe_start_hw(hw);
1256
1257         /* And now turn on interrupts */
1258         ixgbe_enable_intr(adapter);
1259
1260         /* Now inform the stack we're ready */
1261         ifp->if_flags |= IFF_RUNNING;
1262         ifp->if_flags &= ~IFF_OACTIVE;
1263
1264         return;
1265 }
1266
1267 static void
1268 ixgbe_init(void *arg)
1269 {
1270         struct adapter *adapter = arg;
1271
1272         IXGBE_CORE_LOCK(adapter);
1273         ixgbe_init_locked(adapter);
1274         IXGBE_CORE_UNLOCK(adapter);
1275         return;
1276 }
1277
1278
1279 /*
1280 **
1281 ** MSIX Interrupt Handlers and Tasklets
1282 **
1283 */
1284
1285 static inline void
1286 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1287 {
1288         struct ixgbe_hw *hw = &adapter->hw;
1289         u64     queue = (u64)(1 << vector);
1290         u32     mask;
1291
1292         if (hw->mac.type == ixgbe_mac_82598EB) {
1293                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1294                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1295         } else {
1296                 mask = (queue & 0xFFFFFFFF);
1297                 if (mask)
1298                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1299                 mask = (queue >> 32);
1300                 if (mask)
1301                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1302         }
1303 }
1304
1305 static inline void
1306 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1307 {
1308         struct ixgbe_hw *hw = &adapter->hw;
1309         u64     queue = (u64)(1 << vector);
1310         u32     mask;
1311
1312         if (hw->mac.type == ixgbe_mac_82598EB) {
1313                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1314                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1315         } else {
1316                 mask = (queue & 0xFFFFFFFF);
1317                 if (mask)
1318                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1319                 mask = (queue >> 32);
1320                 if (mask)
1321                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1322         }
1323 }
1324
1325 static inline void
1326 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1327 {
1328         u32 mask;
1329
1330         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1331                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1332                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1333         } else {
1334                 mask = (queues & 0xFFFFFFFF);
1335                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1336                 mask = (queues >> 32);
1337                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1338         }
1339 }
1340
1341 /*********************************************************************
1342  *
1343  *  Legacy Interrupt Service routine
1344  *
1345  **********************************************************************/
1346
1347 static void
1348 ixgbe_legacy_irq(void *arg)
1349 {
1350         struct ix_queue *que = arg;
1351         struct adapter  *adapter = que->adapter;
1352         struct ixgbe_hw *hw = &adapter->hw;
1353         struct          tx_ring *txr = adapter->tx_rings;
1354         u32             reg_eicr;
1355
1356
1357         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1358
1359         ++que->irqs;
1360         if (reg_eicr == 0) {
1361                 ixgbe_enable_intr(adapter);
1362                 return;
1363         }
1364
1365         ixgbe_rxeof(que, adapter->rx_process_limit);
1366
1367         IXGBE_TX_LOCK(txr);
1368         ixgbe_txeof(txr);
1369         if (!ifq_is_empty(&adapter->ifp->if_snd))
1370                 ixgbe_start_locked(txr, adapter->ifp);
1371         IXGBE_TX_UNLOCK(txr);
1372
1373         /* Check for fan failure */
1374         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1375             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1376                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1377                     "REPLACE IMMEDIATELY!!\n");
1378                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1379         }
1380
1381         /* Link status change */
1382         if (reg_eicr & IXGBE_EICR_LSC) {
1383                 ixgbe_check_link(&adapter->hw,
1384                     &adapter->link_speed, &adapter->link_up, 0);
1385                 ixgbe_update_link_status(adapter);
1386         }
1387
1388         ixgbe_enable_intr(adapter);
1389 }
1390
1391
1392 /*********************************************************************
1393  *
1394  *  MSIX Queue Interrupt Service routine
1395  *
1396  **********************************************************************/
1397 void
1398 ixgbe_msix_que(void *arg)
1399 {
1400         struct ix_queue *que = arg;
1401         struct adapter  *adapter = que->adapter;
1402         struct tx_ring  *txr = que->txr;
1403
1404         ixgbe_disable_queue(adapter, que->msix);
1405         ++que->irqs;
1406
1407         ixgbe_rxeof(que, adapter->rx_process_limit);
1408
1409         IXGBE_TX_LOCK(txr);
1410         ixgbe_txeof(txr);
1411         if (!ifq_is_empty(&adapter->ifp->if_snd))
1412                 ixgbe_start_locked(txr, adapter->ifp);
1413         IXGBE_TX_UNLOCK(txr);
1414
1415         /* Reenable this interrupt */
1416         ixgbe_enable_queue(adapter, que->msix);
1417 }
1418
1419
1420 static void
1421 ixgbe_msix_link(void *arg)
1422 {
1423         struct adapter  *adapter = arg;
1424         struct ixgbe_hw *hw = &adapter->hw;
1425         u32             reg_eicr;
1426
1427         ++adapter->link_irq;
1428
1429         /* First get the cause */
1430         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1431         /* Clear interrupt with write */
1432         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1433
1434         /* Link status change */
1435         if (reg_eicr & IXGBE_EICR_LSC) {
1436                 ixgbe_check_link(&adapter->hw,
1437                     &adapter->link_speed, &adapter->link_up, 0);
1438                 ixgbe_update_link_status(adapter);
1439         }
1440
1441         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1442 #ifdef IXGBE_FDIR
1443                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1444                         /* This is probably overkill :) */
1445                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1446                                 return;
1447                         /* Disable the interrupt */
1448                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1449                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1450                 } else
1451 #endif
1452                 if (reg_eicr & IXGBE_EICR_ECC) {
1453                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1454                             "Please Reboot!!\n");
1455                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1456                 } else
1457
1458                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1459                         /* Clear the interrupt */
1460                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1461                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1462                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1463                         /* Clear the interrupt */
1464                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1465                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1466                 }
1467         }
1468
1469         /* Check for fan failure */
1470         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1471             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1472                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1473                     "REPLACE IMMEDIATELY!!\n");
1474                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1475         }
1476
1477         /* Check for over temp condition */
1478         if ((hw->mac.type == ixgbe_mac_X540) &&
1479             (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1480                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1481                     "PHY IS SHUT DOWN!!\n");
1482                 device_printf(adapter->dev, "System shutdown required\n");
1483                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1484         }
1485
1486         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1487         return;
1488 }
1489
1490 /*********************************************************************
1491  *
1492  *  Media Ioctl callback
1493  *
1494  *  This routine is called whenever the user queries the status of
1495  *  the interface using ifconfig.
1496  *
1497  **********************************************************************/
1498 static void
1499 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1500 {
1501         struct adapter *adapter = ifp->if_softc;
1502
1503         INIT_DEBUGOUT("ixgbe_media_status: begin");
1504         IXGBE_CORE_LOCK(adapter);
1505         ixgbe_update_link_status(adapter);
1506
1507         ifmr->ifm_status = IFM_AVALID;
1508         ifmr->ifm_active = IFM_ETHER;
1509
1510         if (!adapter->link_active) {
1511                 IXGBE_CORE_UNLOCK(adapter);
1512                 return;
1513         }
1514
1515         ifmr->ifm_status |= IFM_ACTIVE;
1516
1517         switch (adapter->link_speed) {
1518                 case IXGBE_LINK_SPEED_100_FULL:
1519                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1520                         break;
1521                 case IXGBE_LINK_SPEED_1GB_FULL:
1522                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1523                         break;
1524                 case IXGBE_LINK_SPEED_10GB_FULL:
1525                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1526                         break;
1527         }
1528
1529         IXGBE_CORE_UNLOCK(adapter);
1530
1531         return;
1532 }
1533
1534 /*********************************************************************
1535  *
1536  *  Media Ioctl callback
1537  *
1538  *  This routine is called when the user changes speed/duplex using
1539  *  media/mediopt option with ifconfig.
1540  *
1541  **********************************************************************/
1542 static int
1543 ixgbe_media_change(struct ifnet * ifp)
1544 {
1545         struct adapter *adapter = ifp->if_softc;
1546         struct ifmedia *ifm = &adapter->media;
1547
1548         INIT_DEBUGOUT("ixgbe_media_change: begin");
1549
1550         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1551                 return (EINVAL);
1552
1553         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1554         case IFM_AUTO:
1555                 adapter->hw.phy.autoneg_advertised =
1556                     IXGBE_LINK_SPEED_100_FULL |
1557                     IXGBE_LINK_SPEED_1GB_FULL |
1558                     IXGBE_LINK_SPEED_10GB_FULL;
1559                 break;
1560         default:
1561                 device_printf(adapter->dev, "Only auto media type\n");
1562                 return (EINVAL);
1563         }
1564
1565         return (0);
1566 }
1567
1568 /*********************************************************************
1569  *
1570  *  This routine maps the mbufs to tx descriptors, allowing the
1571  *  TX engine to transmit the packets.
1572  *      - return 0 on success, positive on failure
1573  *
1574  **********************************************************************/
1575
1576 static int
1577 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1578 {
1579         struct adapter  *adapter = txr->adapter;
1580         u32             olinfo_status = 0, cmd_type_len;
1581         u32             paylen = 0;
1582         int             i, j, error, nsegs, maxsegs;
1583         int             first, last = 0;
1584         struct mbuf     *m_head;
1585         bus_dma_segment_t segs[adapter->num_segs];
1586         bus_dmamap_t    map;
1587         struct ixgbe_tx_buf *txbuf;
1588         union ixgbe_adv_tx_desc *txd = NULL;
1589
1590         m_head = *m_headp;
1591
1592         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1593                 error = ixgbe_tso_pullup(txr, m_headp);
1594                 if (error)
1595                         return error;
1596                 m_head = *m_headp;
1597         }
1598
1599         /* Basic descriptor defines */
1600         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1601             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1602
1603         if (m_head->m_flags & M_VLANTAG)
1604                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1605
1606         /*
1607          * Important to capture the first descriptor
1608          * used because it will contain the index of
1609          * the one we tell the hardware to report back
1610          */
1611         first = txr->next_avail_desc;
1612         txbuf = &txr->tx_buffers[first];
1613         map = txbuf->map;
1614
1615         /*
1616          * Map the packet for DMA.
1617          */
1618         maxsegs = txr->tx_avail - IXGBE_TX_RESERVED;
1619         if (maxsegs > adapter->num_segs)
1620                 maxsegs = adapter->num_segs;
1621
1622         error = bus_dmamap_load_mbuf_defrag(txr->txtag, map, m_headp,
1623             segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1624         if (error) {
1625                 if (error == ENOBUFS)
1626                         adapter->mbuf_defrag_failed++;
1627                 else
1628                         adapter->no_tx_dma_setup++;
1629
1630                 m_freem(*m_headp);
1631                 *m_headp = NULL;
1632                 return (error);
1633         }
1634
1635         /* Make certain there are enough descriptors */
1636         if (nsegs > txr->tx_avail - 2) {
1637                 txr->no_desc_avail++;
1638                 error = ENOBUFS;
1639                 goto xmit_fail;
1640         }
1641         m_head = *m_headp;
1642
1643         /*
1644         ** Set up the appropriate offload context
1645         ** this becomes the first descriptor of
1646         ** a packet.
1647         */
1648         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1649                 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1650                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1651                         olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1652                         olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1653                         ++adapter->tso_tx;
1654                 } else
1655                         return (ENXIO);
1656         } else if (ixgbe_tx_ctx_setup(txr, m_head))
1657                 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1658
1659 #ifdef IXGBE_IEEE1588
1660         /* This is changing soon to an mtag detection */
1661         if (we detect this mbuf has a TSTAMP mtag)
1662                 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1663 #endif
1664
1665 #ifdef IXGBE_FDIR
1666         /* Do the flow director magic */
1667         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1668                 ++txr->atr_count;
1669                 if (txr->atr_count >= atr_sample_rate) {
1670                         ixgbe_atr(txr, m_head);
1671                         txr->atr_count = 0;
1672                 }
1673         }
1674 #endif
1675         /* Record payload length */
1676         if (paylen == 0)
1677                 olinfo_status |= m_head->m_pkthdr.len <<
1678                     IXGBE_ADVTXD_PAYLEN_SHIFT;
1679
1680         i = txr->next_avail_desc;
1681         for (j = 0; j < nsegs; j++) {
1682                 bus_size_t seglen;
1683                 bus_addr_t segaddr;
1684
1685                 txbuf = &txr->tx_buffers[i];
1686                 txd = &txr->tx_base[i];
1687                 seglen = segs[j].ds_len;
1688                 segaddr = htole64(segs[j].ds_addr);
1689
1690                 txd->read.buffer_addr = segaddr;
1691                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1692                     cmd_type_len |seglen);
1693                 txd->read.olinfo_status = htole32(olinfo_status);
1694                 last = i; /* descriptor that will get completion IRQ */
1695
1696                 if (++i == adapter->num_tx_desc)
1697                         i = 0;
1698
1699                 txbuf->m_head = NULL;
1700                 txbuf->eop_index = -1;
1701         }
1702
1703         txd->read.cmd_type_len |=
1704             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1705         txr->tx_avail -= nsegs;
1706         txr->next_avail_desc = i;
1707
1708         txbuf->m_head = m_head;
1709         /* Swap the dma map between the first and last descriptor */
1710         txr->tx_buffers[first].map = txbuf->map;
1711         txbuf->map = map;
1712         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1713
1714         /* Set the index of the descriptor that will be marked done */
1715         txbuf = &txr->tx_buffers[first];
1716         txbuf->eop_index = last;
1717
1718         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1719             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1720         /*
1721          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1722          * hardware that this frame is available to transmit.
1723          */
1724         ++txr->total_packets;
1725         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1726
1727         return (0);
1728
1729 xmit_fail:
1730         bus_dmamap_unload(txr->txtag, txbuf->map);
1731         return (error);
1732
1733 }
1734
1735 static void
1736 ixgbe_set_promisc(struct adapter *adapter)
1737 {
1738         u_int32_t       reg_rctl;
1739         struct ifnet   *ifp = adapter->ifp;
1740
1741         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1742         reg_rctl &= (~IXGBE_FCTRL_UPE);
1743         reg_rctl &= (~IXGBE_FCTRL_MPE);
1744         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1745
1746         if (ifp->if_flags & IFF_PROMISC) {
1747                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1748                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1749         } else if (ifp->if_flags & IFF_ALLMULTI) {
1750                 reg_rctl |= IXGBE_FCTRL_MPE;
1751                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1752                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1753         }
1754         return;
1755 }
1756
1757
1758 /*********************************************************************
1759  *  Multicast Update
1760  *
1761  *  This routine is called whenever multicast address list is updated.
1762  *
1763  **********************************************************************/
1764 #define IXGBE_RAR_ENTRIES 16
1765
1766 static void
1767 ixgbe_set_multi(struct adapter *adapter)
1768 {
1769         u32     fctrl;
1770         u8      *mta;
1771         u8      *update_ptr;
1772         struct  ifmultiaddr *ifma;
1773         int     mcnt = 0;
1774         struct ifnet   *ifp = adapter->ifp;
1775
1776         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1777
1778         mta = adapter->mta;
1779         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1780             MAX_NUM_MULTICAST_ADDRESSES);
1781
1782         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1783         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1784         if (ifp->if_flags & IFF_PROMISC)
1785                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1786         else if (ifp->if_flags & IFF_ALLMULTI) {
1787                 fctrl |= IXGBE_FCTRL_MPE;
1788                 fctrl &= ~IXGBE_FCTRL_UPE;
1789         } else
1790                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1791
1792         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1793
1794         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1795                 if (ifma->ifma_addr->sa_family != AF_LINK)
1796                         continue;
1797                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1798                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1799                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1800                 mcnt++;
1801         }
1802
1803         update_ptr = mta;
1804         ixgbe_update_mc_addr_list(&adapter->hw,
1805             update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1806
1807         return;
1808 }
1809
1810 /*
1811  * This is an iterator function now needed by the multicast
1812  * shared code. It simply feeds the shared code routine the
1813  * addresses in the array of ixgbe_set_multi() one by one.
1814  */
1815 static u8 *
1816 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1817 {
1818         u8 *addr = *update_ptr;
1819         u8 *newptr;
1820         *vmdq = 0;
1821
1822         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1823         *update_ptr = newptr;
1824         return addr;
1825 }
1826
1827
1828 /*********************************************************************
1829  *  Timer routine
1830  *
1831  *  This routine checks for link status,updates statistics,
1832  *  and runs the watchdog check.
1833  *
1834  **********************************************************************/
1835
1836 static void
1837 ixgbe_local_timer(void *arg)
1838 {
1839         struct adapter  *adapter = arg;
1840         device_t        dev = adapter->dev;
1841         struct ifnet    *ifp = adapter->ifp;
1842         struct ix_queue *que = adapter->queues;
1843         struct tx_ring  *txr = adapter->tx_rings;
1844         int             hung, busy, paused;
1845
1846         IXGBE_CORE_LOCK(adapter);
1847         hung = busy = paused = 0;
1848
1849         /* Check for pluggable optics */
1850         if (adapter->sfp_probe)
1851                 if (!ixgbe_sfp_probe(adapter))
1852                         goto out; /* Nothing to do */
1853
1854         ixgbe_update_link_status(adapter);
1855         ixgbe_update_stats_counters(adapter);
1856
1857         /*
1858          * If the interface has been paused
1859          * then don't do the watchdog check
1860          */
1861         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
1862                 paused = 1;
1863
1864         /*
1865         ** Check the TX queues status
1866         **      - central locked handling of OACTIVE
1867         **      - watchdog only if all queues show hung
1868         */
1869         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
1870                 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
1871                     (paused == 0))
1872                         ++hung;
1873                 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
1874                         ++busy;
1875         }
1876         /* Only truely watchdog if all queues show hung */
1877         if (hung == adapter->num_queues)
1878                 goto watchdog;
1879         /* Only turn off the stack flow when ALL are depleted */
1880         if (busy == adapter->num_queues)
1881                 ifp->if_flags |= IFF_OACTIVE;
1882         else if ((ifp->if_flags & IFF_OACTIVE) &&
1883             (busy < adapter->num_queues))
1884                 ifp->if_flags &= ~IFF_OACTIVE;
1885
1886 out:
1887         ixgbe_rearm_queues(adapter, adapter->que_mask);
1888         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1889         IXGBE_CORE_UNLOCK(adapter);
1890         return;
1891
1892 watchdog:
1893         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
1894         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
1895             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
1896             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
1897         device_printf(dev,"TX(%d) desc avail = %d,"
1898             "Next TX to Clean = %d\n",
1899             txr->me, txr->tx_avail, txr->next_to_clean);
1900         adapter->ifp->if_flags &= ~IFF_RUNNING;
1901         adapter->watchdog_events++;
1902         ixgbe_init_locked(adapter);
1903
1904         IXGBE_CORE_UNLOCK(adapter);
1905 }
1906
1907 /*
1908 ** Note: this routine updates the OS on the link state
1909 **      the real check of the hardware only happens with
1910 **      a link interrupt.
1911 */
1912 static void
1913 ixgbe_update_link_status(struct adapter *adapter)
1914 {
1915         struct ifnet    *ifp = adapter->ifp;
1916         struct tx_ring *txr = adapter->tx_rings;
1917         device_t dev = adapter->dev;
1918
1919
1920         if (adapter->link_up){
1921                 if (adapter->link_active == FALSE) {
1922                         if (bootverbose)
1923                                 device_printf(dev,"Link is up %d Gbps %s \n",
1924                                     ((adapter->link_speed == 128)? 10:1),
1925                                     "Full Duplex");
1926                         adapter->link_active = TRUE;
1927                         /* Update any Flow Control changes */
1928                         ixgbe_fc_enable(&adapter->hw);
1929                         ifp->if_link_state = LINK_STATE_UP;
1930                         if_link_state_change(ifp);
1931                 }
1932         } else { /* Link down */
1933                 if (adapter->link_active == TRUE) {
1934                         if (bootverbose)
1935                                 device_printf(dev,"Link is Down\n");
1936                         ifp->if_link_state = LINK_STATE_DOWN;
1937                         if_link_state_change(ifp);
1938                         adapter->link_active = FALSE;
1939                         for (int i = 0; i < adapter->num_queues;
1940                             i++, txr++)
1941                                 txr->queue_status = IXGBE_QUEUE_IDLE;
1942                 }
1943         }
1944
1945         return;
1946 }
1947
1948
1949 /*********************************************************************
1950  *
1951  *  This routine disables all traffic on the adapter by issuing a
1952  *  global reset on the MAC and deallocates TX/RX buffers.
1953  *
1954  **********************************************************************/
1955
1956 static void
1957 ixgbe_stop(void *arg)
1958 {
1959         struct ifnet   *ifp;
1960         struct adapter *adapter = arg;
1961         struct ixgbe_hw *hw = &adapter->hw;
1962         ifp = adapter->ifp;
1963
1964         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1965
1966         INIT_DEBUGOUT("ixgbe_stop: begin\n");
1967         ixgbe_disable_intr(adapter);
1968         callout_stop(&adapter->timer);
1969
1970         /* Let the stack know...*/
1971         ifp->if_flags &= ~IFF_RUNNING;
1972         ifp->if_flags |= IFF_OACTIVE;
1973
1974         ixgbe_reset_hw(hw);
1975         hw->adapter_stopped = FALSE;
1976         ixgbe_stop_adapter(hw);
1977         /* Turn off the laser */
1978         if (hw->phy.multispeed_fiber)
1979                 ixgbe_disable_tx_laser(hw);
1980
1981         /* reprogram the RAR[0] in case user changed it. */
1982         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1983
1984         return;
1985 }
1986
1987
1988 /*********************************************************************
1989  *
1990  *  Determine hardware revision.
1991  *
1992  **********************************************************************/
1993 static void
1994 ixgbe_identify_hardware(struct adapter *adapter)
1995 {
1996         device_t        dev = adapter->dev;
1997         struct ixgbe_hw *hw = &adapter->hw;
1998
1999         /* Save off the information about this board */
2000         hw->vendor_id = pci_get_vendor(dev);
2001         hw->device_id = pci_get_device(dev);
2002         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2003         hw->subsystem_vendor_id =
2004             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2005         hw->subsystem_device_id =
2006             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2007
2008         /* We need this here to set the num_segs below */
2009         ixgbe_set_mac_type(hw);
2010
2011         /* Pick up the 82599 and VF settings */
2012         if (hw->mac.type != ixgbe_mac_82598EB) {
2013                 hw->phy.smart_speed = ixgbe_smart_speed;
2014                 adapter->num_segs = IXGBE_82599_SCATTER;
2015         } else
2016                 adapter->num_segs = IXGBE_82598_SCATTER;
2017
2018         return;
2019 }
2020
2021 /*********************************************************************
2022  *
2023  *  Determine optic type
2024  *
2025  **********************************************************************/
2026 static void
2027 ixgbe_setup_optics(struct adapter *adapter)
2028 {
2029         struct ixgbe_hw *hw = &adapter->hw;
2030         int             layer;
2031
2032         layer = ixgbe_get_supported_physical_layer(hw);
2033
2034         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2035                 adapter->optics = IFM_10G_T;
2036                 return;
2037         }
2038
2039         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2040                 adapter->optics = IFM_1000_T;
2041                 return;
2042         }
2043
2044         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2045             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2046                 adapter->optics = IFM_10G_LR;
2047                 return;
2048         }
2049
2050         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2051                 adapter->optics = IFM_10G_SR;
2052                 return;
2053         }
2054
2055         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2056                 adapter->optics = IFM_10G_TWINAX;
2057                 return;
2058         }
2059
2060         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2061             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2062                 adapter->optics = IFM_10G_CX4;
2063                 return;
2064         }
2065
2066         /* If we get here just set the default */
2067         adapter->optics = IFM_ETHER | IFM_AUTO;
2068         return;
2069 }
2070
2071 /*********************************************************************
2072  *
2073  *  Setup the Legacy or MSI Interrupt handler
2074  *
2075  **********************************************************************/
2076 static int
2077 ixgbe_allocate_legacy(struct adapter *adapter)
2078 {
2079         device_t dev = adapter->dev;
2080         struct          ix_queue *que = adapter->queues;
2081         int error, rid = 0;
2082         unsigned int intr_flags;
2083
2084         /* MSI RID at 1 */
2085         if (adapter->msix == 1)
2086                 rid = 1;
2087
2088         /* Try allocating a MSI interrupt first */
2089         adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2090                 &rid, &intr_flags);
2091
2092         /* We allocate a single interrupt resource */
2093         adapter->res = bus_alloc_resource_any(dev,
2094             SYS_RES_IRQ, &rid, intr_flags);
2095         if (adapter->res == NULL) {
2096                 device_printf(dev, "Unable to allocate bus resource: "
2097                     "interrupt\n");
2098                 return (ENXIO);
2099         }
2100
2101         /* Tasklets for Link, SFP and Multispeed Fiber */
2102         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2103         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2104 #ifdef IXGBE_FDIR
2105         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2106 #endif
2107         adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2108             taskqueue_thread_enqueue, &adapter->tq);
2109         taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2110             device_get_nameunit(adapter->dev));
2111
2112         if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2113             ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2114                 device_printf(dev, "Failed to register fast interrupt "
2115                     "handler: %d\n", error);
2116                 taskqueue_free(adapter->tq);
2117                 adapter->tq = NULL;
2118                 return (error);
2119         }
2120         /* For simplicity in the handlers */
2121         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2122
2123         return (0);
2124 }
2125
2126
2127 /*********************************************************************
2128  *
2129  *  Setup MSIX Interrupt resources and handlers
2130  *
2131  **********************************************************************/
2132 static int
2133 ixgbe_allocate_msix(struct adapter *adapter)
2134 {
2135         device_t        dev = adapter->dev;
2136         struct          ix_queue *que = adapter->queues;
2137         int             error, rid, vector = 0;
2138         char            desc[16];
2139
2140         error = pci_setup_msix(dev);
2141         if (error) {
2142                 device_printf(dev, "MSI-X setup failed\n");
2143                 return (error);
2144         }
2145
2146         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2147                 rid = vector + 1;
2148
2149                 /*
2150                 ** Bind the msix vector, and thus the
2151                 ** ring to the corresponding cpu.
2152                 */
2153                 error = pci_alloc_msix_vector(dev, vector, &rid, i);
2154                 if (error) {
2155                         device_printf(dev, "pci_alloc_msix_vector failed\n");
2156                         return (error);
2157                 }
2158
2159                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2160                     RF_SHAREABLE | RF_ACTIVE);
2161                 if (que->res == NULL) {
2162                         device_printf(dev,"Unable to allocate"
2163                             " bus resource: que interrupt [%d]\n", vector);
2164                         return (ENXIO);
2165                 }
2166                 /* Set the handler function */
2167                 ksnprintf(desc, sizeof(desc), "%s que %d",
2168                     device_get_nameunit(dev), i);
2169                 error = bus_setup_intr_descr(dev, que->res, INTR_MPSAFE,
2170                     ixgbe_msix_que, que, &que->tag, &que->serializer, desc);
2171                 if (error) {
2172                         que->res = NULL;
2173                         device_printf(dev, "Failed to register QUE handler");
2174                         return (error);
2175                 }
2176                 que->msix = vector;
2177                 adapter->que_mask |= (u64)(1 << que->msix);
2178         }
2179
2180         /* and Link, bind vector to cpu #0 */
2181         rid = vector + 1;
2182         error = pci_alloc_msix_vector(dev, vector, &rid, 0);
2183         if (error) {
2184                 device_printf(dev, "pci_alloc_msix_vector failed\n");
2185                 return (error);
2186         }
2187         adapter->res = bus_alloc_resource_any(dev,
2188             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2189         if (!adapter->res) {
2190                 device_printf(dev,"Unable to allocate"
2191             " bus resource: Link interrupt [%d]\n", rid);
2192                 return (ENXIO);
2193         }
2194         /* Set the link handler function */
2195         error = bus_setup_intr_descr(dev, adapter->res, INTR_MPSAFE,
2196             ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer,
2197             "link");
2198         if (error) {
2199                 adapter->res = NULL;
2200                 device_printf(dev, "Failed to register LINK handler");
2201                 return (error);
2202         }
2203         pci_enable_msix(dev);
2204
2205         adapter->linkvec = vector;
2206         /* Tasklets for Link, SFP and Multispeed Fiber */
2207         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2208         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2209 #ifdef IXGBE_FDIR
2210         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2211 #endif
2212         adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2213             taskqueue_thread_enqueue, &adapter->tq);
2214         taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2215             device_get_nameunit(adapter->dev));
2216
2217         return (0);
2218 }
2219
2220 /*
2221  * Setup Either MSI/X or MSI
2222  */
2223 static int
2224 ixgbe_setup_msix(struct adapter *adapter)
2225 {
2226         device_t dev = adapter->dev;
2227         int rid, want, queues, msgs;
2228
2229         /* Override by tuneable */
2230         if (ixgbe_enable_msix == 0)
2231                 goto msi;
2232
2233         /* First try MSI/X */
2234         rid = PCIR_BAR(MSIX_82598_BAR);
2235         adapter->msix_mem = bus_alloc_resource_any(dev,
2236             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2237         if (!adapter->msix_mem) {
2238                 rid += 4;       /* 82599 maps in higher BAR */
2239                 adapter->msix_mem = bus_alloc_resource_any(dev,
2240                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2241         }
2242         if (!adapter->msix_mem) {
2243                 /* May not be enabled */
2244                 device_printf(adapter->dev,
2245                     "Unable to map MSIX table \n");
2246                 goto msi;
2247         }
2248
2249         msgs = pci_msix_count(dev);
2250         if (msgs == 0) { /* system has msix disabled */
2251                 bus_release_resource(dev, SYS_RES_MEMORY,
2252                     rid, adapter->msix_mem);
2253                 adapter->msix_mem = NULL;
2254                 goto msi;
2255         }
2256
2257         /* Figure out a reasonable auto config value */
2258         queues = (ncpus > (msgs-1)) ? (msgs-1) : ncpus;
2259
2260         if (ixgbe_num_queues != 0)
2261                 queues = ixgbe_num_queues;
2262         /* Set max queues to 8 when autoconfiguring */
2263         else if ((ixgbe_num_queues == 0) && (queues > 8))
2264                 queues = 8;
2265
2266         /*
2267         ** Want one vector (RX/TX pair) per queue
2268         ** plus an additional for Link.
2269         */
2270         want = queues + 1;
2271         if (msgs >= want)
2272                 msgs = want;
2273         else {
2274                 device_printf(adapter->dev,
2275                     "MSIX Configuration Problem, "
2276                     "%d vectors but %d queues wanted!\n",
2277                     msgs, want);
2278                 return (0); /* Will go to Legacy setup */
2279         }
2280         if (msgs) {
2281                 device_printf(adapter->dev,
2282                     "Using MSIX interrupts with %d vectors\n", msgs);
2283                 adapter->num_queues = queues;
2284                 return (msgs);
2285         }
2286 msi:
2287         msgs = pci_msi_count(dev);
2288         return (msgs);
2289 }
2290
2291
2292 static int
2293 ixgbe_allocate_pci_resources(struct adapter *adapter)
2294 {
2295         int             rid;
2296         device_t        dev = adapter->dev;
2297
2298         rid = PCIR_BAR(0);
2299         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2300             &rid, RF_ACTIVE);
2301
2302         if (!(adapter->pci_mem)) {
2303                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2304                 return (ENXIO);
2305         }
2306
2307         adapter->osdep.mem_bus_space_tag =
2308                 rman_get_bustag(adapter->pci_mem);
2309         adapter->osdep.mem_bus_space_handle =
2310                 rman_get_bushandle(adapter->pci_mem);
2311         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2312
2313         /* Legacy defaults */
2314         adapter->num_queues = 1;
2315         adapter->hw.back = &adapter->osdep;
2316
2317         /*
2318         ** Now setup MSI or MSI/X, should
2319         ** return us the number of supported
2320         ** vectors. (Will be 1 for MSI)
2321         */
2322         adapter->msix = ixgbe_setup_msix(adapter);
2323         return (0);
2324 }
2325
2326 static void
2327 ixgbe_free_pci_resources(struct adapter * adapter)
2328 {
2329         struct          ix_queue *que = adapter->queues;
2330         device_t        dev = adapter->dev;
2331         int             rid, memrid;
2332
2333         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2334                 memrid = PCIR_BAR(MSIX_82598_BAR);
2335         else
2336                 memrid = PCIR_BAR(MSIX_82599_BAR);
2337
2338         /*
2339         ** There is a slight possibility of a failure mode
2340         ** in attach that will result in entering this function
2341         ** before interrupt resources have been initialized, and
2342         ** in that case we do not want to execute the loops below
2343         ** We can detect this reliably by the state of the adapter
2344         ** res pointer.
2345         */
2346         if (adapter->res == NULL)
2347                 goto mem;
2348
2349         /*
2350         **  Release all msix queue resources:
2351         */
2352         for (int i = 0; i < adapter->num_queues; i++, que++) {
2353                 rid = que->msix + 1;
2354                 if (que->tag != NULL) {
2355                         bus_teardown_intr(dev, que->res, que->tag);
2356                         que->tag = NULL;
2357                 }
2358                 if (que->res != NULL)
2359                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2360         }
2361
2362
2363         /* Clean the Legacy or Link interrupt last */
2364         if (adapter->linkvec) /* we are doing MSIX */
2365                 rid = adapter->linkvec + 1;
2366         else
2367                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2368
2369         if (adapter->tag != NULL) {
2370                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2371                 adapter->tag = NULL;
2372         }
2373         if (adapter->res != NULL)
2374                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2375         if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2376                 pci_release_msi(adapter->dev);
2377
2378 mem:
2379         if (adapter->msix)
2380                 pci_release_msi(dev);
2381
2382         if (adapter->msix_mem != NULL)
2383                 bus_release_resource(dev, SYS_RES_MEMORY,
2384                     memrid, adapter->msix_mem);
2385
2386         if (adapter->pci_mem != NULL)
2387                 bus_release_resource(dev, SYS_RES_MEMORY,
2388                     PCIR_BAR(0), adapter->pci_mem);
2389
2390         return;
2391 }
2392
2393 /*********************************************************************
2394  *
2395  *  Setup networking device structure and register an interface.
2396  *
2397  **********************************************************************/
2398 static int
2399 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2400 {
2401         struct ixgbe_hw *hw = &adapter->hw;
2402         struct ifnet   *ifp;
2403
2404         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2405
2406         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2407         if (ifp == NULL) {
2408                 device_printf(dev, "can not allocate ifnet structure\n");
2409                 return (-1);
2410         }
2411         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2412         ifp->if_baudrate = 1000000000;
2413         ifp->if_init = ixgbe_init;
2414         ifp->if_softc = adapter;
2415         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2416         ifp->if_ioctl = ixgbe_ioctl;
2417         ifp->if_start = ixgbe_start;
2418 #if 0 /* __FreeBSD_version >= 800000 */
2419         ifp->if_transmit = ixgbe_mq_start;
2420         ifp->if_qflush = ixgbe_qflush;
2421 #endif
2422         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2423
2424         ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2425
2426         adapter->max_frame_size =
2427             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2428
2429         /*
2430          * Tell the upper layer(s) we support long frames.
2431          */
2432         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2433
2434         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2435         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2436         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2437 #if 0 /* NET_TSO */
2438                              |  IFCAP_VLAN_HWTSO
2439 #endif
2440                              |  IFCAP_VLAN_MTU;
2441         ifp->if_capenable = ifp->if_capabilities;
2442
2443         /* Don't enable LRO by default */
2444 #if 0 /* NET_LRO */
2445         ifp->if_capabilities |= IFCAP_LRO;
2446 #endif
2447
2448         /*
2449         ** Don't turn this on by default, if vlans are
2450         ** created on another pseudo device (eg. lagg)
2451         ** then vlan events are not passed thru, breaking
2452         ** operation, but with HW FILTER off it works. If
2453         ** using vlans directly on the ixgbe driver you can
2454         ** enable this and get full hardware tag filtering.
2455         */
2456         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2457
2458         /*
2459          * Specify the media types supported by this adapter and register
2460          * callbacks to update media and link information
2461          */
2462         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2463                      ixgbe_media_status);
2464         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2465         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2466         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2467                 ifmedia_add(&adapter->media,
2468                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2469                 ifmedia_add(&adapter->media,
2470                     IFM_ETHER | IFM_1000_T, 0, NULL);
2471         }
2472         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2473         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2474
2475         return (0);
2476 }
2477
2478 static void
2479 ixgbe_config_link(struct adapter *adapter)
2480 {
2481         struct ixgbe_hw *hw = &adapter->hw;
2482         u32     autoneg, err = 0;
2483         bool    sfp, negotiate;
2484
2485         sfp = ixgbe_is_sfp(hw);
2486
2487         if (sfp) {
2488                 if (hw->phy.multispeed_fiber) {
2489                         hw->mac.ops.setup_sfp(hw);
2490                         ixgbe_enable_tx_laser(hw);
2491                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2492                 } else
2493                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2494         } else {
2495                 if (hw->mac.ops.check_link)
2496                         err = ixgbe_check_link(hw, &autoneg,
2497                             &adapter->link_up, FALSE);
2498                 if (err)
2499                         goto out;
2500                 autoneg = hw->phy.autoneg_advertised;
2501                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2502                         err  = hw->mac.ops.get_link_capabilities(hw,
2503                             &autoneg, &negotiate);
2504                 if (err)
2505                         goto out;
2506                 if (hw->mac.ops.setup_link)
2507                         err = hw->mac.ops.setup_link(hw, autoneg,
2508                             negotiate, adapter->link_up);
2509         }
2510 out:
2511         return;
2512 }
2513
2514 /********************************************************************
2515  * Manage DMA'able memory.
2516  *******************************************************************/
2517 static void
2518 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2519 {
2520         if (error)
2521                 return;
2522         *(bus_addr_t *) arg = segs->ds_addr;
2523         return;
2524 }
2525
2526 static int
2527 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2528                 struct ixgbe_dma_alloc *dma, int mapflags)
2529 {
2530         device_t dev = adapter->dev;
2531         int             r;
2532
2533         r = bus_dma_tag_create(NULL,    /* parent */
2534                                DBA_ALIGN, 0,    /* alignment, bounds */
2535                                BUS_SPACE_MAXADDR,       /* lowaddr */
2536                                BUS_SPACE_MAXADDR,       /* highaddr */
2537                                NULL, NULL,      /* filter, filterarg */
2538                                size,    /* maxsize */
2539                                1,       /* nsegments */
2540                                size,    /* maxsegsize */
2541                                BUS_DMA_ALLOCNOW,        /* flags */
2542                                &dma->dma_tag);
2543         if (r != 0) {
2544                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2545                        "error %u\n", r);
2546                 goto fail_0;
2547         }
2548         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2549                              BUS_DMA_NOWAIT, &dma->dma_map);
2550         if (r != 0) {
2551                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2552                        "error %u\n", r);
2553                 goto fail_1;
2554         }
2555         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2556                             size,
2557                             ixgbe_dmamap_cb,
2558                             &dma->dma_paddr,
2559                             mapflags | BUS_DMA_NOWAIT);
2560         if (r != 0) {
2561                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2562                        "error %u\n", r);
2563                 goto fail_2;
2564         }
2565         dma->dma_size = size;
2566         return (0);
2567 fail_2:
2568         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2569 fail_1:
2570         bus_dma_tag_destroy(dma->dma_tag);
2571 fail_0:
2572         dma->dma_map = NULL;
2573         dma->dma_tag = NULL;
2574         return (r);
2575 }
2576
2577 static void
2578 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2579 {
2580         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2581             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2582         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2583         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2584         bus_dma_tag_destroy(dma->dma_tag);
2585 }
2586
2587
2588 /*********************************************************************
2589  *
2590  *  Allocate memory for the transmit and receive rings, and then
2591  *  the descriptors associated with each, called only once at attach.
2592  *
2593  **********************************************************************/
2594 static int
2595 ixgbe_allocate_queues(struct adapter *adapter)
2596 {
2597         device_t        dev = adapter->dev;
2598         struct ix_queue *que;
2599         struct tx_ring  *txr;
2600         struct rx_ring  *rxr;
2601         int rsize, tsize, error = IXGBE_SUCCESS;
2602         int txconf = 0, rxconf = 0;
2603
2604         /* First allocate the top level queue structs */
2605         if (!(adapter->queues =
2606             (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2607             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2608                 device_printf(dev, "Unable to allocate queue memory\n");
2609                 error = ENOMEM;
2610                 goto fail;
2611         }
2612
2613         /* First allocate the TX ring struct memory */
2614         if (!(adapter->tx_rings =
2615             (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2616             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2617                 device_printf(dev, "Unable to allocate TX ring memory\n");
2618                 error = ENOMEM;
2619                 goto tx_fail;
2620         }
2621
2622         /* Next allocate the RX */
2623         if (!(adapter->rx_rings =
2624             (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2625             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2626                 device_printf(dev, "Unable to allocate RX ring memory\n");
2627                 error = ENOMEM;
2628                 goto rx_fail;
2629         }
2630
2631         /* For the ring itself */
2632         tsize = roundup2(adapter->num_tx_desc *
2633             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2634
2635         /*
2636          * Now set up the TX queues, txconf is needed to handle the
2637          * possibility that things fail midcourse and we need to
2638          * undo memory gracefully
2639          */
2640         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2641                 /* Set up some basics */
2642                 txr = &adapter->tx_rings[i];
2643                 txr->adapter = adapter;
2644                 txr->me = i;
2645
2646                 /* Initialize the TX side lock */
2647                 IXGBE_TX_LOCK_INIT(txr);
2648
2649                 if (ixgbe_dma_malloc(adapter, tsize,
2650                         &txr->txdma, BUS_DMA_NOWAIT)) {
2651                         device_printf(dev,
2652                             "Unable to allocate TX Descriptor memory\n");
2653                         error = ENOMEM;
2654                         goto err_tx_desc;
2655                 }
2656                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2657                 bzero((void *)txr->tx_base, tsize);
2658
2659                 /* Now allocate transmit buffers for the ring */
2660                 if (ixgbe_allocate_transmit_buffers(txr)) {
2661                         device_printf(dev,
2662                             "Critical Failure setting up transmit buffers\n");
2663                         error = ENOMEM;
2664                         goto err_tx_desc;
2665                 }
2666 #if 0 /* __FreeBSD_version >= 800000 */
2667                 /* Allocate a buf ring */
2668                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2669                     M_WAITOK, &txr->tx_mtx);
2670                 if (txr->br == NULL) {
2671                         device_printf(dev,
2672                             "Critical Failure setting up buf ring\n");
2673                         error = ENOMEM;
2674                         goto err_tx_desc;
2675                 }
2676 #endif
2677         }
2678
2679         /*
2680          * Next the RX queues...
2681          */
2682         rsize = roundup2(adapter->num_rx_desc *
2683             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2684         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2685                 rxr = &adapter->rx_rings[i];
2686                 /* Set up some basics */
2687                 rxr->adapter = adapter;
2688                 rxr->me = i;
2689
2690                 /* Initialize the RX side lock */
2691                 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2692                     device_get_nameunit(dev), rxr->me);
2693                 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2694
2695                 if (ixgbe_dma_malloc(adapter, rsize,
2696                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2697                         device_printf(dev,
2698                             "Unable to allocate RxDescriptor memory\n");
2699                         error = ENOMEM;
2700                         goto err_rx_desc;
2701                 }
2702                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2703                 bzero((void *)rxr->rx_base, rsize);
2704
2705                 /* Allocate receive buffers for the ring*/
2706                 if (ixgbe_allocate_receive_buffers(rxr)) {
2707                         device_printf(dev,
2708                             "Critical Failure setting up receive buffers\n");
2709                         error = ENOMEM;
2710                         goto err_rx_desc;
2711                 }
2712         }
2713
2714         /*
2715         ** Finally set up the queue holding structs
2716         */
2717         for (int i = 0; i < adapter->num_queues; i++) {
2718                 que = &adapter->queues[i];
2719                 que->adapter = adapter;
2720                 que->txr = &adapter->tx_rings[i];
2721                 que->rxr = &adapter->rx_rings[i];
2722         }
2723
2724         return (0);
2725
2726 err_rx_desc:
2727         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2728                 ixgbe_dma_free(adapter, &rxr->rxdma);
2729 err_tx_desc:
2730         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2731                 ixgbe_dma_free(adapter, &txr->txdma);
2732         kfree(adapter->rx_rings, M_DEVBUF);
2733 rx_fail:
2734         kfree(adapter->tx_rings, M_DEVBUF);
2735 tx_fail:
2736         kfree(adapter->queues, M_DEVBUF);
2737 fail:
2738         return (error);
2739 }
2740
2741 /*********************************************************************
2742  *
2743  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2744  *  the information needed to transmit a packet on the wire. This is
2745  *  called only once at attach, setup is done every reset.
2746  *
2747  **********************************************************************/
2748 static int
2749 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2750 {
2751         struct adapter *adapter = txr->adapter;
2752         device_t dev = adapter->dev;
2753         struct ixgbe_tx_buf *txbuf;
2754         int error, i;
2755
2756         /*
2757          * Setup DMA descriptor areas.
2758          */
2759         if ((error = bus_dma_tag_create(
2760                                NULL,    /* parent */
2761                                1, 0,            /* alignment, bounds */
2762                                BUS_SPACE_MAXADDR,       /* lowaddr */
2763                                BUS_SPACE_MAXADDR,       /* highaddr */
2764                                NULL, NULL,              /* filter, filterarg */
2765                                IXGBE_TSO_SIZE,          /* maxsize */
2766                                adapter->num_segs,       /* nsegments */
2767                                PAGE_SIZE,               /* maxsegsize */
2768                                0,                       /* flags */
2769                                &txr->txtag))) {
2770                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2771                 goto fail;
2772         }
2773
2774         if (!(txr->tx_buffers =
2775             (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2776             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2777                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2778                 error = ENOMEM;
2779                 goto fail;
2780         }
2781
2782         /* Create the descriptor buffer dma maps */
2783         txbuf = txr->tx_buffers;
2784         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2785                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2786                 if (error != 0) {
2787                         device_printf(dev, "Unable to create TX DMA map\n");
2788                         goto fail;
2789                 }
2790         }
2791
2792         return 0;
2793 fail:
2794         /* We free all, it handles case where we are in the middle */
2795         ixgbe_free_transmit_structures(adapter);
2796         return (error);
2797 }
2798
2799 /*********************************************************************
2800  *
2801  *  Initialize a transmit ring.
2802  *
2803  **********************************************************************/
2804 static void
2805 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2806 {
2807         struct adapter *adapter = txr->adapter;
2808         struct ixgbe_tx_buf *txbuf;
2809         int i;
2810 #ifdef DEV_NETMAP
2811         struct netmap_adapter *na = NA(adapter->ifp);
2812         struct netmap_slot *slot;
2813 #endif /* DEV_NETMAP */
2814
2815         /* Clear the old ring contents */
2816         IXGBE_TX_LOCK(txr);
2817 #ifdef DEV_NETMAP
2818         /*
2819          * (under lock): if in netmap mode, do some consistency
2820          * checks and set slot to entry 0 of the netmap ring.
2821          */
2822         slot = netmap_reset(na, NR_TX, txr->me, 0);
2823 #endif /* DEV_NETMAP */
2824         bzero((void *)txr->tx_base,
2825               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
2826         /* Reset indices */
2827         txr->next_avail_desc = 0;
2828         txr->next_to_clean = 0;
2829
2830         /* Free any existing tx buffers. */
2831         txbuf = txr->tx_buffers;
2832         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2833                 if (txbuf->m_head != NULL) {
2834                         bus_dmamap_sync(txr->txtag, txbuf->map,
2835                             BUS_DMASYNC_POSTWRITE);
2836                         bus_dmamap_unload(txr->txtag, txbuf->map);
2837                         m_freem(txbuf->m_head);
2838                         txbuf->m_head = NULL;
2839                 }
2840 #ifdef DEV_NETMAP
2841                 /*
2842                  * In netmap mode, set the map for the packet buffer.
2843                  * NOTE: Some drivers (not this one) also need to set
2844                  * the physical buffer address in the NIC ring.
2845                  * Slots in the netmap ring (indexed by "si") are
2846                  * kring->nkr_hwofs positions "ahead" wrt the
2847                  * corresponding slot in the NIC ring. In some drivers
2848                  * (not here) nkr_hwofs can be negative. Function
2849                  * netmap_idx_n2k() handles wraparounds properly.
2850                  */
2851                 if (slot) {
2852                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
2853                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
2854                 }
2855 #endif /* DEV_NETMAP */
2856                 /* Clear the EOP index */
2857                 txbuf->eop_index = -1;
2858         }
2859
2860 #ifdef IXGBE_FDIR
2861         /* Set the rate at which we sample packets */
2862         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
2863                 txr->atr_sample = atr_sample_rate;
2864 #endif
2865
2866         /* Set number of descriptors available */
2867         txr->tx_avail = adapter->num_tx_desc;
2868
2869         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2870             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2871         IXGBE_TX_UNLOCK(txr);
2872 }
2873
2874 /*********************************************************************
2875  *
2876  *  Initialize all transmit rings.
2877  *
2878  **********************************************************************/
2879 static int
2880 ixgbe_setup_transmit_structures(struct adapter *adapter)
2881 {
2882         struct tx_ring *txr = adapter->tx_rings;
2883
2884         for (int i = 0; i < adapter->num_queues; i++, txr++)
2885                 ixgbe_setup_transmit_ring(txr);
2886
2887         return (0);
2888 }
2889
2890 /*********************************************************************
2891  *
2892  *  Enable transmit unit.
2893  *
2894  **********************************************************************/
2895 static void
2896 ixgbe_initialize_transmit_units(struct adapter *adapter)
2897 {
2898         struct tx_ring  *txr = adapter->tx_rings;
2899         struct ixgbe_hw *hw = &adapter->hw;
2900
2901         /* Setup the Base and Length of the Tx Descriptor Ring */
2902
2903         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2904                 u64     tdba = txr->txdma.dma_paddr;
2905                 u32     txctrl;
2906
2907                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
2908                        (tdba & 0x00000000ffffffffULL));
2909                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
2910                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
2911                     adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
2912
2913                 /* Setup the HW Tx Head and Tail descriptor pointers */
2914                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
2915                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
2916
2917                 /* Setup Transmit Descriptor Cmd Settings */
2918                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
2919                 txr->queue_status = IXGBE_QUEUE_IDLE;
2920
2921                 /* Disable Head Writeback */
2922                 switch (hw->mac.type) {
2923                 case ixgbe_mac_82598EB:
2924                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
2925                         break;
2926                 case ixgbe_mac_82599EB:
2927                 case ixgbe_mac_X540:
2928                 default:
2929                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
2930                         break;
2931                 }
2932                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2933                 switch (hw->mac.type) {
2934                 case ixgbe_mac_82598EB:
2935                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
2936                         break;
2937                 case ixgbe_mac_82599EB:
2938                 case ixgbe_mac_X540:
2939                 default:
2940                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
2941                         break;
2942                 }
2943
2944         }
2945
2946         if (hw->mac.type != ixgbe_mac_82598EB) {
2947                 u32 dmatxctl, rttdcs;
2948                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2949                 dmatxctl |= IXGBE_DMATXCTL_TE;
2950                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2951                 /* Disable arbiter to set MTQC */
2952                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2953                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
2954                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2955                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2956                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2957                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2958         }
2959
2960         return;
2961 }
2962
2963 /*********************************************************************
2964  *
2965  *  Free all transmit rings.
2966  *
2967  **********************************************************************/
2968 static void
2969 ixgbe_free_transmit_structures(struct adapter *adapter)
2970 {
2971         struct tx_ring *txr = adapter->tx_rings;
2972
2973         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2974                 IXGBE_TX_LOCK(txr);
2975                 ixgbe_free_transmit_buffers(txr);
2976                 ixgbe_dma_free(adapter, &txr->txdma);
2977                 IXGBE_TX_UNLOCK(txr);
2978                 IXGBE_TX_LOCK_DESTROY(txr);
2979         }
2980         kfree(adapter->tx_rings, M_DEVBUF);
2981 }
2982
2983 /*********************************************************************
2984  *
2985  *  Free transmit ring related data structures.
2986  *
2987  **********************************************************************/
2988 static void
2989 ixgbe_free_transmit_buffers(struct tx_ring *txr)
2990 {
2991         struct adapter *adapter = txr->adapter;
2992         struct ixgbe_tx_buf *tx_buffer;
2993         int             i;
2994
2995         INIT_DEBUGOUT("free_transmit_ring: begin");
2996
2997         if (txr->tx_buffers == NULL)
2998                 return;
2999
3000         tx_buffer = txr->tx_buffers;
3001         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3002                 if (tx_buffer->m_head != NULL) {
3003                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3004                             BUS_DMASYNC_POSTWRITE);
3005                         bus_dmamap_unload(txr->txtag,
3006                             tx_buffer->map);
3007                         m_freem(tx_buffer->m_head);
3008                         tx_buffer->m_head = NULL;
3009                         if (tx_buffer->map != NULL) {
3010                                 bus_dmamap_destroy(txr->txtag,
3011                                     tx_buffer->map);
3012                                 tx_buffer->map = NULL;
3013                         }
3014                 } else if (tx_buffer->map != NULL) {
3015                         bus_dmamap_unload(txr->txtag,
3016                             tx_buffer->map);
3017                         bus_dmamap_destroy(txr->txtag,
3018                             tx_buffer->map);
3019                         tx_buffer->map = NULL;
3020                 }
3021         }
3022 #if 0 /* __FreeBSD_version >= 800000 */
3023         if (txr->br != NULL)
3024                 buf_ring_free(txr->br, M_DEVBUF);
3025 #endif
3026         if (txr->tx_buffers != NULL) {
3027                 kfree(txr->tx_buffers, M_DEVBUF);
3028                 txr->tx_buffers = NULL;
3029         }
3030         if (txr->txtag != NULL) {
3031                 bus_dma_tag_destroy(txr->txtag);
3032                 txr->txtag = NULL;
3033         }
3034         return;
3035 }
3036
3037 /*********************************************************************
3038  *
3039  *  Advanced Context Descriptor setup for VLAN or CSUM
3040  *
3041  **********************************************************************/
3042
3043 static bool
3044 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3045 {
3046         struct adapter *adapter = txr->adapter;
3047         struct ixgbe_adv_tx_context_desc *TXD;
3048         struct ixgbe_tx_buf        *tx_buffer;
3049         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3050         struct ether_vlan_header *eh;
3051         struct ip *ip;
3052         struct ip6_hdr *ip6;
3053         int  ehdrlen, ip_hlen = 0;
3054         u16     etype;
3055         u8      ipproto = 0;
3056         bool    offload = TRUE;
3057         int ctxd = txr->next_avail_desc;
3058         u16 vtag = 0;
3059
3060
3061         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3062                 offload = FALSE;
3063
3064         tx_buffer = &txr->tx_buffers[ctxd];
3065         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3066
3067         /*
3068         ** In advanced descriptors the vlan tag must
3069         ** be placed into the descriptor itself.
3070         */
3071         if (mp->m_flags & M_VLANTAG) {
3072                 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3073                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3074         } else if (offload == FALSE)
3075                 return FALSE;
3076
3077         /*
3078          * Determine where frame payload starts.
3079          * Jump over vlan headers if already present,
3080          * helpful for QinQ too.
3081          */
3082         eh = mtod(mp, struct ether_vlan_header *);
3083         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3084                 etype = ntohs(eh->evl_proto);
3085                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3086         } else {
3087                 etype = ntohs(eh->evl_encap_proto);
3088                 ehdrlen = ETHER_HDR_LEN;
3089         }
3090
3091         /* Set the ether header length */
3092         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3093
3094         switch (etype) {
3095                 case ETHERTYPE_IP:
3096                         ip = (struct ip *)(mp->m_data + ehdrlen);
3097                         ip_hlen = ip->ip_hl << 2;
3098                         ipproto = ip->ip_p;
3099                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3100                         break;
3101                 case ETHERTYPE_IPV6:
3102                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3103                         ip_hlen = sizeof(struct ip6_hdr);
3104                         /* XXX-BZ this will go badly in case of ext hdrs. */
3105                         ipproto = ip6->ip6_nxt;
3106                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3107                         break;
3108                 default:
3109                         offload = FALSE;
3110                         break;
3111         }
3112
3113         vlan_macip_lens |= ip_hlen;
3114         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3115
3116         switch (ipproto) {
3117                 case IPPROTO_TCP:
3118                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3119                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3120                         break;
3121
3122                 case IPPROTO_UDP:
3123                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3124                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3125                         break;
3126
3127 #if 0
3128                 case IPPROTO_SCTP:
3129                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3130                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3131                         break;
3132 #endif
3133                 default:
3134                         offload = FALSE;
3135                         break;
3136         }
3137
3138         /* Now copy bits into descriptor */
3139         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3140         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3141         TXD->seqnum_seed = htole32(0);
3142         TXD->mss_l4len_idx = htole32(0);
3143
3144         tx_buffer->m_head = NULL;
3145         tx_buffer->eop_index = -1;
3146
3147         /* We've consumed the first desc, adjust counters */
3148         if (++ctxd == adapter->num_tx_desc)
3149                 ctxd = 0;
3150         txr->next_avail_desc = ctxd;
3151         --txr->tx_avail;
3152
3153         return (offload);
3154 }
3155
3156 /**********************************************************************
3157  *
3158  *  Setup work for hardware segmentation offload (TSO) on
3159  *  adapters using advanced tx descriptors
3160  *
3161  **********************************************************************/
3162 static bool
3163 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3164     u32 *olinfo_status)
3165 {
3166         struct adapter *adapter = txr->adapter;
3167         struct ixgbe_adv_tx_context_desc *TXD;
3168         struct ixgbe_tx_buf        *tx_buffer;
3169         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3170         u16 vtag = 0, eh_type;
3171         u32 mss_l4len_idx = 0, len;
3172         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3173         struct ether_vlan_header *eh;
3174 #if 0 /* IPv6 TSO */
3175 #ifdef INET6
3176         struct ip6_hdr *ip6;
3177 #endif
3178 #endif
3179 #ifdef INET
3180         struct ip *ip;
3181 #endif
3182         struct tcphdr *th;
3183
3184
3185         /*
3186          * Determine where frame payload starts.
3187          * Jump over vlan headers if already present
3188          */
3189         eh = mtod(mp, struct ether_vlan_header *);
3190         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3191                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3192                 eh_type = eh->evl_proto;
3193         } else {
3194                 ehdrlen = ETHER_HDR_LEN;
3195                 eh_type = eh->evl_encap_proto;
3196         }
3197
3198         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3199         len = ehdrlen + sizeof(struct tcphdr);
3200         switch (ntohs(eh_type)) {
3201 #if 0 /* IPv6 TSO */
3202 #ifdef INET6
3203         case ETHERTYPE_IPV6:
3204                 if (mp->m_len < len + sizeof(struct ip6_hdr))
3205                         return FALSE;
3206                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3207                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3208                 if (ip6->ip6_nxt != IPPROTO_TCP)
3209                         return FALSE;
3210                 ip_hlen = sizeof(struct ip6_hdr);
3211                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3212                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3213                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3214                 break;
3215 #endif
3216 #endif
3217 #ifdef INET
3218         case ETHERTYPE_IP:
3219                 if (mp->m_len < len + sizeof(struct ip))
3220                         return FALSE;
3221                 ip = (struct ip *)(mp->m_data + ehdrlen);
3222                 if (ip->ip_p != IPPROTO_TCP)
3223                         return FALSE;
3224                 ip->ip_sum = 0;
3225                 ip_hlen = ip->ip_hl << 2;
3226                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3227                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3228                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3229                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3230                 /* Tell transmit desc to also do IPv4 checksum. */
3231                 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3232                 break;
3233 #endif
3234         default:
3235                 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3236                     __func__, ntohs(eh_type));
3237                 break;
3238         }
3239
3240         ctxd = txr->next_avail_desc;
3241         tx_buffer = &txr->tx_buffers[ctxd];
3242         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3243
3244         tcp_hlen = th->th_off << 2;
3245
3246         /* This is used in the transmit desc in encap */
3247         *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3248
3249         /* VLAN MACLEN IPLEN */
3250         if (mp->m_flags & M_VLANTAG) {
3251                 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3252                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3253         }
3254
3255         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3256         vlan_macip_lens |= ip_hlen;
3257         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3258
3259         /* ADV DTYPE TUCMD */
3260         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3261         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3262         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3263
3264         /* MSS L4LEN IDX */
3265         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3266         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3267         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3268
3269         TXD->seqnum_seed = htole32(0);
3270         tx_buffer->m_head = NULL;
3271         tx_buffer->eop_index = -1;
3272
3273         if (++ctxd == adapter->num_tx_desc)
3274                 ctxd = 0;
3275
3276         txr->tx_avail--;
3277         txr->next_avail_desc = ctxd;
3278         return TRUE;
3279 }
3280
3281 #ifdef IXGBE_FDIR
3282 /*
3283 ** This routine parses packet headers so that Flow
3284 ** Director can make a hashed filter table entry
3285 ** allowing traffic flows to be identified and kept
3286 ** on the same cpu.  This would be a performance
3287 ** hit, but we only do it at IXGBE_FDIR_RATE of
3288 ** packets.
3289 */
3290 static void
3291 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3292 {
3293         struct adapter                  *adapter = txr->adapter;
3294         struct ix_queue                 *que;
3295         struct ip                       *ip;
3296         struct tcphdr                   *th;
3297         struct udphdr                   *uh;
3298         struct ether_vlan_header        *eh;
3299         union ixgbe_atr_hash_dword      input = {.dword = 0};
3300         union ixgbe_atr_hash_dword      common = {.dword = 0};
3301         int                             ehdrlen, ip_hlen;
3302         u16                             etype;
3303
3304         eh = mtod(mp, struct ether_vlan_header *);
3305         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3306                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3307                 etype = eh->evl_proto;
3308         } else {
3309                 ehdrlen = ETHER_HDR_LEN;
3310                 etype = eh->evl_encap_proto;
3311         }
3312
3313         /* Only handling IPv4 */
3314         if (etype != htons(ETHERTYPE_IP))
3315                 return;
3316
3317         ip = (struct ip *)(mp->m_data + ehdrlen);
3318         ip_hlen = ip->ip_hl << 2;
3319
3320         /* check if we're UDP or TCP */
3321         switch (ip->ip_p) {
3322         case IPPROTO_TCP:
3323                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3324                 /* src and dst are inverted */
3325                 common.port.dst ^= th->th_sport;
3326                 common.port.src ^= th->th_dport;
3327                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3328                 break;
3329         case IPPROTO_UDP:
3330                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3331                 /* src and dst are inverted */
3332                 common.port.dst ^= uh->uh_sport;
3333                 common.port.src ^= uh->uh_dport;
3334                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3335                 break;
3336         default:
3337                 return;
3338         }
3339
3340         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3341         if (mp->m_pkthdr.ether_vtag)
3342                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3343         else
3344                 common.flex_bytes ^= etype;
3345         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3346
3347         que = &adapter->queues[txr->me];
3348         /*
3349         ** This assumes the Rx queue and Tx
3350         ** queue are bound to the same CPU
3351         */
3352         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3353             input, common, que->msix);
3354 }
3355 #endif /* IXGBE_FDIR */
3356
3357 /**********************************************************************
3358  *
3359  *  Examine each tx_buffer in the used queue. If the hardware is done
3360  *  processing the packet then free associated resources. The
3361  *  tx_buffer is put back on the free queue.
3362  *
3363  **********************************************************************/
3364 static void
3365 ixgbe_txeof(struct tx_ring *txr)
3366 {
3367         struct adapter  *adapter = txr->adapter;
3368         struct ifnet    *ifp = adapter->ifp;
3369         u32     first, last, done, processed;
3370         struct ixgbe_tx_buf *tx_buffer;
3371         struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3372
3373         IXGBE_TX_LOCK_ASSERT(txr);
3374
3375 #ifdef DEV_NETMAP
3376         if (ifp->if_capenable & IFCAP_NETMAP) {
3377                 struct netmap_adapter *na = NA(ifp);
3378                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3379
3380                 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3381
3382                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3383                     BUS_DMASYNC_POSTREAD);
3384                 /*
3385                  * In netmap mode, all the work is done in the context
3386                  * of the client thread. Interrupt handlers only wake up
3387                  * clients, which may be sleeping on individual rings
3388                  * or on a global resource for all rings.
3389                  * To implement tx interrupt mitigation, we wake up the client
3390                  * thread roughly every half ring, even if the NIC interrupts
3391                  * more frequently. This is implemented as follows:
3392                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3393                  *   the slot that should wake up the thread (nkr_num_slots
3394                  *   means the user thread should not be woken up);
3395                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3396                  *   or the slot has the DD bit set.
3397                  *
3398                  * When the driver has separate locks, we need to
3399                  * release and re-acquire txlock to avoid deadlocks.
3400                  * XXX see if we can find a better way.
3401                  */
3402                 if (!netmap_mitigate ||
3403                     (kring->nr_kflags < kring->nkr_num_slots &&
3404                      tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3405                         kring->nr_kflags = kring->nkr_num_slots;
3406                         selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3407                         IXGBE_TX_UNLOCK(txr);
3408                         IXGBE_CORE_LOCK(adapter);
3409                         selwakeuppri(&na->tx_si, PI_NET);
3410                         IXGBE_CORE_UNLOCK(adapter);
3411                         IXGBE_TX_LOCK(txr);
3412                 }
3413                 return FALSE;
3414         }
3415 #endif /* DEV_NETMAP */
3416
3417         if (txr->tx_avail == adapter->num_tx_desc) {
3418                 txr->queue_status = IXGBE_QUEUE_IDLE;
3419                 return;
3420         }
3421
3422         processed = 0;
3423         first = txr->next_to_clean;
3424         tx_buffer = &txr->tx_buffers[first];
3425         /* For cleanup we just use legacy struct */
3426         tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3427         last = tx_buffer->eop_index;
3428         if (last == -1)
3429                 return;
3430         eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3431
3432         /*
3433         ** Get the index of the first descriptor
3434         ** BEYOND the EOP and call that 'done'.
3435         ** I do this so the comparison in the
3436         ** inner while loop below can be simple
3437         */
3438         if (++last == adapter->num_tx_desc) last = 0;
3439         done = last;
3440
3441         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3442             BUS_DMASYNC_POSTREAD);
3443         /*
3444         ** Only the EOP descriptor of a packet now has the DD
3445         ** bit set, this is what we look for...
3446         */
3447         while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3448                 /* We clean the range of the packet */
3449                 while (first != done) {
3450                         tx_desc->upper.data = 0;
3451                         tx_desc->lower.data = 0;
3452                         tx_desc->buffer_addr = 0;
3453                         ++txr->tx_avail;
3454                         ++processed;
3455
3456                         if (tx_buffer->m_head) {
3457                                 txr->bytes +=
3458                                     tx_buffer->m_head->m_pkthdr.len;
3459                                 bus_dmamap_sync(txr->txtag,
3460                                     tx_buffer->map,
3461                                     BUS_DMASYNC_POSTWRITE);
3462                                 bus_dmamap_unload(txr->txtag,
3463                                     tx_buffer->map);
3464                                 m_freem(tx_buffer->m_head);
3465                                 tx_buffer->m_head = NULL;
3466                                 tx_buffer->map = NULL;
3467                         }
3468                         tx_buffer->eop_index = -1;
3469                         txr->watchdog_time = ticks;
3470
3471                         if (++first == adapter->num_tx_desc)
3472                                 first = 0;
3473
3474                         tx_buffer = &txr->tx_buffers[first];
3475                         tx_desc =
3476                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3477                 }
3478                 ++txr->packets;
3479                 ++ifp->if_opackets;
3480                 /* See if there is more work now */
3481                 last = tx_buffer->eop_index;
3482                 if (last != -1) {
3483                         eop_desc =
3484                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3485                         /* Get next done point */
3486                         if (++last == adapter->num_tx_desc) last = 0;
3487                         done = last;
3488                 } else
3489                         break;
3490         }
3491         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3492             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3493
3494         txr->next_to_clean = first;
3495
3496         /*
3497         ** Watchdog calculation, we know there's
3498         ** work outstanding or the first return
3499         ** would have been taken, so none processed
3500         ** for too long indicates a hang.
3501         */
3502         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3503                 txr->queue_status = IXGBE_QUEUE_HUNG;
3504
3505         /* With a minimum free clear the depleted state bit.  */
3506         if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3507                 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3508
3509         if (txr->tx_avail == adapter->num_tx_desc) {
3510                 txr->queue_status = IXGBE_QUEUE_IDLE;
3511         }
3512 }
3513
3514 /*********************************************************************
3515  *
3516  *  Refresh mbuf buffers for RX descriptor rings
3517  *   - now keeps its own state so discards due to resource
3518  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3519  *     it just returns, keeping its placeholder, thus it can simply
3520  *     be recalled to try again.
3521  *
3522  **********************************************************************/
3523 static void
3524 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3525 {
3526         struct adapter          *adapter = rxr->adapter;
3527         bus_dma_segment_t       hseg[1];
3528         bus_dma_segment_t       pseg[1];
3529         struct ixgbe_rx_buf     *rxbuf;
3530         struct mbuf             *mh, *mp;
3531         int                     i, j, nsegs, error;
3532         bool                    refreshed = FALSE;
3533
3534         i = j = rxr->next_to_refresh;
3535         /* Control the loop with one beyond */
3536         if (++j == adapter->num_rx_desc)
3537                 j = 0;
3538
3539         while (j != limit) {
3540                 rxbuf = &rxr->rx_buffers[i];
3541                 if (rxr->hdr_split == FALSE)
3542                         goto no_split;
3543
3544                 if (rxbuf->m_head == NULL) {
3545                         mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3546                         if (mh == NULL)
3547                                 goto update;
3548                 } else
3549                         mh = rxbuf->m_head;
3550
3551                 mh->m_pkthdr.len = mh->m_len = MHLEN;
3552                 mh->m_len = MHLEN;
3553                 mh->m_flags |= M_PKTHDR;
3554                 /* Get the memory mapping */
3555                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3556                     rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3557                 if (error != 0) {
3558                         kprintf("Refresh mbufs: hdr dmamap load"
3559                             " failure - %d\n", error);
3560                         m_free(mh);
3561                         rxbuf->m_head = NULL;
3562                         goto update;
3563                 }
3564                 rxbuf->m_head = mh;
3565                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3566                     BUS_DMASYNC_PREREAD);
3567                 rxr->rx_base[i].read.hdr_addr =
3568                     htole64(hseg[0].ds_addr);
3569
3570 no_split:
3571                 if (rxbuf->m_pack == NULL) {
3572                         mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3573                             M_PKTHDR, adapter->rx_mbuf_sz);
3574                         if (mp == NULL)
3575                                 goto update;
3576                 } else
3577                         mp = rxbuf->m_pack;
3578
3579                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3580                 /* Get the memory mapping */
3581                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3582                     rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3583                 if (error != 0) {
3584                         kprintf("Refresh mbufs: payload dmamap load"
3585                             " failure - %d\n", error);
3586                         m_free(mp);
3587                         rxbuf->m_pack = NULL;
3588                         goto update;
3589                 }
3590                 rxbuf->m_pack = mp;
3591                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3592                     BUS_DMASYNC_PREREAD);
3593                 rxr->rx_base[i].read.pkt_addr =
3594                     htole64(pseg[0].ds_addr);
3595
3596                 refreshed = TRUE;
3597                 /* Next is precalculated */
3598                 i = j;
3599                 rxr->next_to_refresh = i;
3600                 if (++j == adapter->num_rx_desc)
3601                         j = 0;
3602         }
3603 update:
3604         if (refreshed) /* Update hardware tail index */
3605                 IXGBE_WRITE_REG(&adapter->hw,
3606                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3607         return;
3608 }
3609
3610 /*********************************************************************
3611  *
3612  *  Allocate memory for rx_buffer structures. Since we use one
3613  *  rx_buffer per received packet, the maximum number of rx_buffer's
3614  *  that we'll need is equal to the number of receive descriptors
3615  *  that we've allocated.
3616  *
3617  **********************************************************************/
3618 static int
3619 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3620 {
3621         struct  adapter         *adapter = rxr->adapter;
3622         device_t                dev = adapter->dev;
3623         struct ixgbe_rx_buf     *rxbuf;
3624         int                     i, bsize, error;
3625
3626         bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3627         if (!(rxr->rx_buffers =
3628             (struct ixgbe_rx_buf *) kmalloc(bsize,
3629             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3630                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3631                 error = ENOMEM;
3632                 goto fail;
3633         }
3634
3635         if ((error = bus_dma_tag_create(NULL,   /* parent */
3636                                    1, 0,        /* alignment, bounds */
3637                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3638                                    BUS_SPACE_MAXADDR,   /* highaddr */
3639                                    NULL, NULL,          /* filter, filterarg */
3640                                    MSIZE,               /* maxsize */
3641                                    1,                   /* nsegments */
3642                                    MSIZE,               /* maxsegsize */
3643                                    0,                   /* flags */
3644                                    &rxr->htag))) {
3645                 device_printf(dev, "Unable to create RX DMA tag\n");
3646                 goto fail;
3647         }
3648
3649         if ((error = bus_dma_tag_create(NULL,   /* parent */
3650                                    1, 0,        /* alignment, bounds */
3651                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3652                                    BUS_SPACE_MAXADDR,   /* highaddr */
3653                                    NULL, NULL,          /* filter, filterarg */
3654                                    MJUM16BYTES,         /* maxsize */
3655                                    1,                   /* nsegments */
3656                                    MJUM16BYTES,         /* maxsegsize */
3657                                    0,                   /* flags */
3658                                    &rxr->ptag))) {
3659                 device_printf(dev, "Unable to create RX DMA tag\n");
3660                 goto fail;
3661         }
3662
3663         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3664                 rxbuf = &rxr->rx_buffers[i];
3665                 error = bus_dmamap_create(rxr->htag,
3666                     BUS_DMA_NOWAIT, &rxbuf->hmap);
3667                 if (error) {
3668                         device_printf(dev, "Unable to create RX head map\n");
3669                         goto fail;
3670                 }
3671                 error = bus_dmamap_create(rxr->ptag,
3672                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3673                 if (error) {
3674                         device_printf(dev, "Unable to create RX pkt map\n");
3675                         goto fail;
3676                 }
3677         }
3678
3679         return (0);
3680
3681 fail:
3682         /* Frees all, but can handle partial completion */
3683         ixgbe_free_receive_structures(adapter);
3684         return (error);
3685 }
3686
3687 /*
3688 ** Used to detect a descriptor that has
3689 ** been merged by Hardware RSC.
3690 */
3691 static inline u32
3692 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3693 {
3694         return (le32toh(rx->wb.lower.lo_dword.data) &
3695             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3696 }
3697
3698 /*********************************************************************
3699  *
3700  *  Initialize Hardware RSC (LRO) feature on 82599
3701  *  for an RX ring, this is toggled by the LRO capability
3702  *  even though it is transparent to the stack.
3703  *
3704  **********************************************************************/
3705 #if 0   /* NET_LRO */
3706 static void
3707 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3708 {
3709         struct  adapter         *adapter = rxr->adapter;
3710         struct  ixgbe_hw        *hw = &adapter->hw;
3711         u32                     rscctrl, rdrxctl;
3712
3713         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3714         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3715 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3716         if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3717 #endif /* DEV_NETMAP */
3718         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3719         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3720         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3721
3722         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3723         rscctrl |= IXGBE_RSCCTL_RSCEN;
3724         /*
3725         ** Limit the total number of descriptors that
3726         ** can be combined, so it does not exceed 64K
3727         */
3728         if (adapter->rx_mbuf_sz == MCLBYTES)
3729                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3730         else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3731                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3732         else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3733                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3734         else  /* Using 16K cluster */
3735                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3736
3737         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3738
3739         /* Enable TCP header recognition */
3740         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3741             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3742             IXGBE_PSRTYPE_TCPHDR));
3743
3744         /* Disable RSC for ACK packets */
3745         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3746             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3747
3748         rxr->hw_rsc = TRUE;
3749 }
3750 #endif
3751
3752 static void
3753 ixgbe_free_receive_ring(struct rx_ring *rxr)
3754 {
3755         struct  adapter         *adapter;
3756         struct ixgbe_rx_buf       *rxbuf;
3757         int i;
3758
3759         adapter = rxr->adapter;
3760         for (i = 0; i < adapter->num_rx_desc; i++) {
3761                 rxbuf = &rxr->rx_buffers[i];
3762                 if (rxbuf->m_head != NULL) {
3763                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3764                             BUS_DMASYNC_POSTREAD);
3765                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3766                         rxbuf->m_head->m_flags |= M_PKTHDR;
3767                         m_freem(rxbuf->m_head);
3768                 }
3769                 if (rxbuf->m_pack != NULL) {
3770                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3771                             BUS_DMASYNC_POSTREAD);
3772                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3773                         rxbuf->m_pack->m_flags |= M_PKTHDR;
3774                         m_freem(rxbuf->m_pack);
3775                 }
3776                 rxbuf->m_head = NULL;
3777                 rxbuf->m_pack = NULL;
3778         }
3779 }
3780
3781
3782 /*********************************************************************
3783  *
3784  *  Initialize a receive ring and its buffers.
3785  *
3786  **********************************************************************/
3787 static int
3788 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3789 {
3790         struct  adapter         *adapter;
3791         struct ifnet            *ifp;
3792         device_t                dev;
3793         struct ixgbe_rx_buf     *rxbuf;
3794         bus_dma_segment_t       pseg[1], hseg[1];
3795 #if 0   /* NET_LRO */
3796         struct lro_ctrl         *lro = &rxr->lro;
3797 #endif
3798         int                     rsize, nsegs, error = 0;
3799 #ifdef DEV_NETMAP
3800         struct netmap_adapter *na = NA(rxr->adapter->ifp);
3801         struct netmap_slot *slot;
3802 #endif /* DEV_NETMAP */
3803
3804         adapter = rxr->adapter;
3805         ifp = adapter->ifp;
3806         dev = adapter->dev;
3807
3808         /* Clear the ring contents */
3809         IXGBE_RX_LOCK(rxr);
3810 #ifdef DEV_NETMAP
3811         /* same as in ixgbe_setup_transmit_ring() */
3812         slot = netmap_reset(na, NR_RX, rxr->me, 0);
3813 #endif /* DEV_NETMAP */
3814         rsize = roundup2(adapter->num_rx_desc *
3815             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3816         bzero((void *)rxr->rx_base, rsize);
3817
3818         /* Free current RX buffer structs and their mbufs */
3819         ixgbe_free_receive_ring(rxr);
3820
3821         /* Configure header split? */
3822         if (ixgbe_header_split)
3823                 rxr->hdr_split = TRUE;
3824
3825         /* Now replenish the mbufs */
3826         for (int j = 0; j != adapter->num_rx_desc; ++j) {
3827                 struct mbuf     *mh, *mp;
3828
3829                 rxbuf = &rxr->rx_buffers[j];
3830 #ifdef DEV_NETMAP
3831                 /*
3832                  * In netmap mode, fill the map and set the buffer
3833                  * address in the NIC ring, considering the offset
3834                  * between the netmap and NIC rings (see comment in
3835                  * ixgbe_setup_transmit_ring() ). No need to allocate
3836                  * an mbuf, so end the block with a continue;
3837                  */
3838                 if (slot) {
3839                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
3840                         uint64_t paddr;
3841                         void *addr;
3842
3843                         addr = PNMB(slot + sj, &paddr);
3844                         netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
3845                         /* Update descriptor */
3846                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
3847                         continue;
3848                 }
3849 #endif /* DEV_NETMAP */
3850                 /*
3851                 ** Don't allocate mbufs if not
3852                 ** doing header split, its wasteful
3853                 */
3854                 if (rxr->hdr_split == FALSE)
3855                         goto skip_head;
3856
3857                 /* First the header */
3858                 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
3859                 if (rxbuf->m_head == NULL) {
3860                         error = ENOBUFS;
3861                         goto fail;
3862                 }
3863                 m_adj(rxbuf->m_head, ETHER_ALIGN);
3864                 mh = rxbuf->m_head;
3865                 mh->m_len = mh->m_pkthdr.len = MHLEN;
3866                 mh->m_flags |= M_PKTHDR;
3867                 /* Get the memory mapping */
3868                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3869                     rxbuf->hmap, rxbuf->m_head, hseg, 1,
3870                     &nsegs, BUS_DMA_NOWAIT);
3871
3872                 if (error != 0) /* Nothing elegant to do here */
3873                         goto fail;
3874                 bus_dmamap_sync(rxr->htag,
3875                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
3876                 /* Update descriptor */
3877                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
3878
3879 skip_head:
3880                 /* Now the payload cluster */
3881                 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
3882                     M_PKTHDR, adapter->rx_mbuf_sz);
3883                 if (rxbuf->m_pack == NULL) {
3884                         error = ENOBUFS;
3885                         goto fail;
3886                 }
3887                 mp = rxbuf->m_pack;
3888                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3889                 /* Get the memory mapping */
3890                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3891                     rxbuf->pmap, mp, hseg, 1,
3892                     &nsegs, BUS_DMA_NOWAIT);
3893                 if (error != 0)
3894                         goto fail;
3895                 bus_dmamap_sync(rxr->ptag,
3896                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
3897                 /* Update descriptor */
3898                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
3899         }
3900
3901
3902         /* Setup our descriptor indices */
3903         rxr->next_to_check = 0;
3904         rxr->next_to_refresh = 0;
3905         rxr->lro_enabled = FALSE;
3906         rxr->rx_split_packets = 0;
3907         rxr->rx_bytes = 0;
3908         rxr->discard = FALSE;
3909         rxr->vtag_strip = FALSE;
3910
3911         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3912             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3913
3914         /*
3915         ** Now set up the LRO interface:
3916         ** 82598 uses software LRO, the
3917         ** 82599 and X540 use a hardware assist.
3918         */
3919 #if 0 /* NET_LRO */
3920         if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
3921             (ifp->if_capenable & IFCAP_RXCSUM) &&
3922             (ifp->if_capenable & IFCAP_LRO))
3923                 ixgbe_setup_hw_rsc(rxr);
3924         else if (ifp->if_capenable & IFCAP_LRO) {
3925                 int err = tcp_lro_init(lro);
3926                 if (err) {
3927                         device_printf(dev, "LRO Initialization failed!\n");
3928                         goto fail;
3929                 }
3930                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
3931                 rxr->lro_enabled = TRUE;
3932                 lro->ifp = adapter->ifp;
3933         }
3934 #endif
3935
3936         IXGBE_RX_UNLOCK(rxr);
3937         return (0);
3938
3939 fail:
3940         ixgbe_free_receive_ring(rxr);
3941         IXGBE_RX_UNLOCK(rxr);
3942         return (error);
3943 }
3944
3945 /*********************************************************************
3946  *
3947  *  Initialize all receive rings.
3948  *
3949  **********************************************************************/
3950 static int
3951 ixgbe_setup_receive_structures(struct adapter *adapter)
3952 {
3953         struct rx_ring *rxr = adapter->rx_rings;
3954         int j;
3955
3956         for (j = 0; j < adapter->num_queues; j++, rxr++)
3957                 if (ixgbe_setup_receive_ring(rxr))
3958                         goto fail;
3959
3960         return (0);
3961 fail:
3962         /*
3963          * Free RX buffers allocated so far, we will only handle
3964          * the rings that completed, the failing case will have
3965          * cleaned up for itself. 'j' failed, so its the terminus.
3966          */
3967         for (int i = 0; i < j; ++i) {
3968                 rxr = &adapter->rx_rings[i];
3969                 ixgbe_free_receive_ring(rxr);
3970         }
3971
3972         return (ENOBUFS);
3973 }
3974
3975 /*********************************************************************
3976  *
3977  *  Setup receive registers and features.
3978  *
3979  **********************************************************************/
3980 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
3981
3982 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
3983
3984 static void
3985 ixgbe_initialize_receive_units(struct adapter *adapter)
3986 {
3987         struct  rx_ring *rxr = adapter->rx_rings;
3988         struct ixgbe_hw *hw = &adapter->hw;
3989         struct ifnet   *ifp = adapter->ifp;
3990         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
3991         u32             reta, mrqc = 0, hlreg, random[10];
3992
3993
3994         /*
3995          * Make sure receives are disabled while
3996          * setting up the descriptor ring
3997          */
3998         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
3999         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4000             rxctrl & ~IXGBE_RXCTRL_RXEN);
4001
4002         /* Enable broadcasts */
4003         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4004         fctrl |= IXGBE_FCTRL_BAM;
4005         fctrl |= IXGBE_FCTRL_DPF;
4006         fctrl |= IXGBE_FCTRL_PMCF;
4007         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4008
4009         /* Set for Jumbo Frames? */
4010         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4011         if (ifp->if_mtu > ETHERMTU)
4012                 hlreg |= IXGBE_HLREG0_JUMBOEN;
4013         else
4014                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4015 #ifdef DEV_NETMAP
4016         /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4017         if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4018                 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4019         else
4020                 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4021 #endif /* DEV_NETMAP */
4022         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4023
4024         bufsz = (adapter->rx_mbuf_sz +
4025             BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4026
4027         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4028                 u64 rdba = rxr->rxdma.dma_paddr;
4029
4030                 /* Setup the Base and Length of the Rx Descriptor Ring */
4031                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4032                                (rdba & 0x00000000ffffffffULL));
4033                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4034                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4035                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4036
4037                 /* Set up the SRRCTL register */
4038                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4039                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4040                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4041                 srrctl |= bufsz;
4042                 if (rxr->hdr_split) {
4043                         /* Use a standard mbuf for the header */
4044                         srrctl |= ((IXGBE_RX_HDR <<
4045                             IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
4046                             & IXGBE_SRRCTL_BSIZEHDR_MASK);
4047                         srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4048                 } else
4049                         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4050                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4051
4052                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4053                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4054                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4055         }
4056
4057         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4058                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4059                               IXGBE_PSRTYPE_UDPHDR |
4060                               IXGBE_PSRTYPE_IPV4HDR |
4061                               IXGBE_PSRTYPE_IPV6HDR;
4062                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4063         }
4064
4065         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4066
4067         /* Setup RSS */
4068         if (adapter->num_queues > 1) {
4069                 int i, j;
4070                 reta = 0;
4071
4072                 /* set up random bits */
4073                 karc4rand(&random, sizeof(random));
4074
4075                 /* Set up the redirection table */
4076                 for (i = 0, j = 0; i < 128; i++, j++) {
4077                         if (j == adapter->num_queues) j = 0;
4078                         reta = (reta << 8) | (j * 0x11);
4079                         if ((i & 3) == 3)
4080                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4081                 }
4082
4083                 /* Now fill our hash function seeds */
4084                 for (int i = 0; i < 10; i++)
4085                         IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4086
4087                 /* Perform hash on these packet types */
4088                 mrqc = IXGBE_MRQC_RSSEN
4089                      | IXGBE_MRQC_RSS_FIELD_IPV4
4090                      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4091                      | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4092                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4093                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4094                      | IXGBE_MRQC_RSS_FIELD_IPV6
4095                      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4096                      | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4097                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4098                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4099
4100                 /* RSS and RX IPP Checksum are mutually exclusive */
4101                 rxcsum |= IXGBE_RXCSUM_PCSD;
4102         }
4103
4104         if (ifp->if_capenable & IFCAP_RXCSUM)
4105                 rxcsum |= IXGBE_RXCSUM_PCSD;
4106
4107         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4108                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4109
4110         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4111
4112         return;
4113 }
4114
4115 /*********************************************************************
4116  *
4117  *  Free all receive rings.
4118  *
4119  **********************************************************************/
4120 static void
4121 ixgbe_free_receive_structures(struct adapter *adapter)
4122 {
4123         struct rx_ring *rxr = adapter->rx_rings;
4124
4125         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4126 #if 0   /* NET_LRO */
4127                 struct lro_ctrl         *lro = &rxr->lro;
4128 #endif
4129                 ixgbe_free_receive_buffers(rxr);
4130                 /* Free LRO memory */
4131 #if 0   /* NET_LRO */
4132                 tcp_lro_free(lro);
4133 #endif
4134                 /* Free the ring memory as well */
4135                 ixgbe_dma_free(adapter, &rxr->rxdma);
4136         }
4137
4138         kfree(adapter->rx_rings, M_DEVBUF);
4139 }
4140
4141
4142 /*********************************************************************
4143  *
4144  *  Free receive ring data structures
4145  *
4146  **********************************************************************/
4147 static void
4148 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4149 {
4150         struct adapter          *adapter = rxr->adapter;
4151         struct ixgbe_rx_buf     *rxbuf;
4152
4153         INIT_DEBUGOUT("free_receive_structures: begin");
4154
4155         /* Cleanup any existing buffers */
4156         if (rxr->rx_buffers != NULL) {
4157                 for (int i = 0; i < adapter->num_rx_desc; i++) {
4158                         rxbuf = &rxr->rx_buffers[i];
4159                         if (rxbuf->m_head != NULL) {
4160                                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4161                                     BUS_DMASYNC_POSTREAD);
4162                                 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4163                                 rxbuf->m_head->m_flags |= M_PKTHDR;
4164                                 m_freem(rxbuf->m_head);
4165                         }
4166                         if (rxbuf->m_pack != NULL) {
4167                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4168                                     BUS_DMASYNC_POSTREAD);
4169                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4170                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
4171                                 m_freem(rxbuf->m_pack);
4172                         }
4173                         rxbuf->m_head = NULL;
4174                         rxbuf->m_pack = NULL;
4175                         if (rxbuf->hmap != NULL) {
4176                                 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4177                                 rxbuf->hmap = NULL;
4178                         }
4179                         if (rxbuf->pmap != NULL) {
4180                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4181                                 rxbuf->pmap = NULL;
4182                         }
4183                 }
4184                 if (rxr->rx_buffers != NULL) {
4185                         kfree(rxr->rx_buffers, M_DEVBUF);
4186                         rxr->rx_buffers = NULL;
4187                 }
4188         }
4189
4190         if (rxr->htag != NULL) {
4191                 bus_dma_tag_destroy(rxr->htag);
4192                 rxr->htag = NULL;
4193         }
4194         if (rxr->ptag != NULL) {
4195                 bus_dma_tag_destroy(rxr->ptag);
4196                 rxr->ptag = NULL;
4197         }
4198
4199         return;
4200 }
4201
4202 static __inline void
4203 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4204 {
4205
4206         /*
4207          * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4208          * should be computed by hardware. Also it should not have VLAN tag in
4209          * ethernet header.  In case of IPv6 we do not yet support ext. hdrs.
4210          */
4211 #if 0   /* NET_LRO */
4212         if (rxr->lro_enabled &&
4213             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4214             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4215             ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4216             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4217             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4218             (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4219             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4220             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4221                 /*
4222                  * Send to the stack if:
4223                  **  - LRO not enabled, or
4224                  **  - no LRO resources, or
4225                  **  - lro enqueue fails
4226                  */
4227                 if (rxr->lro.lro_cnt != 0)
4228                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4229                                 return;
4230         }
4231 #endif
4232         IXGBE_RX_UNLOCK(rxr);
4233         (*ifp->if_input)(ifp, m);
4234         IXGBE_RX_LOCK(rxr);
4235 }
4236
4237 static __inline void
4238 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4239 {
4240         struct ixgbe_rx_buf     *rbuf;
4241
4242         rbuf = &rxr->rx_buffers[i];
4243
4244         if (rbuf->fmp != NULL) {/* Partial chain ? */
4245                 rbuf->fmp->m_flags |= M_PKTHDR;
4246                 m_freem(rbuf->fmp);
4247                 rbuf->fmp = NULL;
4248         }
4249
4250         /*
4251         ** With advanced descriptors the writeback
4252         ** clobbers the buffer addrs, so its easier
4253         ** to just free the existing mbufs and take
4254         ** the normal refresh path to get new buffers
4255         ** and mapping.
4256         */
4257         if (rbuf->m_head) {
4258                 m_free(rbuf->m_head);
4259                 rbuf->m_head = NULL;
4260         }
4261
4262         if (rbuf->m_pack) {
4263                 m_free(rbuf->m_pack);
4264                 rbuf->m_pack = NULL;
4265         }
4266
4267         return;
4268 }
4269
4270 static void
4271 ixgbe_add_sysctl(struct adapter *adapter)
4272 {
4273         sysctl_ctx_init(&adapter->sysctl_ctx);
4274         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
4275             SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
4276             device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
4277         if (adapter->sysctl_tree == NULL) {
4278                 device_printf(adapter->dev, "can't add sysctl node\n");
4279                 return;
4280         }
4281         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4282                         SYSCTL_CHILDREN(adapter->sysctl_tree),
4283                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
4284                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
4285
4286         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4287             SYSCTL_CHILDREN(adapter->sysctl_tree),
4288             OID_AUTO, "intr_rate", CTLTYPE_INT | CTLFLAG_RW,
4289             adapter, 0, ixgbe_sysctl_intr_rate, "I", "interrupt rate");
4290
4291         /*
4292         ** Allow a kind of speed control by forcing the autoneg
4293         ** advertised speed list to only a certain value, this
4294         ** supports 1G on 82599 devices, and 100Mb on x540.
4295         */
4296         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4297                         SYSCTL_CHILDREN(adapter->sysctl_tree),
4298                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
4299                         adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
4300
4301         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4302                         SYSCTL_CHILDREN(adapter->sysctl_tree),
4303                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
4304                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
4305
4306         /* Sysctl for limiting the amount of work done in the taskqueue */
4307         ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
4308             "max number of rx packets to process", &adapter->rx_process_limit,
4309             ixgbe_rx_process_limit);
4310 }
4311
4312 /*********************************************************************
4313  *
4314  *  This routine executes in interrupt context. It replenishes
4315  *  the mbufs in the descriptor and sends data which has been
4316  *  dma'ed into host memory to upper layer.
4317  *
4318  *  We loop at most count times if count is > 0, or until done if
4319  *  count < 0.
4320  *
4321  *  Return TRUE for more work, FALSE for all clean.
4322  *********************************************************************/
4323 static void
4324 ixgbe_rxeof(struct ix_queue *que, int count)
4325 {
4326         struct adapter          *adapter = que->adapter;
4327         struct rx_ring          *rxr = que->rxr;
4328         struct ifnet            *ifp = adapter->ifp;
4329 #if 0   /* NET_LRO */
4330         struct lro_ctrl         *lro = &rxr->lro;
4331         struct lro_entry        *queued;
4332 #endif
4333         int                     i, nextp, processed = 0;
4334         u32                     staterr = 0;
4335         union ixgbe_adv_rx_desc *cur;
4336         struct ixgbe_rx_buf     *rbuf, *nbuf;
4337
4338         IXGBE_RX_LOCK(rxr);
4339
4340 #ifdef DEV_NETMAP
4341         if (ifp->if_capenable & IFCAP_NETMAP) {
4342                 /*
4343                  * Same as the txeof routine: only wakeup clients on intr.
4344                  * NKR_PENDINTR in nr_kflags is used to implement interrupt
4345                  * mitigation (ixgbe_rxsync() will not look for new packets
4346                  * unless NKR_PENDINTR is set).
4347                  */
4348                 struct netmap_adapter *na = NA(ifp);
4349
4350                 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4351                 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4352                 IXGBE_RX_UNLOCK(rxr);
4353                 IXGBE_CORE_LOCK(adapter);
4354                 selwakeuppri(&na->rx_si, PI_NET);
4355                 IXGBE_CORE_UNLOCK(adapter);
4356                 return (FALSE);
4357         }
4358 #endif /* DEV_NETMAP */
4359         for (i = rxr->next_to_check; count != 0;) {
4360                 struct mbuf     *sendmp, *mh, *mp;
4361                 u32             rsc, ptype;
4362                 u16             hlen, plen, hdr;
4363                 u16             vtag = 0;
4364                 bool            eop;
4365
4366                 /* Sync the ring. */
4367                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4368                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4369
4370                 cur = &rxr->rx_base[i];
4371                 staterr = le32toh(cur->wb.upper.status_error);
4372
4373                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4374                         break;
4375                 if ((ifp->if_flags & IFF_RUNNING) == 0)
4376                         break;
4377
4378                 count--;
4379                 sendmp = NULL;
4380                 nbuf = NULL;
4381                 rsc = 0;
4382                 cur->wb.upper.status_error = 0;
4383                 rbuf = &rxr->rx_buffers[i];
4384                 mh = rbuf->m_head;
4385                 mp = rbuf->m_pack;
4386
4387                 plen = le16toh(cur->wb.upper.length);
4388                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4389                     IXGBE_RXDADV_PKTTYPE_MASK;
4390                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4391                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4392
4393                 /* Process vlan info */
4394                 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
4395                         vtag = le16toh(cur->wb.upper.vlan);
4396
4397                 /* Make sure bad packets are discarded */
4398                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4399                     (rxr->discard)) {
4400                         ifp->if_ierrors++;
4401                         rxr->rx_discarded++;
4402                         if (eop)
4403                                 rxr->discard = FALSE;
4404                         else
4405                                 rxr->discard = TRUE;
4406                         ixgbe_rx_discard(rxr, i);
4407                         goto next_desc;
4408                 }
4409
4410                 /*
4411                 ** On 82599 which supports a hardware
4412                 ** LRO (called HW RSC), packets need
4413                 ** not be fragmented across sequential
4414                 ** descriptors, rather the next descriptor
4415                 ** is indicated in bits of the descriptor.
4416                 ** This also means that we might proceses
4417                 ** more than one packet at a time, something
4418                 ** that has never been true before, it
4419                 ** required eliminating global chain pointers
4420                 ** in favor of what we are doing here.  -jfv
4421                 */
4422                 if (!eop) {
4423                         /*
4424                         ** Figure out the next descriptor
4425                         ** of this frame.
4426                         */
4427                         if (rxr->hw_rsc == TRUE) {
4428                                 rsc = ixgbe_rsc_count(cur);
4429                                 rxr->rsc_num += (rsc - 1);
4430                         }
4431                         if (rsc) { /* Get hardware index */
4432                                 nextp = ((staterr &
4433                                     IXGBE_RXDADV_NEXTP_MASK) >>
4434                                     IXGBE_RXDADV_NEXTP_SHIFT);
4435                         } else { /* Just sequential */
4436                                 nextp = i + 1;
4437                                 if (nextp == adapter->num_rx_desc)
4438                                         nextp = 0;
4439                         }
4440                         nbuf = &rxr->rx_buffers[nextp];
4441                         prefetch(nbuf);
4442                 }
4443                 /*
4444                 ** The header mbuf is ONLY used when header
4445                 ** split is enabled, otherwise we get normal
4446                 ** behavior, ie, both header and payload
4447                 ** are DMA'd into the payload buffer.
4448                 **
4449                 ** Rather than using the fmp/lmp global pointers
4450                 ** we now keep the head of a packet chain in the
4451                 ** buffer struct and pass this along from one
4452                 ** descriptor to the next, until we get EOP.
4453                 */
4454                 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4455                         /* This must be an initial descriptor */
4456                         hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4457                             IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4458                         if (hlen > IXGBE_RX_HDR)
4459                                 hlen = IXGBE_RX_HDR;
4460                         mh->m_len = hlen;
4461                         mh->m_flags |= M_PKTHDR;
4462                         mh->m_next = NULL;
4463                         mh->m_pkthdr.len = mh->m_len;
4464                         /* Null buf pointer so it is refreshed */
4465                         rbuf->m_head = NULL;
4466                         /*
4467                         ** Check the payload length, this
4468                         ** could be zero if its a small
4469                         ** packet.
4470                         */
4471                         if (plen > 0) {
4472                                 mp->m_len = plen;
4473                                 mp->m_next = NULL;
4474                                 mp->m_flags &= ~M_PKTHDR;
4475                                 mh->m_next = mp;
4476                                 mh->m_pkthdr.len += mp->m_len;
4477                                 /* Null buf pointer so it is refreshed */
4478                                 rbuf->m_pack = NULL;
4479                                 rxr->rx_split_packets++;
4480                         }
4481                         /*
4482                         ** Now create the forward
4483                         ** chain so when complete
4484                         ** we wont have to.
4485                         */
4486                         if (eop == 0) {
4487                                 /* stash the chain head */
4488                                 nbuf->fmp = mh;
4489                                 /* Make forward chain */
4490                                 if (plen)
4491                                         mp->m_next = nbuf->m_pack;
4492                                 else
4493                                         mh->m_next = nbuf->m_pack;
4494                         } else {
4495                                 /* Singlet, prepare to send */
4496                                 sendmp = mh;
4497                                 /* If hardware handled vtag */
4498                                 if (vtag) {
4499                                         sendmp->m_pkthdr.ether_vlantag = vtag;
4500                                         sendmp->m_flags |= M_VLANTAG;
4501                                 }
4502                         }
4503                 } else {
4504                         /*
4505                         ** Either no header split, or a
4506                         ** secondary piece of a fragmented
4507                         ** split packet.
4508                         */
4509                         mp->m_len = plen;
4510                         /*
4511                         ** See if there is a stored head
4512                         ** that determines what we are
4513                         */
4514                         sendmp = rbuf->fmp;
4515                         rbuf->m_pack = rbuf->fmp = NULL;
4516
4517                         if (sendmp != NULL) {  /* secondary frag */
4518                                 mp->m_flags &= ~M_PKTHDR;
4519                                 sendmp->m_pkthdr.len += mp->m_len;
4520                         } else {
4521                                 /* first desc of a non-ps chain */
4522                                 sendmp = mp;
4523                                 sendmp->m_flags |= M_PKTHDR;
4524                                 sendmp->m_pkthdr.len = mp->m_len;
4525                                 if (staterr & IXGBE_RXD_STAT_VP) {
4526                                         sendmp->m_pkthdr.ether_vlantag = vtag;
4527                                         sendmp->m_flags |= M_VLANTAG;
4528                                 }
4529                         }
4530                         /* Pass the head pointer on */
4531                         if (eop == 0) {
4532                                 nbuf->fmp = sendmp;
4533                                 sendmp = NULL;
4534                                 mp->m_next = nbuf->m_pack;
4535                         }
4536                 }
4537                 ++processed;
4538                 /* Sending this frame? */
4539                 if (eop) {
4540                         sendmp->m_pkthdr.rcvif = ifp;
4541                         ifp->if_ipackets++;
4542                         rxr->rx_packets++;
4543                         /* capture data for AIM */
4544                         rxr->bytes += sendmp->m_pkthdr.len;
4545                         rxr->rx_bytes += sendmp->m_pkthdr.len;
4546                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4547                                 ixgbe_rx_checksum(staterr, sendmp, ptype);
4548 #if 0 /* __FreeBSD_version >= 800000 */
4549                         sendmp->m_pkthdr.flowid = que->msix;
4550                         sendmp->m_flags |= M_FLOWID;
4551 #endif
4552                 }
4553 next_desc:
4554                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4555                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4556
4557                 /* Advance our pointers to the next descriptor. */
4558                 if (++i == adapter->num_rx_desc)
4559                         i = 0;
4560
4561                 /* Now send to the stack or do LRO */
4562                 if (sendmp != NULL) {
4563                         rxr->next_to_check = i;
4564                         ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4565                         i = rxr->next_to_check;
4566                 }
4567
4568                /* Every 8 descriptors we go to refresh mbufs */
4569                 if (processed == 8) {
4570                         ixgbe_refresh_mbufs(rxr, i);
4571                         processed = 0;
4572                 }
4573         }
4574
4575         /* Refresh any remaining buf structs */
4576         if (ixgbe_rx_unrefreshed(rxr))
4577                 ixgbe_refresh_mbufs(rxr, i);
4578
4579         rxr->next_to_check = i;
4580
4581         /*
4582          * Flush any outstanding LRO work
4583          */
4584 #if 0   /* NET_LRO */
4585         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4586                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4587                 tcp_lro_flush(lro, queued);
4588         }
4589 #endif
4590
4591         IXGBE_RX_UNLOCK(rxr);
4592
4593         /*
4594         ** We still have cleaning to do?
4595         ** Schedule another interrupt if so.
4596         */
4597         if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4598                 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4599         }
4600 }
4601
4602
4603 /*********************************************************************
4604  *
4605  *  Verify that the hardware indicated that the checksum is valid.
4606  *  Inform the stack about the status of checksum so that stack
4607  *  doesn't spend time verifying the checksum.
4608  *
4609  *********************************************************************/
4610 static void
4611 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4612 {
4613         u16     status = (u16) staterr;
4614         u8      errors = (u8) (staterr >> 24);
4615         bool    sctp = FALSE;
4616
4617         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4618             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4619                 sctp = TRUE;
4620
4621         if (status & IXGBE_RXD_STAT_IPCS) {
4622                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4623                         /* IP Checksum Good */
4624                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4625                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4626
4627                 } else
4628                         mp->m_pkthdr.csum_flags = 0;
4629         }
4630         if (status & IXGBE_RXD_STAT_L4CS) {
4631                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4632 #if 0
4633                 if (sctp)
4634                         type = CSUM_SCTP_VALID;
4635 #endif
4636                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4637                         mp->m_pkthdr.csum_flags |= type;
4638                         if (!sctp)
4639                                 mp->m_pkthdr.csum_data = htons(0xffff);
4640                 }
4641         }
4642         return;
4643 }
4644
4645
4646 /*
4647 ** This routine is run via an vlan config EVENT,
4648 ** it enables us to use the HW Filter table since
4649 ** we can get the vlan id. This just creates the
4650 ** entry in the soft version of the VFTA, init will
4651 ** repopulate the real table.
4652 */
4653 static void
4654 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4655 {
4656         struct adapter  *adapter = ifp->if_softc;
4657         u16             index, bit;
4658
4659         if (ifp->if_softc !=  arg)   /* Not our event */
4660                 return;
4661
4662         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4663                 return;
4664
4665         IXGBE_CORE_LOCK(adapter);
4666         index = (vtag >> 5) & 0x7F;
4667         bit = vtag & 0x1F;
4668         adapter->shadow_vfta[index] |= (1 << bit);
4669         ++adapter->num_vlans;
4670         ixgbe_init_locked(adapter);
4671         IXGBE_CORE_UNLOCK(adapter);
4672 }
4673
4674 /*
4675 ** This routine is run via an vlan
4676 ** unconfig EVENT, remove our entry
4677 ** in the soft vfta.
4678 */
4679 static void
4680 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4681 {
4682         struct adapter  *adapter = ifp->if_softc;
4683         u16             index, bit;
4684
4685         if (ifp->if_softc !=  arg)
4686                 return;
4687
4688         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4689                 return;
4690
4691         IXGBE_CORE_LOCK(adapter);
4692         index = (vtag >> 5) & 0x7F;
4693         bit = vtag & 0x1F;
4694         adapter->shadow_vfta[index] &= ~(1 << bit);
4695         --adapter->num_vlans;
4696         /* Re-init to load the changes */
4697         ixgbe_init_locked(adapter);
4698         IXGBE_CORE_UNLOCK(adapter);
4699 }
4700
4701 static void
4702 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4703 {
4704         struct ifnet    *ifp = adapter->ifp;
4705         struct ixgbe_hw *hw = &adapter->hw;
4706         struct rx_ring  *rxr;
4707         u32             ctrl;
4708
4709         /*
4710         ** We get here thru init_locked, meaning
4711         ** a soft reset, this has already cleared
4712         ** the VFTA and other state, so if there
4713         ** have been no vlan's registered do nothing.
4714         */
4715         if (adapter->num_vlans == 0)
4716                 return;
4717
4718         /*
4719         ** A soft reset zero's out the VFTA, so
4720         ** we need to repopulate it now.
4721         */
4722         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4723                 if (adapter->shadow_vfta[i] != 0)
4724                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4725                             adapter->shadow_vfta[i]);
4726
4727         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4728         /* Enable the Filter Table if enabled */
4729         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4730                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4731                 ctrl |= IXGBE_VLNCTRL_VFE;
4732         }
4733         if (hw->mac.type == ixgbe_mac_82598EB)
4734                 ctrl |= IXGBE_VLNCTRL_VME;
4735         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4736
4737         /* Setup the queues for vlans */
4738         for (int i = 0; i < adapter->num_queues; i++) {
4739                 rxr = &adapter->rx_rings[i];
4740                 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4741                 if (hw->mac.type != ixgbe_mac_82598EB) {
4742                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4743                         ctrl |= IXGBE_RXDCTL_VME;
4744                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4745                 }
4746                 rxr->vtag_strip = TRUE;
4747         }
4748 }
4749
4750 static void
4751 ixgbe_enable_intr(struct adapter *adapter)
4752 {
4753         struct ixgbe_hw *hw = &adapter->hw;
4754         struct ix_queue *que = adapter->queues;
4755         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4756
4757
4758         /* Enable Fan Failure detection */
4759         if (hw->device_id == IXGBE_DEV_ID_82598AT)
4760                     mask |= IXGBE_EIMS_GPI_SDP1;
4761         else {
4762                     mask |= IXGBE_EIMS_ECC;
4763                     mask |= IXGBE_EIMS_GPI_SDP0;
4764                     mask |= IXGBE_EIMS_GPI_SDP1;
4765                     mask |= IXGBE_EIMS_GPI_SDP2;
4766 #ifdef IXGBE_FDIR
4767                     mask |= IXGBE_EIMS_FLOW_DIR;
4768 #endif
4769         }
4770
4771         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4772
4773         /* With RSS we use auto clear */
4774         if (adapter->msix_mem) {
4775                 mask = IXGBE_EIMS_ENABLE_MASK;
4776                 /* Don't autoclear Link */
4777                 mask &= ~IXGBE_EIMS_OTHER;
4778                 mask &= ~IXGBE_EIMS_LSC;
4779                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4780         }
4781
4782         /*
4783         ** Now enable all queues, this is done separately to
4784         ** allow for handling the extended (beyond 32) MSIX
4785         ** vectors that can be used by 82599
4786         */
4787         for (int i = 0; i < adapter->num_queues; i++, que++)
4788                 ixgbe_enable_queue(adapter, que->msix);
4789
4790         IXGBE_WRITE_FLUSH(hw);
4791
4792         return;
4793 }
4794
4795 static void
4796 ixgbe_disable_intr(struct adapter *adapter)
4797 {
4798         if (adapter->msix_mem)
4799                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4800         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4801                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4802         } else {
4803                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4804                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4805                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4806         }
4807         IXGBE_WRITE_FLUSH(&adapter->hw);
4808         return;
4809 }
4810
4811 u16
4812 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4813 {
4814         u16 value;
4815
4816         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4817             reg, 2);
4818
4819         return (value);
4820 }
4821
4822 void
4823 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4824 {
4825         pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4826             reg, value, 2);
4827
4828         return;
4829 }
4830
4831 /*
4832 ** Setup the correct IVAR register for a particular MSIX interrupt
4833 **   (yes this is all very magic and confusing :)
4834 **  - entry is the register array entry
4835 **  - vector is the MSIX vector for this queue
4836 **  - type is RX/TX/MISC
4837 */
4838 static void
4839 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4840 {
4841         struct ixgbe_hw *hw = &adapter->hw;
4842         u32 ivar, index;
4843
4844         vector |= IXGBE_IVAR_ALLOC_VAL;
4845
4846         switch (hw->mac.type) {
4847
4848         case ixgbe_mac_82598EB:
4849                 if (type == -1)
4850                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4851                 else
4852                         entry += (type * 64);
4853                 index = (entry >> 2) & 0x1F;
4854                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4855                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4856                 ivar |= (vector << (8 * (entry & 0x3)));
4857                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
4858                 break;
4859
4860         case ixgbe_mac_82599EB:
4861         case ixgbe_mac_X540:
4862                 if (type == -1) { /* MISC IVAR */
4863                         index = (entry & 1) * 8;
4864                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4865                         ivar &= ~(0xFF << index);
4866                         ivar |= (vector << index);
4867                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4868                 } else {        /* RX/TX IVARS */
4869                         index = (16 * (entry & 1)) + (8 * type);
4870                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4871                         ivar &= ~(0xFF << index);
4872                         ivar |= (vector << index);
4873                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4874                 }
4875
4876         default:
4877                 break;
4878         }
4879 }
4880
4881 static void
4882 ixgbe_configure_ivars(struct adapter *adapter)
4883 {
4884         struct  ix_queue *que = adapter->queues;
4885
4886         for (int i = 0; i < adapter->num_queues; i++, que++) {
4887                 /* First the RX queue entry */
4888                 ixgbe_set_ivar(adapter, i, que->msix, 0);
4889                 /* ... and the TX */
4890                 ixgbe_set_ivar(adapter, i, que->msix, 1);
4891                 /* Set an Initial EITR value */
4892                 ixgbe_set_eitr(adapter, que->msix, IXGBE_INTR_RATE);
4893         }
4894
4895         /* For the Link interrupt */
4896         ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
4897 }
4898
4899 /*
4900 ** ixgbe_sfp_probe - called in the local timer to
4901 ** determine if a port had optics inserted.
4902 */
4903 static bool ixgbe_sfp_probe(struct adapter *adapter)
4904 {
4905         struct ixgbe_hw *hw = &adapter->hw;
4906         device_t        dev = adapter->dev;
4907         bool            result = FALSE;
4908
4909         if ((hw->phy.type == ixgbe_phy_nl) &&
4910             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
4911                 s32 ret = hw->phy.ops.identify_sfp(hw);
4912                 if (ret)
4913                         goto out;
4914                 ret = hw->phy.ops.reset(hw);
4915                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4916                         device_printf(dev,"Unsupported SFP+ module detected!");
4917                         kprintf(" Reload driver with supported module.\n");
4918                         adapter->sfp_probe = FALSE;
4919                         goto out;
4920                 } else
4921                         device_printf(dev,"SFP+ module detected!\n");
4922                 /* We now have supported optics */
4923                 adapter->sfp_probe = FALSE;
4924                 /* Set the optics type so system reports correctly */
4925                 ixgbe_setup_optics(adapter);
4926                 result = TRUE;
4927         }
4928 out:
4929         return (result);
4930 }
4931
4932 /*
4933 ** Tasklet for handling SFP module interrupts
4934 */
4935 static void
4936 ixgbe_handle_mod(void *context, int pending)
4937 {
4938         struct adapter  *adapter = context;
4939         struct ixgbe_hw *hw = &adapter->hw;
4940         device_t        dev = adapter->dev;
4941         u32 err;
4942
4943         err = hw->phy.ops.identify_sfp(hw);
4944         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4945                 device_printf(dev,
4946                     "Unsupported SFP+ module type was detected.\n");
4947                 return;
4948         }
4949         err = hw->mac.ops.setup_sfp(hw);
4950         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4951                 device_printf(dev,
4952                     "Setup failure - unsupported SFP+ module type.\n");
4953                 return;
4954         }
4955         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
4956         return;
4957 }
4958
4959
4960 /*
4961 ** Tasklet for handling MSF (multispeed fiber) interrupts
4962 */
4963 static void
4964 ixgbe_handle_msf(void *context, int pending)
4965 {
4966         struct adapter  *adapter = context;
4967         struct ixgbe_hw *hw = &adapter->hw;
4968         u32 autoneg;
4969         bool negotiate;
4970
4971         autoneg = hw->phy.autoneg_advertised;
4972         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
4973                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
4974         if (hw->mac.ops.setup_link)
4975                 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
4976         return;
4977 }
4978
4979 #ifdef IXGBE_FDIR
4980 /*
4981 ** Tasklet for reinitializing the Flow Director filter table
4982 */
4983 static void
4984 ixgbe_reinit_fdir(void *context, int pending)
4985 {
4986         struct adapter  *adapter = context;
4987         struct ifnet   *ifp = adapter->ifp;
4988
4989         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
4990                 return;
4991         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
4992         adapter->fdir_reinit = 0;
4993         /* re-enable flow director interrupts */
4994         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
4995         /* Restart the interface */
4996         ifp->if_drv_flags |= IFF_DRV_RUNNING;
4997         return;
4998 }
4999 #endif
5000
5001 /**********************************************************************
5002  *
5003  *  Update the board statistics counters.
5004  *
5005  **********************************************************************/
5006 static void
5007 ixgbe_update_stats_counters(struct adapter *adapter)
5008 {
5009         struct ifnet   *ifp = adapter->ifp;
5010         struct ixgbe_hw *hw = &adapter->hw;
5011         u32  missed_rx = 0, bprc, lxon, lxoff, total;
5012         u64  total_missed_rx = 0;
5013
5014         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5015         adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5016         adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5017         adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5018
5019         for (int i = 0; i < 8; i++) {
5020                 u32 mp;
5021                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5022                 /* missed_rx tallies misses for the gprc workaround */
5023                 missed_rx += mp;
5024                 /* global total per queue */
5025                 adapter->stats.mpc[i] += mp;
5026                 /* Running comprehensive total for stats display */
5027                 total_missed_rx += adapter->stats.mpc[i];
5028                 if (hw->mac.type == ixgbe_mac_82598EB)
5029                         adapter->stats.rnbc[i] +=
5030                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5031                 adapter->stats.pxontxc[i] +=
5032                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5033                 adapter->stats.pxonrxc[i] +=
5034                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5035                 adapter->stats.pxofftxc[i] +=
5036                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5037                 adapter->stats.pxoffrxc[i] +=
5038                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5039                 adapter->stats.pxon2offc[i] +=
5040                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5041         }
5042         for (int i = 0; i < 16; i++) {
5043                 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5044                 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5045                 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5046                 adapter->stats.qbrc[i] +=
5047                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5048                 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5049                 adapter->stats.qbtc[i] +=
5050                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5051                 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5052         }
5053         adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5054         adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5055         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5056
5057         /* Hardware workaround, gprc counts missed packets */
5058         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5059         adapter->stats.gprc -= missed_rx;
5060
5061         if (hw->mac.type != ixgbe_mac_82598EB) {
5062                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5063                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5064                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5065                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5066                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5067                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5068                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5069                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5070         } else {
5071                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5072                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5073                 /* 82598 only has a counter in the high register */
5074                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5075                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5076                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5077         }
5078
5079         /*
5080          * Workaround: mprc hardware is incorrectly counting
5081          * broadcasts, so for now we subtract those.
5082          */
5083         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5084         adapter->stats.bprc += bprc;
5085         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5086         if (hw->mac.type == ixgbe_mac_82598EB)
5087                 adapter->stats.mprc -= bprc;
5088
5089         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5090         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5091         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5092         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5093         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5094         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5095
5096         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5097         adapter->stats.lxontxc += lxon;
5098         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5099         adapter->stats.lxofftxc += lxoff;
5100         total = lxon + lxoff;
5101
5102         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5103         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5104         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5105         adapter->stats.gptc -= total;
5106         adapter->stats.mptc -= total;
5107         adapter->stats.ptc64 -= total;
5108         adapter->stats.gotc -= total * ETHER_MIN_LEN;
5109
5110         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5111         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5112         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5113         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5114         adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5115         adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5116         adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5117         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5118         adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5119         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5120         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5121         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5122         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5123         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5124         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5125         adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5126         adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5127         adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5128         /* Only read FCOE on 82599 */
5129         if (hw->mac.type != ixgbe_mac_82598EB) {
5130                 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5131                 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5132                 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5133                 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5134                 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5135         }
5136
5137         /* Fill out the OS statistics structure */
5138         ifp->if_ipackets = adapter->stats.gprc;
5139         ifp->if_opackets = adapter->stats.gptc;
5140         ifp->if_ibytes = adapter->stats.gorc;
5141         ifp->if_obytes = adapter->stats.gotc;
5142         ifp->if_imcasts = adapter->stats.mprc;
5143         ifp->if_collisions = 0;
5144
5145         /* Rx Errors */
5146         ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs +
5147                 adapter->stats.rlec;
5148 }
5149
5150 /** ixgbe_sysctl_tdh_handler - Handler function
5151  *  Retrieves the TDH value from the hardware
5152  */
5153 static int
5154 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5155 {
5156         int error;
5157
5158         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5159         if (!txr) return 0;
5160
5161         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5162         error = sysctl_handle_int(oidp, &val, 0, req);
5163         if (error || !req->newptr)
5164                 return error;
5165         return 0;
5166 }
5167
5168 /** ixgbe_sysctl_tdt_handler - Handler function
5169  *  Retrieves the TDT value from the hardware
5170  */
5171 static int
5172 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5173 {
5174         int error;
5175
5176         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5177         if (!txr) return 0;
5178
5179         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5180         error = sysctl_handle_int(oidp, &val, 0, req);
5181         if (error || !req->newptr)
5182                 return error;
5183         return 0;
5184 }
5185
5186 /** ixgbe_sysctl_rdh_handler - Handler function
5187  *  Retrieves the RDH value from the hardware
5188  */
5189 static int
5190 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5191 {
5192         int error;
5193
5194         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5195         if (!rxr) return 0;
5196
5197         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5198         error = sysctl_handle_int(oidp, &val, 0, req);
5199         if (error || !req->newptr)
5200                 return error;
5201         return 0;
5202 }
5203
5204 /** ixgbe_sysctl_rdt_handler - Handler function
5205  *  Retrieves the RDT value from the hardware
5206  */
5207 static int
5208 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5209 {
5210         int error;
5211
5212         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5213         if (!rxr) return 0;
5214
5215         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5216         error = sysctl_handle_int(oidp, &val, 0, req);
5217         if (error || !req->newptr)
5218                 return error;
5219         return 0;
5220 }
5221
5222 /*
5223  * Add sysctl variables, one per statistic, to the system.
5224  */
5225 static void
5226 ixgbe_add_hw_stats(struct adapter *adapter)
5227 {
5228         struct tx_ring *txr = adapter->tx_rings;
5229         struct rx_ring *rxr = adapter->rx_rings;
5230
5231         struct sysctl_ctx_list *ctx = &adapter->sysctl_ctx;
5232         struct sysctl_oid *tree = adapter->sysctl_tree;
5233         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5234         struct ixgbe_hw_stats *stats = &adapter->stats;
5235
5236         struct sysctl_oid *stat_node, *queue_node;
5237         struct sysctl_oid_list *stat_list, *queue_list;
5238
5239 #define QUEUE_NAME_LEN 32
5240         char namebuf[QUEUE_NAME_LEN];
5241
5242         /* Driver Statistics */
5243         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5244                         CTLFLAG_RD, &adapter->dropped_pkts,
5245                         "Driver dropped packets");
5246         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5247                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5248                         "m_defrag() failed");
5249         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup",
5250                         CTLFLAG_RD, &adapter->no_tx_dma_setup,
5251                         "Driver tx dma failure in xmit");
5252         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5253                         CTLFLAG_RD, &adapter->watchdog_events,
5254                         "Watchdog timeouts");
5255         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx",
5256                         CTLFLAG_RD, &adapter->tso_tx,
5257                         "TSO");
5258         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5259                         CTLFLAG_RD, &adapter->link_irq,
5260                         "Link MSIX IRQ Handled");
5261
5262         for (int i = 0; i < adapter->num_queues; i++, txr++) {
5263         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5264                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5265                                             CTLFLAG_RD, NULL, "Queue Name");
5266                 queue_list = SYSCTL_CHILDREN(queue_node);
5267                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5268                                 CTLFLAG_RD, &(adapter->queues[i].irqs), 0,
5269                                 "irqs on this queue");
5270                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5271                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5272                                 ixgbe_sysctl_tdh_handler, "IU",
5273                                 "Transmit Descriptor Head");
5274                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5275                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5276                                 ixgbe_sysctl_tdt_handler, "IU",
5277                                 "Transmit Descriptor Tail");
5278                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5279                                 CTLFLAG_RD, &txr->no_desc_avail, 0,
5280                                 "Queue No Descriptor Available");
5281                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5282                                 CTLFLAG_RD, &txr->total_packets, 0,
5283                                 "Queue Packets Transmitted");
5284         }
5285
5286         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5287         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5288                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5289                                             CTLFLAG_RD, NULL, "Queue Name");
5290                 queue_list = SYSCTL_CHILDREN(queue_node);
5291
5292 #if 0   /* NET_LRO */
5293                 struct lro_ctrl *lro = &rxr->lro;
5294 #endif
5295
5296         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5297                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5298                                             CTLFLAG_RD, NULL, "Queue Name");
5299                 queue_list = SYSCTL_CHILDREN(queue_node);
5300
5301                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5302                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5303                                 ixgbe_sysctl_rdh_handler, "IU",
5304                                 "Receive Descriptor Head");
5305                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5306                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5307                                 ixgbe_sysctl_rdt_handler, "IU",
5308                                 "Receive Descriptor Tail");
5309                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5310                                 CTLFLAG_RD, &rxr->rx_packets, 0,
5311                                 "Queue Packets Received");
5312                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5313                                 CTLFLAG_RD, &rxr->rx_bytes, 0,
5314                                 "Queue Bytes Received");
5315 #if 0   /* NET_LRO */
5316                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5317                                 CTLFLAG_RD, &lro->lro_queued, 0,
5318                                 "LRO Queued");
5319                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5320                                 CTLFLAG_RD, &lro->lro_flushed, 0,
5321                                 "LRO Flushed");
5322 #endif
5323         }
5324
5325         /* MAC stats get the own sub node */
5326
5327         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5328                                     CTLFLAG_RD, NULL, "MAC Statistics");
5329         stat_list = SYSCTL_CHILDREN(stat_node);
5330
5331         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5332                         CTLFLAG_RD, &stats->crcerrs, 0,
5333                         "CRC Errors");
5334         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5335                         CTLFLAG_RD, &stats->illerrc, 0,
5336                         "Illegal Byte Errors");
5337         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5338                         CTLFLAG_RD, &stats->errbc, 0,
5339                         "Byte Errors");
5340         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5341                         CTLFLAG_RD, &stats->mspdc, 0,
5342                         "MAC Short Packets Discarded");
5343         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5344                         CTLFLAG_RD, &stats->mlfc, 0,
5345                         "MAC Local Faults");
5346         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5347                         CTLFLAG_RD, &stats->mrfc, 0,
5348                         "MAC Remote Faults");
5349         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5350                         CTLFLAG_RD, &stats->rlec, 0,
5351                         "Receive Length Errors");
5352         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd",
5353                         CTLFLAG_RD, &stats->lxontxc, 0,
5354                         "Link XON Transmitted");
5355         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd",
5356                         CTLFLAG_RD, &stats->lxonrxc, 0,
5357                         "Link XON Received");
5358         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd",
5359                         CTLFLAG_RD, &stats->lxofftxc, 0,
5360                         "Link XOFF Transmitted");
5361         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd",
5362                         CTLFLAG_RD, &stats->lxoffrxc, 0,
5363                         "Link XOFF Received");
5364
5365         /* Packet Reception Stats */
5366         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5367                         CTLFLAG_RD, &stats->tor, 0,
5368                         "Total Octets Received");
5369         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5370                         CTLFLAG_RD, &stats->gorc, 0,
5371                         "Good Octets Received");
5372         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5373                         CTLFLAG_RD, &stats->tpr, 0,
5374                         "Total Packets Received");
5375         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5376                         CTLFLAG_RD, &stats->gprc, 0,
5377                         "Good Packets Received");
5378         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5379                         CTLFLAG_RD, &stats->mprc, 0,
5380                         "Multicast Packets Received");
5381         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5382                         CTLFLAG_RD, &stats->bprc, 0,
5383                         "Broadcast Packets Received");
5384         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5385                         CTLFLAG_RD, &stats->prc64, 0,
5386                         "64 byte frames received ");
5387         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5388                         CTLFLAG_RD, &stats->prc127, 0,
5389                         "65-127 byte frames received");
5390         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5391                         CTLFLAG_RD, &stats->prc255, 0,
5392                         "128-255 byte frames received");
5393         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5394                         CTLFLAG_RD, &stats->prc511, 0,
5395                         "256-511 byte frames received");
5396         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5397                         CTLFLAG_RD, &stats->prc1023, 0,
5398                         "512-1023 byte frames received");
5399         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5400                         CTLFLAG_RD, &stats->prc1522, 0,
5401                         "1023-1522 byte frames received");
5402         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5403                         CTLFLAG_RD, &stats->ruc, 0,
5404                         "Receive Undersized");
5405         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5406                         CTLFLAG_RD, &stats->rfc, 0,
5407                         "Fragmented Packets Received ");
5408         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5409                         CTLFLAG_RD, &stats->roc, 0,
5410                         "Oversized Packets Received");
5411         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5412                         CTLFLAG_RD, &stats->rjc, 0,
5413                         "Received Jabber");
5414         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5415                         CTLFLAG_RD, &stats->mngprc, 0,
5416                         "Management Packets Received");
5417         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5418                         CTLFLAG_RD, &stats->mngptc, 0,
5419                         "Management Packets Dropped");
5420         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5421                         CTLFLAG_RD, &stats->xec, 0,
5422                         "Checksum Errors");
5423
5424         /* Packet Transmission Stats */
5425         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5426                         CTLFLAG_RD, &stats->gotc, 0,
5427                         "Good Octets Transmitted");
5428         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5429                         CTLFLAG_RD, &stats->tpt, 0,
5430                         "Total Packets Transmitted");
5431         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5432                         CTLFLAG_RD, &stats->gptc, 0,
5433                         "Good Packets Transmitted");
5434         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5435                         CTLFLAG_RD, &stats->bptc, 0,
5436                         "Broadcast Packets Transmitted");
5437         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5438                         CTLFLAG_RD, &stats->mptc, 0,
5439                         "Multicast Packets Transmitted");
5440         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5441                         CTLFLAG_RD, &stats->mngptc, 0,
5442                         "Management Packets Transmitted");
5443         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5444                         CTLFLAG_RD, &stats->ptc64, 0,
5445                         "64 byte frames transmitted ");
5446         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5447                         CTLFLAG_RD, &stats->ptc127, 0,
5448                         "65-127 byte frames transmitted");
5449         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5450                         CTLFLAG_RD, &stats->ptc255, 0,
5451                         "128-255 byte frames transmitted");
5452         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5453                         CTLFLAG_RD, &stats->ptc511, 0,
5454                         "256-511 byte frames transmitted");
5455         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5456                         CTLFLAG_RD, &stats->ptc1023, 0,
5457                         "512-1023 byte frames transmitted");
5458         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5459                         CTLFLAG_RD, &stats->ptc1522, 0,
5460                         "1024-1522 byte frames transmitted");
5461
5462         /* FC Stats */
5463         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc",
5464                 CTLFLAG_RD, &stats->fccrc, 0,
5465                 "FC CRC Errors");
5466         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last",
5467                 CTLFLAG_RD, &stats->fclast, 0,
5468                 "FC Last Error");
5469         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd",
5470                 CTLFLAG_RD, &stats->fcoerpdc, 0,
5471                 "FCoE Packets Dropped");
5472         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd",
5473                 CTLFLAG_RD, &stats->fcoeprc, 0,
5474                 "FCoE Packets Received");
5475         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd",
5476                 CTLFLAG_RD, &stats->fcoeptc, 0,
5477                 "FCoE Packets Transmitted");
5478         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd",
5479                 CTLFLAG_RD, &stats->fcoedwrc, 0,
5480                 "FCoE DWords Received");
5481         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd",
5482                 CTLFLAG_RD, &stats->fcoedwtc, 0,
5483                 "FCoE DWords Transmitted");
5484 }
5485
5486 /*
5487 ** Set flow control using sysctl:
5488 ** Flow control values:
5489 **      0 - off
5490 **      1 - rx pause
5491 **      2 - tx pause
5492 **      3 - full
5493 */
5494 static int
5495 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5496 {
5497         int error, last;
5498         struct adapter *adapter = (struct adapter *) arg1;
5499
5500         last = adapter->fc;
5501         error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5502         if ((error) || (req->newptr == NULL))
5503                 return (error);
5504
5505         /* Don't bother if it's not changed */
5506         if (adapter->fc == last)
5507                 return (0);
5508
5509         switch (adapter->fc) {
5510                 case ixgbe_fc_rx_pause:
5511                 case ixgbe_fc_tx_pause:
5512                 case ixgbe_fc_full:
5513                         adapter->hw.fc.requested_mode = adapter->fc;
5514                         break;
5515                 case ixgbe_fc_none:
5516                 default:
5517                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
5518         }
5519         /* Don't autoneg if forcing a value */
5520         adapter->hw.fc.disable_fc_autoneg = TRUE;
5521         ixgbe_fc_enable(&adapter->hw);
5522         return error;
5523 }
5524
5525 static void
5526 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5527         const char *description, int *limit, int value)
5528 {
5529         *limit = value;
5530         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
5531             SYSCTL_CHILDREN(adapter->sysctl_tree),
5532             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5533 }
5534
5535 /*
5536 ** Control link advertise speed:
5537 **      1 - advertise only 1G
5538 **      2 - advertise 100Mb
5539 **      3 - advertise normal
5540 */
5541 static int
5542 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5543 {
5544         int                     error = 0;
5545         struct adapter          *adapter;
5546         device_t                dev;
5547         struct ixgbe_hw         *hw;
5548         ixgbe_link_speed        speed, last;
5549
5550         adapter = (struct adapter *) arg1;
5551         dev = adapter->dev;
5552         hw = &adapter->hw;
5553         last = adapter->advertise;
5554
5555         error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5556         if ((error) || (adapter->advertise == -1))
5557                 return (error);
5558
5559         if (adapter->advertise == last) /* no change */
5560                 return (0);
5561
5562         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5563             (hw->phy.multispeed_fiber)))
5564                 return (error);
5565
5566         if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5567                 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5568                 return (error);
5569         }
5570
5571         if (adapter->advertise == 1)
5572                 speed = IXGBE_LINK_SPEED_1GB_FULL;
5573         else if (adapter->advertise == 2)
5574                 speed = IXGBE_LINK_SPEED_100_FULL;
5575         else if (adapter->advertise == 3)
5576                 speed = IXGBE_LINK_SPEED_1GB_FULL |
5577                         IXGBE_LINK_SPEED_10GB_FULL;
5578         else /* bogus value */
5579                 return (error);
5580
5581         hw->mac.autotry_restart = TRUE;
5582         hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5583
5584         return (error);
5585 }
5586
5587 /*
5588 ** Thermal Shutdown Trigger
5589 **   - cause a Thermal Overtemp IRQ
5590 */
5591 static int
5592 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5593 {
5594         int             error, fire = 0;
5595         struct adapter  *adapter = (struct adapter *) arg1;
5596         struct ixgbe_hw *hw = &adapter->hw;
5597
5598
5599         if (hw->mac.type != ixgbe_mac_X540)
5600                 return (0);
5601
5602         error = sysctl_handle_int(oidp, &fire, 0, req);
5603         if ((error) || (req->newptr == NULL))
5604                 return (error);
5605
5606         if (fire) {
5607                 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5608                 reg |= IXGBE_EICR_TS;
5609                 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5610         }
5611
5612         return (0);
5613 }
5614
5615 static void
5616 ixgbe_set_eitr(struct adapter *sc, int idx, int rate)
5617 {
5618         uint32_t eitr = 0;
5619
5620         /* convert rate in intr/s to hw representation */
5621         if (rate > 0) {
5622                 eitr = 1000000 / rate;
5623                 eitr <<= IXGBE_EITR_INTVL_SHIFT;
5624
5625                 if (eitr == 0) {
5626                         /* Don't disable it */
5627                         eitr = 1 << IXGBE_EITR_INTVL_SHIFT;
5628                 } else if (eitr > IXGBE_EITR_INTVL_MASK) {
5629                         /* Don't allow it to be too large */
5630                         eitr = IXGBE_EITR_INTVL_MASK;
5631                 }
5632         }
5633         IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(idx), eitr);
5634 }
5635
5636 static int
5637 ixgbe_sysctl_intr_rate(SYSCTL_HANDLER_ARGS)
5638 {
5639         struct adapter *sc = (void *)arg1;
5640         struct ifnet *ifp = sc->ifp;
5641         int error, intr_rate, running;
5642         struct ix_queue *que = sc->queues;
5643
5644         intr_rate = sc->intr_rate;
5645         error = sysctl_handle_int(oidp, &intr_rate, 0, req);
5646         if (error || req->newptr == NULL)
5647                 return error;
5648         if (intr_rate < 0)
5649                 return EINVAL;
5650
5651         ifnet_serialize_all(ifp);
5652
5653         sc->intr_rate = intr_rate;
5654         running = ifp->if_flags & IFF_RUNNING;
5655         if (running)
5656                 ixgbe_set_eitr(sc, 0, sc->intr_rate);
5657
5658         if (running && (sc->intr_type == PCI_INTR_TYPE_MSIX)) {
5659                 for (int i = 0; i < sc->num_queues; i++, que++)
5660                         ixgbe_set_eitr(sc, que->msix, sc->intr_rate);
5661         }
5662
5663         if (bootverbose)
5664                 if_printf(ifp, "interrupt rate set to %d/sec\n", sc->intr_rate);
5665
5666         ifnet_deserialize_all(ifp);
5667
5668         return 0;
5669 }
5670
5671 /* rearrange mbuf chain to get contiguous bytes */
5672 static int
5673 ixgbe_tso_pullup(struct tx_ring *txr, struct mbuf **mp)
5674 {
5675         int hoff, iphlen, thoff;
5676         struct mbuf *m;
5677
5678         m = *mp;
5679         KASSERT(M_WRITABLE(m), ("TSO mbuf not writable"));
5680
5681         iphlen = m->m_pkthdr.csum_iphlen;
5682         thoff = m->m_pkthdr.csum_thlen;
5683         hoff = m->m_pkthdr.csum_lhlen;
5684
5685         KASSERT(iphlen > 0, ("invalid ip hlen"));
5686         KASSERT(thoff > 0, ("invalid tcp hlen"));
5687         KASSERT(hoff > 0, ("invalid ether hlen"));
5688
5689         if (__predict_false(m->m_len < hoff + iphlen + thoff)) {
5690                 m = m_pullup(m, hoff + iphlen + thoff);
5691                 if (m == NULL) {
5692                         *mp = NULL;
5693                         return ENOBUFS;
5694                 }
5695                 *mp = m;
5696         }
5697
5698         return 0;
5699 }