sys/dev/netif/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2012, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
  34
  35 #include "opt_inet.h"
  36 #include "opt_inet6.h"
  37
  38 #include "ixgbe.h"
  39
  40 /*********************************************************************
  41  *  Set this to one to display debug statistics
  42  *********************************************************************/
  43 int             ixgbe_display_debug_stats = 0;
  44
  45 /*********************************************************************
  46  *  Driver version
  47  *********************************************************************/
  48 char ixgbe_driver_version[] = "2.4.8";
  49
  50 /*********************************************************************
  51  *  PCI Device ID Table
  52  *
  53  *  Used by probe to select devices to load on
  54  *  Last field stores an index into ixgbe_strings
  55  *  Last entry must be all 0s
  56  *
  57  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  58  *********************************************************************/
  59
  60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  61 {
  62         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
  81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
  82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
  83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
  84         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
  85         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
  86         /* required last entry */
  87         {0, 0, 0, 0, 0}
  88 };
  89
  90 /*********************************************************************
  91  *  Table of branding strings
  92  *********************************************************************/
  93
  94 static char    *ixgbe_strings[] = {
  95         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  96 };
  97
  98 /*********************************************************************
  99  *  Function prototypes
 100  *********************************************************************/
 101 static int      ixgbe_probe(device_t);
 102 static int      ixgbe_attach(device_t);
 103 static int      ixgbe_detach(device_t);
 104 static int      ixgbe_shutdown(device_t);
 105 static void     ixgbe_start(struct ifnet *);
 106 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
 107 #if 0 /* __FreeBSD_version >= 800000 */
 108 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
 109 static int      ixgbe_mq_start_locked(struct ifnet *,
 110                     struct tx_ring *, struct mbuf *);
 111 static void     ixgbe_qflush(struct ifnet *);
 112 #endif
 113 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 114 static void     ixgbe_init(void *);
 115 static void     ixgbe_init_locked(struct adapter *);
 116 static void     ixgbe_stop(void *);
 117 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 118 static int      ixgbe_media_change(struct ifnet *);
 119 static void     ixgbe_identify_hardware(struct adapter *);
 120 static int      ixgbe_allocate_pci_resources(struct adapter *);
 121 static int      ixgbe_allocate_msix(struct adapter *);
 122 static int      ixgbe_allocate_legacy(struct adapter *);
 123 static int      ixgbe_allocate_queues(struct adapter *);
 124 static int      ixgbe_setup_msix(struct adapter *);
 125 static void     ixgbe_free_pci_resources(struct adapter *);
 126 static void     ixgbe_local_timer(void *);
 127 static int      ixgbe_setup_interface(device_t, struct adapter *);
 128 static void     ixgbe_config_link(struct adapter *);
 129
 130 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 131 static int      ixgbe_setup_transmit_structures(struct adapter *);
 132 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 133 static void     ixgbe_initialize_transmit_units(struct adapter *);
 134 static void     ixgbe_free_transmit_structures(struct adapter *);
 135 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 136
 137 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 138 static int      ixgbe_setup_receive_structures(struct adapter *);
 139 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 140 static void     ixgbe_initialize_receive_units(struct adapter *);
 141 static void     ixgbe_free_receive_structures(struct adapter *);
 142 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 143 #if 0   /* NET_LRO */
 144 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
 145 #endif
 146
 147 static void     ixgbe_enable_intr(struct adapter *);
 148 static void     ixgbe_disable_intr(struct adapter *);
 149 static void     ixgbe_update_stats_counters(struct adapter *);
 150 static void     ixgbe_txeof(struct tx_ring *);
 151 static void     ixgbe_rxeof(struct ix_queue *, int);
 152 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
 153 static void     ixgbe_set_promisc(struct adapter *);
 154 static void     ixgbe_set_multi(struct adapter *);
 155 static void     ixgbe_update_link_status(struct adapter *);
 156 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
 157 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 158 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 159 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
 160 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
 161 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 162                     struct ixgbe_dma_alloc *, int);
 163 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 164 static void     ixgbe_add_rx_process_limit(struct adapter *, const char *,
 165                     const char *, int *, int);
 166 static bool     ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 167 static bool     ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
 168 static int      ixgbe_tso_pullup(struct tx_ring *, struct mbuf **);
 169 static void     ixgbe_add_sysctl(struct adapter *);
 170 static void     ixgbe_set_eitr(struct adapter *, int, int);
 171 static int      ixgbe_sysctl_intr_rate(SYSCTL_HANDLER_ARGS);
 172 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 173 static void     ixgbe_configure_ivars(struct adapter *);
 174 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 175
 176 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 177 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 178 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 179
 180 static void     ixgbe_add_hw_stats(struct adapter *adapter);
 181
 182 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
 183 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
 184                     struct mbuf *, u32);
 185
 186 /* Support for pluggable optic modules */
 187 static bool     ixgbe_sfp_probe(struct adapter *);
 188 static void     ixgbe_setup_optics(struct adapter *);
 189
 190 /* Legacy (single vector interrupt handler */
 191 static void     ixgbe_legacy_irq(void *);
 192
 193 /* The MSI/X Interrupt handlers */
 194 static void     ixgbe_msix_que(void *);
 195 static void     ixgbe_msix_link(void *);
 196
 197 /* Deferred interrupt tasklets */
 198 static void     ixgbe_handle_msf(void *, int);
 199 static void     ixgbe_handle_mod(void *, int);
 200
 201 #ifdef IXGBE_FDIR
 202 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
 203 static void     ixgbe_reinit_fdir(void *, int);
 204 #endif
 205
 206 /*********************************************************************
 207  *  FreeBSD Device Interface Entry Points
 208  *********************************************************************/
 209
 210 static device_method_t ixgbe_methods[] = {
 211         /* Device interface */
 212         DEVMETHOD(device_probe, ixgbe_probe),
 213         DEVMETHOD(device_attach, ixgbe_attach),
 214         DEVMETHOD(device_detach, ixgbe_detach),
 215         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 216         {0, 0}
 217 };
 218
 219 static driver_t ixgbe_driver = {
 220         "ix", ixgbe_methods, sizeof(struct adapter),
 221 };
 222
 223 devclass_t ixgbe_devclass;
 224 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
 225
 226 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 227 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 228
 229 /*
 230 ** TUNEABLE PARAMETERS:
 231 */
 232
 233 /* How many packets rxeof tries to clean at a time */
 234 static int ixgbe_rx_process_limit = 128;
 235 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 236
 237 /*
 238 ** Smart speed setting, default to on
 239 ** this only works as a compile option
 240 ** right now as its during attach, set
 241 ** this to 'ixgbe_smart_speed_off' to
 242 ** disable.
 243 */
 244 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 245
 246 static int ixgbe_msi_enable = 1;
 247 TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
 248
 249 /*
 250  * MSIX should be the default for best performance,
 251  * but this allows it to be forced off for testing.
 252  */
 253 static int ixgbe_enable_msix = 1;
 254 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 255
 256 /*
 257  * Header split: this causes the hardware to DMA
 258  * the header into a separate mbuf from the payload,
 259  * it can be a performance win in some workloads, but
 260  * in others it actually hurts, its off by default.
 261  */
 262 static int ixgbe_header_split = FALSE;
 263 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
 264
 265 /*
 266  * Number of Queues, can be set to 0,
 267  * it then autoconfigures based on the
 268  * number of cpus with a max of 8. This
 269  * can be overriden manually here.
 270  */
 271 static int ixgbe_num_queues = 0;
 272 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
 273
 274 /*
 275 ** Number of TX descriptors per ring,
 276 ** setting higher than RX as this seems
 277 ** the better performing choice.
 278 */
 279 static int ixgbe_txd = PERFORM_TXD;
 280 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 281
 282 /* Number of RX descriptors per ring */
 283 static int ixgbe_rxd = PERFORM_RXD;
 284 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 285
 286 /* Keep running tab on them for sanity check */
 287 static int ixgbe_total_ports;
 288
 289 #ifdef IXGBE_FDIR
 290 /*
 291 ** For Flow Director: this is the
 292 ** number of TX packets we sample
 293 ** for the filter pool, this means
 294 ** every 20th packet will be probed.
 295 **
 296 ** This feature can be disabled by
 297 ** setting this to 0.
 298 */
 299 static int atr_sample_rate = 20;
 300 /*
 301 ** Flow Director actually 'steals'
 302 ** part of the packet buffer as its
 303 ** filter pool, this variable controls
 304 ** how much it uses:
 305 **  0 = 64K, 1 = 128K, 2 = 256K
 306 */
 307 static int fdir_pballoc = 1;
 308 #endif
 309
 310 #ifdef DEV_NETMAP
 311 /*
 312  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
 313  * be a reference on how to implement netmap support in a driver.
 314  * Additional comments are in ixgbe_netmap.h .
 315  *
 316  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
 317  * that extend the standard driver.
 318  */
 319 #include <dev/netmap/ixgbe_netmap.h>
 320 #endif /* DEV_NETMAP */
 321
 322 /*********************************************************************
 323  *  Device identification routine
 324  *
 325  *  ixgbe_probe determines if the driver should be loaded on
 326  *  adapter based on PCI vendor/device id of the adapter.
 327  *
 328  *  return BUS_PROBE_DEFAULT on success, positive on failure
 329  *********************************************************************/
 330
 331 static int
 332 ixgbe_probe(device_t dev)
 333 {
 334         ixgbe_vendor_info_t *ent;
 335
 336         u16     pci_vendor_id = 0;
 337         u16     pci_device_id = 0;
 338         u16     pci_subvendor_id = 0;
 339         u16     pci_subdevice_id = 0;
 340         char    adapter_name[256];
 341
 342         INIT_DEBUGOUT("ixgbe_probe: begin");
 343
 344         pci_vendor_id = pci_get_vendor(dev);
 345         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 346                 return (ENXIO);
 347
 348         pci_device_id = pci_get_device(dev);
 349         pci_subvendor_id = pci_get_subvendor(dev);
 350         pci_subdevice_id = pci_get_subdevice(dev);
 351
 352         ent = ixgbe_vendor_info_array;
 353         while (ent->vendor_id != 0) {
 354                 if ((pci_vendor_id == ent->vendor_id) &&
 355                     (pci_device_id == ent->device_id) &&
 356
 357                     ((pci_subvendor_id == ent->subvendor_id) ||
 358                      (ent->subvendor_id == 0)) &&
 359
 360                     ((pci_subdevice_id == ent->subdevice_id) ||
 361                      (ent->subdevice_id == 0))) {
 362                         ksprintf(adapter_name, "%s, Version - %s",
 363                                 ixgbe_strings[ent->index],
 364                                 ixgbe_driver_version);
 365                         device_set_desc_copy(dev, adapter_name);
 366                         ++ixgbe_total_ports;
 367                         return (BUS_PROBE_DEFAULT);
 368                 }
 369                 ent++;
 370         }
 371         return (ENXIO);
 372 }
 373
 374 /*********************************************************************
 375  *  Device initialization routine
 376  *
 377  *  The attach entry point is called when the driver is being loaded.
 378  *  This routine identifies the type of hardware, allocates all resources
 379  *  and initializes the hardware.
 380  *
 381  *  return 0 on success, positive on failure
 382  *********************************************************************/
 383
 384 static int
 385 ixgbe_attach(device_t dev)
 386 {
 387         struct adapter *adapter;
 388         struct ixgbe_hw *hw;
 389         int             error = 0;
 390         u16             csum;
 391         u32             ctrl_ext;
 392
 393         INIT_DEBUGOUT("ixgbe_attach: begin");
 394
 395         if (resource_disabled("ixgbe", device_get_unit(dev))) {
 396                 device_printf(dev, "Disabled by device hint\n");
 397                 return (ENXIO);
 398         }
 399
 400         /* Allocate, clear, and link in our adapter structure */
 401         adapter = device_get_softc(dev);
 402         adapter->dev = adapter->osdep.dev = dev;
 403         hw = &adapter->hw;
 404
 405         /* Core Lock Init*/
 406         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 407
 408         /* Set up the timer callout */
 409         callout_init_mp(&adapter->timer);
 410
 411         /* Determine hardware revision */
 412         ixgbe_identify_hardware(adapter);
 413
 414         /* Enable bus mastering */
 415         pci_enable_busmaster(dev);
 416
 417         /* Do base PCI setup - map BAR0 */
 418         if (ixgbe_allocate_pci_resources(adapter)) {
 419                 device_printf(dev, "Allocation of PCI resources failed\n");
 420                 error = ENXIO;
 421                 goto err_out;
 422         }
 423
 424         /* Do descriptor calc and sanity checks */
 425         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 426             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 427                 device_printf(dev, "TXD config issue, using default!\n");
 428                 adapter->num_tx_desc = DEFAULT_TXD;
 429         } else
 430                 adapter->num_tx_desc = ixgbe_txd;
 431
 432         /*
 433         ** With many RX rings it is easy to exceed the
 434         ** system mbuf allocation. Tuning nmbclusters
 435         ** can alleviate this.
 436         */
 437         if (nmbclusters > 0 ) {
 438                 int s;
 439                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 440                 if (s > nmbclusters) {
 441                         device_printf(dev, "RX Descriptors exceed "
 442                             "system mbuf max, using default instead!\n");
 443                         ixgbe_rxd = DEFAULT_RXD;
 444                 }
 445         }
 446
 447         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 448             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 449                 device_printf(dev, "RXD config issue, using default!\n");
 450                 adapter->num_rx_desc = DEFAULT_RXD;
 451         } else
 452                 adapter->num_rx_desc = ixgbe_rxd;
 453
 454         /* Allocate our TX/RX Queues */
 455         if (ixgbe_allocate_queues(adapter)) {
 456                 error = ENOMEM;
 457                 goto err_out;
 458         }
 459
 460         /* Allocate multicast array memory. */
 461         adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
 462             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 463         if (adapter->mta == NULL) {
 464                 device_printf(dev, "Can not allocate multicast setup array\n");
 465                 error = ENOMEM;
 466                 goto err_late;
 467         }
 468
 469         /* Initialize the shared code */
 470         error = ixgbe_init_shared_code(hw);
 471         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 472                 /*
 473                 ** No optics in this port, set up
 474                 ** so the timer routine will probe
 475                 ** for later insertion.
 476                 */
 477                 adapter->sfp_probe = TRUE;
 478                 error = 0;
 479         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 480                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 481                 error = EIO;
 482                 goto err_late;
 483         } else if (error) {
 484                 device_printf(dev,"Unable to initialize the shared code\n");
 485                 error = EIO;
 486                 goto err_late;
 487         }
 488
 489         /* Make sure we have a good EEPROM before we read from it */
 490         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 491                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 492                 error = EIO;
 493                 goto err_late;
 494         }
 495
 496         error = ixgbe_init_hw(hw);
 497         switch (error) {
 498         case IXGBE_ERR_EEPROM_VERSION:
 499                 device_printf(dev, "This device is a pre-production adapter/"
 500                     "LOM.  Please be aware there may be issues associated "
 501                     "with your hardware.\n If you are experiencing problems "
 502                     "please contact your Intel or hardware representative "
 503                     "who provided you with this hardware.\n");
 504                 break;
 505         case IXGBE_ERR_SFP_NOT_SUPPORTED:
 506                 device_printf(dev,"Unsupported SFP+ Module\n");
 507                 error = EIO;
 508                 device_printf(dev,"Hardware Initialization Failure\n");
 509                 goto err_late;
 510         case IXGBE_ERR_SFP_NOT_PRESENT:
 511                 device_printf(dev,"No SFP+ Module found\n");
 512                 /* falls thru */
 513         default:
 514                 break;
 515         }
 516
 517         /* Detect and set physical type */
 518         ixgbe_setup_optics(adapter);
 519
 520         if ((adapter->msix > 1) && (ixgbe_enable_msix)) {
 521                 adapter->intr_type = PCI_INTR_TYPE_MSIX;
 522                 error = ixgbe_allocate_msix(adapter);
 523         } else {
 524                 error = ixgbe_allocate_legacy(adapter);
 525         }
 526         if (error)
 527                 goto err_late;
 528
 529         /* Setup OS specific network interface */
 530         if (ixgbe_setup_interface(dev, adapter) != 0)
 531                 goto err_late;
 532
 533         /* Add sysctl tree */
 534         ixgbe_add_sysctl(adapter);
 535
 536         /* Initialize statistics */
 537         ixgbe_update_stats_counters(adapter);
 538
 539         /* Register for VLAN events */
 540         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 541             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 542         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 543             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 544
 545         /* Print PCIE bus type/speed/width info */
 546         ixgbe_get_bus_info(hw);
 547         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
 548             ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
 549             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
 550             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
 551             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
 552             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
 553             ("Unknown"));
 554
 555         if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
 556             (hw->bus.speed == ixgbe_bus_speed_2500)) {
 557                 device_printf(dev, "PCI-Express bandwidth available"
 558                     " for this card\n     is not sufficient for"
 559                     " optimal performance.\n");
 560                 device_printf(dev, "For optimal performance a x8 "
 561                     "PCIE, or x4 PCIE 2 slot is required.\n");
 562         }
 563
 564         /* let hardware know driver is loaded */
 565         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 566         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 567         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 568
 569         ixgbe_add_hw_stats(adapter);
 570
 571 #ifdef DEV_NETMAP
 572         ixgbe_netmap_attach(adapter);
 573 #endif /* DEV_NETMAP */
 574         INIT_DEBUGOUT("ixgbe_attach: end");
 575         return (0);
 576 err_late:
 577         ixgbe_free_transmit_structures(adapter);
 578         ixgbe_free_receive_structures(adapter);
 579 err_out:
 580         if (adapter->ifp != NULL)
 581                 if_free(adapter->ifp);
 582         ixgbe_free_pci_resources(adapter);
 583         kfree(adapter->mta, M_DEVBUF);
 584         return (error);
 585
 586 }
 587
 588 /*********************************************************************
 589  *  Device removal routine
 590  *
 591  *  The detach entry point is called when the driver is being removed.
 592  *  This routine stops the adapter and deallocates all the resources
 593  *  that were allocated for driver operation.
 594  *
 595  *  return 0 on success, positive on failure
 596  *********************************************************************/
 597
 598 static int
 599 ixgbe_detach(device_t dev)
 600 {
 601         struct adapter *adapter = device_get_softc(dev);
 602         u32     ctrl_ext;
 603
 604         INIT_DEBUGOUT("ixgbe_detach: begin");
 605
 606         /* Make sure VLANS are not using driver */
 607         if (adapter->ifp->if_vlantrunks != NULL) {
 608                 device_printf(dev,"Vlan in use, detach first\n");
 609                 return (EBUSY);
 610         }
 611
 612         IXGBE_CORE_LOCK(adapter);
 613         ixgbe_stop(adapter);
 614         IXGBE_CORE_UNLOCK(adapter);
 615
 616         /* Drain the Link queue */
 617         if (adapter->tq) {
 618                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 619                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 620 #ifdef IXGBE_FDIR
 621                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 622 #endif
 623                 taskqueue_free(adapter->tq);
 624         }
 625
 626         /* let hardware know driver is unloading */
 627         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 628         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 629         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 630
 631         /* Unregister VLAN events */
 632         if (adapter->vlan_attach != NULL)
 633                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 634         if (adapter->vlan_detach != NULL)
 635                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 636
 637         ether_ifdetach(adapter->ifp);
 638         callout_stop(&adapter->timer);
 639 #ifdef DEV_NETMAP
 640         netmap_detach(adapter->ifp);
 641 #endif /* DEV_NETMAP */
 642         ixgbe_free_pci_resources(adapter);
 643         bus_generic_detach(dev);
 644         if_free(adapter->ifp);
 645
 646         ixgbe_free_transmit_structures(adapter);
 647         ixgbe_free_receive_structures(adapter);
 648         kfree(adapter->mta, M_DEVBUF);
 649         sysctl_ctx_free(&adapter->sysctl_ctx);
 650
 651         IXGBE_CORE_LOCK_DESTROY(adapter);
 652         return (0);
 653 }
 654
 655 /*********************************************************************
 656  *
 657  *  Shutdown entry point
 658  *
 659  **********************************************************************/
 660
 661 static int
 662 ixgbe_shutdown(device_t dev)
 663 {
 664         struct adapter *adapter = device_get_softc(dev);
 665         IXGBE_CORE_LOCK(adapter);
 666         ixgbe_stop(adapter);
 667         IXGBE_CORE_UNLOCK(adapter);
 668         return (0);
 669 }
 670
 671
 672 /*********************************************************************
 673  *  Transmit entry point
 674  *
 675  *  ixgbe_start is called by the stack to initiate a transmit.
 676  *  The driver will remain in this routine as long as there are
 677  *  packets to transmit and transmit resources are available.
 678  *  In case resources are not available stack is notified and
 679  *  the packet is requeued.
 680  **********************************************************************/
 681
 682 static void
 683 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 684 {
 685         struct mbuf    *m_head;
 686         struct adapter *adapter = txr->adapter;
 687
 688         IXGBE_TX_LOCK_ASSERT(txr);
 689
 690         if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
 691                 return;
 692
 693         if (!adapter->link_active) {
 694                 ifq_purge(&ifp->if_snd);
 695                 return;
 696         }
 697
 698         while (!ifq_is_empty(&ifp->if_snd)) {
 699                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
 700                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 701                         break;
 702                 }
 703
 704                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
 705                 if (m_head == NULL)
 706                         break;
 707
 708                 if (ixgbe_xmit(txr, &m_head)) {
 709 #if 0 /* XXX: prepend to an ALTQ queue ? */
 710                         if (m_head != NULL)
 711                                 IF_PREPEND(&ifp->if_snd, m_head);
 712 #endif
 713                         if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
 714                                 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 715                         break;
 716                 }
 717                 /* Send a copy of the frame to the BPF listener */
 718                 ETHER_BPF_MTAP(ifp, m_head);
 719
 720                 /* Set watchdog on */
 721                 txr->watchdog_time = ticks;
 722                 txr->queue_status = IXGBE_QUEUE_WORKING;
 723
 724         }
 725         return;
 726 }
 727
 728 /*
 729  * Legacy TX start - called by the stack, this
 730  * always uses the first tx ring, and should
 731  * not be used with multiqueue tx enabled.
 732  */
 733 static void
 734 ixgbe_start(struct ifnet *ifp)
 735 {
 736         struct adapter *adapter = ifp->if_softc;
 737         struct tx_ring  *txr = adapter->tx_rings;
 738
 739         if (ifp->if_flags & IFF_RUNNING) {
 740                 IXGBE_TX_LOCK(txr);
 741                 ixgbe_start_locked(txr, ifp);
 742                 IXGBE_TX_UNLOCK(txr);
 743         }
 744         return;
 745 }
 746
 747 #if 0 /* __FreeBSD_version >= 800000 */
 748 /*
 749 ** Multiqueue Transmit driver
 750 **
 751 */
 752 static int
 753 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
 754 {
 755         struct adapter  *adapter = ifp->if_softc;
 756         struct ix_queue *que;
 757         struct tx_ring  *txr;
 758         int             i = 0, err = 0;
 759
 760         /* Which queue to use */
 761         if ((m->m_flags & M_FLOWID) != 0)
 762                 i = m->m_pkthdr.flowid % adapter->num_queues;
 763         else
 764                 i = curcpu % adapter->num_queues;
 765
 766         txr = &adapter->tx_rings[i];
 767         que = &adapter->queues[i];
 768
 769         if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
 770             IXGBE_TX_TRYLOCK(txr)) {
 771                 err = ixgbe_mq_start_locked(ifp, txr, m);
 772                 IXGBE_TX_UNLOCK(txr);
 773         } else {
 774                 err = drbr_enqueue(ifp, txr->br, m);
 775                 taskqueue_enqueue(que->tq, &que->que_task);
 776         }
 777
 778         return (err);
 779 }
 780
 781 static int
 782 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 783 {
 784         struct adapter  *adapter = txr->adapter;
 785         struct mbuf     *next;
 786         int             enqueued, err = 0;
 787
 788         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
 789             (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
 790             adapter->link_active == 0) {
 791                 if (m != NULL)
 792                         err = drbr_enqueue(ifp, txr->br, m);
 793                 return (err);
 794         }
 795
 796         enqueued = 0;
 797         if (m == NULL) {
 798                 next = drbr_dequeue(ifp, txr->br);
 799         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 800                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 801                         return (err);
 802                 next = drbr_dequeue(ifp, txr->br);
 803         } else
 804                 next = m;
 805
 806         /* Process the queue */
 807         while (next != NULL) {
 808                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 809                         if (next != NULL)
 810                                 err = drbr_enqueue(ifp, txr->br, next);
 811                         break;
 812                 }
 813                 enqueued++;
 814                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 815                 /* Send a copy of the frame to the BPF listener */
 816                 ETHER_BPF_MTAP(ifp, next);
 817                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 818                         break;
 819                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
 820                         ixgbe_txeof(txr);
 821                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
 822                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 823                         break;
 824                 }
 825                 next = drbr_dequeue(ifp, txr->br);
 826         }
 827
 828         if (enqueued > 0) {
 829                 /* Set watchdog on */
 830                 txr->queue_status |= IXGBE_QUEUE_WORKING;
 831                 txr->watchdog_time = ticks;
 832         }
 833
 834         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
 835                 ixgbe_txeof(txr);
 836
 837         return (err);
 838 }
 839
 840 /*
 841 ** Flush all ring buffers
 842 */
 843 static void
 844 ixgbe_qflush(struct ifnet *ifp)
 845 {
 846         struct adapter  *adapter = ifp->if_softc;
 847         struct tx_ring  *txr = adapter->tx_rings;
 848         struct mbuf     *m;
 849
 850         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 851                 IXGBE_TX_LOCK(txr);
 852                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 853                         m_freem(m);
 854                 IXGBE_TX_UNLOCK(txr);
 855         }
 856         if_qflush(ifp);
 857 }
 858 #endif /* __FreeBSD_version >= 800000 */
 859
 860 /*********************************************************************
 861  *  Ioctl entry point
 862  *
 863  *  ixgbe_ioctl is called when the user wants to configure the
 864  *  interface.
 865  *
 866  *  return 0 on success, positive on failure
 867  **********************************************************************/
 868
 869 static int
 870 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
 871 {
 872         struct adapter  *adapter = ifp->if_softc;
 873         struct ifreq    *ifr = (struct ifreq *) data;
 874 #if defined(INET) || defined(INET6)
 875         struct ifaddr *ifa = (struct ifaddr *)data;
 876         bool            avoid_reset = FALSE;
 877 #endif
 878         int             error = 0;
 879
 880         switch (command) {
 881
 882         case SIOCSIFADDR:
 883 #ifdef INET
 884                 if (ifa->ifa_addr->sa_family == AF_INET)
 885                         avoid_reset = TRUE;
 886 #endif
 887 #ifdef INET6
 888                 if (ifa->ifa_addr->sa_family == AF_INET6)
 889                         avoid_reset = TRUE;
 890 #endif
 891 #if defined(INET) || defined(INET6)
 892                 /*
 893                 ** Calling init results in link renegotiation,
 894                 ** so we avoid doing it when possible.
 895                 */
 896                 if (avoid_reset) {
 897                         ifp->if_flags |= IFF_UP;
 898                         if (!(ifp->if_flags & IFF_RUNNING))
 899                                 ixgbe_init(adapter);
 900                         if (!(ifp->if_flags & IFF_NOARP))
 901                                 arp_ifinit(ifp, ifa);
 902                 } else
 903                         error = ether_ioctl(ifp, command, data);
 904 #endif
 905                 break;
 906         case SIOCSIFMTU:
 907                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 908                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 909                         error = EINVAL;
 910                 } else {
 911                         IXGBE_CORE_LOCK(adapter);
 912                         ifp->if_mtu = ifr->ifr_mtu;
 913                         adapter->max_frame_size =
 914                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 915                         ixgbe_init_locked(adapter);
 916                         IXGBE_CORE_UNLOCK(adapter);
 917                 }
 918                 break;
 919         case SIOCSIFFLAGS:
 920                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 921                 IXGBE_CORE_LOCK(adapter);
 922                 if (ifp->if_flags & IFF_UP) {
 923                         if ((ifp->if_flags & IFF_RUNNING)) {
 924                                 if ((ifp->if_flags ^ adapter->if_flags) &
 925                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 926                                         ixgbe_set_promisc(adapter);
 927                                 }
 928                         } else
 929                                 ixgbe_init_locked(adapter);
 930                 } else
 931                         if (ifp->if_flags & IFF_RUNNING)
 932                                 ixgbe_stop(adapter);
 933                 adapter->if_flags = ifp->if_flags;
 934                 IXGBE_CORE_UNLOCK(adapter);
 935                 break;
 936         case SIOCADDMULTI:
 937         case SIOCDELMULTI:
 938                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 939                 if (ifp->if_flags & IFF_RUNNING) {
 940                         IXGBE_CORE_LOCK(adapter);
 941                         ixgbe_disable_intr(adapter);
 942                         ixgbe_set_multi(adapter);
 943                         ixgbe_enable_intr(adapter);
 944                         IXGBE_CORE_UNLOCK(adapter);
 945                 }
 946                 break;
 947         case SIOCSIFMEDIA:
 948         case SIOCGIFMEDIA:
 949                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
 950                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 951                 break;
 952         case SIOCSIFCAP:
 953         {
 954                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 955                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
 956                 if (mask & IFCAP_HWCSUM)
 957                         ifp->if_capenable ^= IFCAP_HWCSUM;
 958                 if (mask & IFCAP_TSO4)
 959                         ifp->if_capenable ^= IFCAP_TSO4;
 960                 if (mask & IFCAP_TSO6)
 961                         ifp->if_capenable ^= IFCAP_TSO6;
 962 #if 0 /* NET_LRO */
 963                 if (mask & IFCAP_LRO)
 964                         ifp->if_capenable ^= IFCAP_LRO;
 965 #endif
 966                 if (mask & IFCAP_VLAN_HWTAGGING)
 967                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 968                 if (mask & IFCAP_VLAN_HWFILTER)
 969                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
 970 #if 0 /* NET_TSO */
 971                 if (mask & IFCAP_VLAN_HWTSO)
 972                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
 973 #endif
 974                 if (ifp->if_flags & IFF_RUNNING) {
 975                         IXGBE_CORE_LOCK(adapter);
 976                         ixgbe_init_locked(adapter);
 977                         IXGBE_CORE_UNLOCK(adapter);
 978                 }
 979 #if 0
 980                 VLAN_CAPABILITIES(ifp);
 981 #endif
 982                 break;
 983         }
 984
 985         default:
 986                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
 987                 error = ether_ioctl(ifp, command, data);
 988                 break;
 989         }
 990
 991         return (error);
 992 }
 993
 994 /*********************************************************************
 995  *  Init entry point
 996  *
 997  *  This routine is used in two ways. It is used by the stack as
 998  *  init entry point in network interface structure. It is also used
 999  *  by the driver as a hw/sw initialization routine to get to a
1000  *  consistent state.
1001  *
1002  *  return 0 on success, positive on failure
1003  **********************************************************************/
1004 #define IXGBE_MHADD_MFS_SHIFT 16
1005
1006 static void
1007 ixgbe_init_locked(struct adapter *adapter)
1008 {
1009         struct ifnet   *ifp = adapter->ifp;
1010         device_t        dev = adapter->dev;
1011         struct ixgbe_hw *hw = &adapter->hw;
1012         u32             k, txdctl, mhadd, gpie;
1013         u32             rxdctl, rxctrl;
1014
1015         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1016         INIT_DEBUGOUT("ixgbe_init: begin");
1017         hw->adapter_stopped = FALSE;
1018         ixgbe_stop_adapter(hw);
1019         callout_stop(&adapter->timer);
1020
1021         /* reprogram the RAR[0] in case user changed it. */
1022         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1023
1024         /* Get the latest mac address, User can use a LAA */
1025         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1026               IXGBE_ETH_LENGTH_OF_ADDRESS);
1027         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1028         hw->addr_ctrl.rar_used_count = 1;
1029
1030         /* Set the various hardware offload abilities */
1031         ifp->if_hwassist = 0;
1032         if (ifp->if_capenable & IFCAP_TSO)
1033                 ifp->if_hwassist |= CSUM_TSO;
1034         if (ifp->if_capenable & IFCAP_TXCSUM) {
1035                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1036 #if 0
1037                 if (hw->mac.type != ixgbe_mac_82598EB)
1038                         ifp->if_hwassist |= CSUM_SCTP;
1039 #endif
1040         }
1041
1042         /* Prepare transmit descriptors and buffers */
1043         if (ixgbe_setup_transmit_structures(adapter)) {
1044                 device_printf(dev,"Could not setup transmit structures\n");
1045                 ixgbe_stop(adapter);
1046                 return;
1047         }
1048
1049         ixgbe_init_hw(hw);
1050         ixgbe_initialize_transmit_units(adapter);
1051
1052         /* Setup Multicast table */
1053         ixgbe_set_multi(adapter);
1054
1055         /*
1056         ** Determine the correct mbuf pool
1057         ** for doing jumbo/headersplit
1058         */
1059         if (adapter->max_frame_size <= 2048)
1060                 adapter->rx_mbuf_sz = MCLBYTES;
1061         else if (adapter->max_frame_size <= 4096)
1062                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1063         else if (adapter->max_frame_size <= 9216)
1064                 adapter->rx_mbuf_sz = MJUM9BYTES;
1065         else
1066                 adapter->rx_mbuf_sz = MJUM16BYTES;
1067
1068         /* Prepare receive descriptors and buffers */
1069         if (ixgbe_setup_receive_structures(adapter)) {
1070                 device_printf(dev,"Could not setup receive structures\n");
1071                 ixgbe_stop(adapter);
1072                 return;
1073         }
1074
1075         /* Configure RX settings */
1076         ixgbe_initialize_receive_units(adapter);
1077
1078         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1079
1080         /* Enable Fan Failure Interrupt */
1081         gpie |= IXGBE_SDP1_GPIEN;
1082
1083         /* Add for Module detection */
1084         if (hw->mac.type == ixgbe_mac_82599EB)
1085                 gpie |= IXGBE_SDP2_GPIEN;
1086
1087         /* Thermal Failure Detection */
1088         if (hw->mac.type == ixgbe_mac_X540)
1089                 gpie |= IXGBE_SDP0_GPIEN;
1090
1091         if (adapter->msix > 1) {
1092                 /* Enable Enhanced MSIX mode */
1093                 gpie |= IXGBE_GPIE_MSIX_MODE;
1094                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1095                     IXGBE_GPIE_OCD;
1096         }
1097         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1098
1099         /* Set MTU size */
1100         if (ifp->if_mtu > ETHERMTU) {
1101                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1102                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1103                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1104                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1105         }
1106
1107         /* Now enable all the queues */
1108
1109         for (int i = 0; i < adapter->num_queues; i++) {
1110                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1111                 txdctl |= IXGBE_TXDCTL_ENABLE;
1112                 /* Set WTHRESH to 8, burst writeback */
1113                 txdctl |= (8 << 16);
1114                 /*
1115                  * When the internal queue falls below PTHRESH (32),
1116                  * start prefetching as long as there are at least
1117                  * HTHRESH (1) buffers ready. The values are taken
1118                  * from the Intel linux driver 3.8.21.
1119                  * Prefetching enables tx line rate even with 1 queue.
1120                  */
1121                 txdctl |= (32 << 0) | (1 << 8);
1122                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1123         }
1124
1125         for (int i = 0; i < adapter->num_queues; i++) {
1126                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1127                 if (hw->mac.type == ixgbe_mac_82598EB) {
1128                         /*
1129                         ** PTHRESH = 21
1130                         ** HTHRESH = 4
1131                         ** WTHRESH = 8
1132                         */
1133                         rxdctl &= ~0x3FFFFF;
1134                         rxdctl |= 0x080420;
1135                 }
1136                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1137                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1138                 for (k = 0; k < 10; k++) {
1139                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1140                             IXGBE_RXDCTL_ENABLE)
1141                                 break;
1142                         else
1143                                 msec_delay(1);
1144                 }
1145                 wmb();
1146 #ifdef DEV_NETMAP
1147                 /*
1148                  * In netmap mode, we must preserve the buffers made
1149                  * available to userspace before the if_init()
1150                  * (this is true by default on the TX side, because
1151                  * init makes all buffers available to userspace).
1152                  *
1153                  * netmap_reset() and the device specific routines
1154                  * (e.g. ixgbe_setup_receive_rings()) map these
1155                  * buffers at the end of the NIC ring, so here we
1156                  * must set the RDT (tail) register to make sure
1157                  * they are not overwritten.
1158                  *
1159                  * In this driver the NIC ring starts at RDH = 0,
1160                  * RDT points to the last slot available for reception (?),
1161                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1162                  */
1163                 if (ifp->if_capenable & IFCAP_NETMAP) {
1164                         struct netmap_adapter *na = NA(adapter->ifp);
1165                         struct netmap_kring *kring = &na->rx_rings[i];
1166                         int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1167
1168                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1169                 } else
1170 #endif /* DEV_NETMAP */
1171                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1172         }
1173
1174         /* Set up VLAN support and filter */
1175         ixgbe_setup_vlan_hw_support(adapter);
1176
1177         /* Enable Receive engine */
1178         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1179         if (hw->mac.type == ixgbe_mac_82598EB)
1180                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1181         rxctrl |= IXGBE_RXCTRL_RXEN;
1182         ixgbe_enable_rx_dma(hw, rxctrl);
1183
1184         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1185
1186         /* Set up MSI/X routing */
1187         if (ixgbe_enable_msix)  {
1188                 ixgbe_configure_ivars(adapter);
1189                 /* Set up auto-mask */
1190                 if (hw->mac.type == ixgbe_mac_82598EB)
1191                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1192                 else {
1193                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1194                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1195                 }
1196         } else {  /* Simple settings for Legacy/MSI */
1197                 ixgbe_set_ivar(adapter, 0, 0, 0);
1198                 ixgbe_set_ivar(adapter, 0, 0, 1);
1199                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1200         }
1201
1202 #ifdef IXGBE_FDIR
1203         /* Init Flow director */
1204         if (hw->mac.type != ixgbe_mac_82598EB) {
1205                 u32 hdrm = 32 << fdir_pballoc;
1206
1207                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1208                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1209         }
1210 #endif
1211
1212         /*
1213         ** Check on any SFP devices that
1214         ** need to be kick-started
1215         */
1216         if (hw->phy.type == ixgbe_phy_none) {
1217                 int err = hw->phy.ops.identify(hw);
1218                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1219                         device_printf(dev,
1220                             "Unsupported SFP+ module type was detected.\n");
1221                         return;
1222                 }
1223         }
1224
1225         /* Set moderation on the Link interrupt */
1226         ixgbe_set_eitr(adapter, adapter->linkvec, IXGBE_LINK_ITR);
1227
1228         /* Config/Enable Link */
1229         ixgbe_config_link(adapter);
1230
1231         /* Hardware Packet Buffer & Flow Control setup */
1232         {
1233                 u32 rxpb, frame, size, tmp;
1234
1235                 frame = adapter->max_frame_size;
1236
1237                 /* Calculate High Water */
1238                 if (hw->mac.type == ixgbe_mac_X540)
1239                         tmp = IXGBE_DV_X540(frame, frame);
1240                 else
1241                         tmp = IXGBE_DV(frame, frame);
1242                 size = IXGBE_BT2KB(tmp);
1243                 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1244                 hw->fc.high_water[0] = rxpb - size;
1245
1246                 /* Now calculate Low Water */
1247                 if (hw->mac.type == ixgbe_mac_X540)
1248                         tmp = IXGBE_LOW_DV_X540(frame);
1249                 else
1250                         tmp = IXGBE_LOW_DV(frame);
1251                 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1252
1253                 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1254                 hw->fc.pause_time = IXGBE_FC_PAUSE;
1255                 hw->fc.send_xon = TRUE;
1256         }
1257         /* Initialize the FC settings */
1258         ixgbe_start_hw(hw);
1259
1260         /* And now turn on interrupts */
1261         ixgbe_enable_intr(adapter);
1262
1263         /* Now inform the stack we're ready */
1264         ifp->if_flags |= IFF_RUNNING;
1265         ifp->if_flags &= ~IFF_OACTIVE;
1266
1267         return;
1268 }
1269
1270 static void
1271 ixgbe_init(void *arg)
1272 {
1273         struct adapter *adapter = arg;
1274
1275         IXGBE_CORE_LOCK(adapter);
1276         ixgbe_init_locked(adapter);
1277         IXGBE_CORE_UNLOCK(adapter);
1278         return;
1279 }
1280
1281
1282 /*
1283 **
1284 ** MSIX Interrupt Handlers and Tasklets
1285 **
1286 */
1287
1288 static inline void
1289 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1290 {
1291         struct ixgbe_hw *hw = &adapter->hw;
1292         u64     queue = (u64)(1 << vector);
1293         u32     mask;
1294
1295         if (hw->mac.type == ixgbe_mac_82598EB) {
1296                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1297                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1298         } else {
1299                 mask = (queue & 0xFFFFFFFF);
1300                 if (mask)
1301                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1302                 mask = (queue >> 32);
1303                 if (mask)
1304                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1305         }
1306 }
1307
1308 static inline void
1309 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1310 {
1311         struct ixgbe_hw *hw = &adapter->hw;
1312         u64     queue = (u64)(1 << vector);
1313         u32     mask;
1314
1315         if (hw->mac.type == ixgbe_mac_82598EB) {
1316                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1317                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1318         } else {
1319                 mask = (queue & 0xFFFFFFFF);
1320                 if (mask)
1321                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1322                 mask = (queue >> 32);
1323                 if (mask)
1324                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1325         }
1326 }
1327
1328 static inline void
1329 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1330 {
1331         u32 mask;
1332
1333         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1334                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1335                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1336         } else {
1337                 mask = (queues & 0xFFFFFFFF);
1338                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1339                 mask = (queues >> 32);
1340                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1341         }
1342 }
1343
1344 /*********************************************************************
1345  *
1346  *  Legacy Interrupt Service routine
1347  *
1348  **********************************************************************/
1349
1350 static void
1351 ixgbe_legacy_irq(void *arg)
1352 {
1353         struct ix_queue *que = arg;
1354         struct adapter  *adapter = que->adapter;
1355         struct ixgbe_hw *hw = &adapter->hw;
1356         struct          tx_ring *txr = adapter->tx_rings;
1357         u32             reg_eicr;
1358
1359
1360         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1361
1362         ++que->irqs;
1363         if (reg_eicr == 0) {
1364                 ixgbe_enable_intr(adapter);
1365                 return;
1366         }
1367
1368         ixgbe_rxeof(que, adapter->rx_process_limit);
1369
1370         IXGBE_TX_LOCK(txr);
1371         ixgbe_txeof(txr);
1372         if (!ifq_is_empty(&adapter->ifp->if_snd))
1373                 ixgbe_start_locked(txr, adapter->ifp);
1374         IXGBE_TX_UNLOCK(txr);
1375
1376         /* Check for fan failure */
1377         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1378             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1379                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1380                     "REPLACE IMMEDIATELY!!\n");
1381                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1382         }
1383
1384         /* Link status change */
1385         if (reg_eicr & IXGBE_EICR_LSC) {
1386                 ixgbe_check_link(&adapter->hw,
1387                     &adapter->link_speed, &adapter->link_up, 0);
1388                 ixgbe_update_link_status(adapter);
1389         }
1390
1391         ixgbe_enable_intr(adapter);
1392 }
1393
1394
1395 /*********************************************************************
1396  *
1397  *  MSIX Queue Interrupt Service routine
1398  *
1399  **********************************************************************/
1400 void
1401 ixgbe_msix_que(void *arg)
1402 {
1403         struct ix_queue *que = arg;
1404         struct adapter  *adapter = que->adapter;
1405         struct tx_ring  *txr = que->txr;
1406
1407         ixgbe_disable_queue(adapter, que->msix);
1408         ++que->irqs;
1409
1410         ixgbe_rxeof(que, adapter->rx_process_limit);
1411
1412         IXGBE_TX_LOCK(txr);
1413         ixgbe_txeof(txr);
1414         if (!ifq_is_empty(&adapter->ifp->if_snd))
1415                 ixgbe_start_locked(txr, adapter->ifp);
1416         IXGBE_TX_UNLOCK(txr);
1417
1418         /* Reenable this interrupt */
1419         ixgbe_enable_queue(adapter, que->msix);
1420 }
1421
1422
1423 static void
1424 ixgbe_msix_link(void *arg)
1425 {
1426         struct adapter  *adapter = arg;
1427         struct ixgbe_hw *hw = &adapter->hw;
1428         u32             reg_eicr;
1429
1430         ++adapter->link_irq;
1431
1432         /* First get the cause */
1433         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1434         /* Clear interrupt with write */
1435         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1436
1437         /* Link status change */
1438         if (reg_eicr & IXGBE_EICR_LSC) {
1439                 ixgbe_check_link(&adapter->hw,
1440                     &adapter->link_speed, &adapter->link_up, 0);
1441                 ixgbe_update_link_status(adapter);
1442         }
1443
1444         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1445 #ifdef IXGBE_FDIR
1446                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1447                         /* This is probably overkill :) */
1448                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1449                                 return;
1450                         /* Disable the interrupt */
1451                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1452                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1453                 } else
1454 #endif
1455                 if (reg_eicr & IXGBE_EICR_ECC) {
1456                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1457                             "Please Reboot!!\n");
1458                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1459                 } else
1460
1461                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1462                         /* Clear the interrupt */
1463                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1464                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1465                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1466                         /* Clear the interrupt */
1467                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1468                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1469                 }
1470         }
1471
1472         /* Check for fan failure */
1473         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1474             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1475                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1476                     "REPLACE IMMEDIATELY!!\n");
1477                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1478         }
1479
1480         /* Check for over temp condition */
1481         if ((hw->mac.type == ixgbe_mac_X540) &&
1482             (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1483                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1484                     "PHY IS SHUT DOWN!!\n");
1485                 device_printf(adapter->dev, "System shutdown required\n");
1486                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1487         }
1488
1489         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1490         return;
1491 }
1492
1493 /*********************************************************************
1494  *
1495  *  Media Ioctl callback
1496  *
1497  *  This routine is called whenever the user queries the status of
1498  *  the interface using ifconfig.
1499  *
1500  **********************************************************************/
1501 static void
1502 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1503 {
1504         struct adapter *adapter = ifp->if_softc;
1505
1506         ASSERT_IFNET_SERIALIZED_ALL(ifp);
1507
1508         INIT_DEBUGOUT("ixgbe_media_status: begin");
1509         ixgbe_update_link_status(adapter);
1510
1511         ifmr->ifm_status = IFM_AVALID;
1512         ifmr->ifm_active = IFM_ETHER;
1513
1514         if (!adapter->link_active)
1515                 return;
1516
1517         ifmr->ifm_status |= IFM_ACTIVE;
1518
1519         switch (adapter->link_speed) {
1520                 case IXGBE_LINK_SPEED_100_FULL:
1521                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1522                         break;
1523                 case IXGBE_LINK_SPEED_1GB_FULL:
1524                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1525                         break;
1526                 case IXGBE_LINK_SPEED_10GB_FULL:
1527                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1528                         break;
1529         }
1530
1531         return;
1532 }
1533
1534 /*********************************************************************
1535  *
1536  *  Media Ioctl callback
1537  *
1538  *  This routine is called when the user changes speed/duplex using
1539  *  media/mediopt option with ifconfig.
1540  *
1541  **********************************************************************/
1542 static int
1543 ixgbe_media_change(struct ifnet * ifp)
1544 {
1545         struct adapter *adapter = ifp->if_softc;
1546         struct ifmedia *ifm = &adapter->media;
1547
1548         INIT_DEBUGOUT("ixgbe_media_change: begin");
1549
1550         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1551                 return (EINVAL);
1552
1553         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1554         case IFM_AUTO:
1555                 adapter->hw.phy.autoneg_advertised =
1556                     IXGBE_LINK_SPEED_100_FULL |
1557                     IXGBE_LINK_SPEED_1GB_FULL |
1558                     IXGBE_LINK_SPEED_10GB_FULL;
1559                 break;
1560         default:
1561                 device_printf(adapter->dev, "Only auto media type\n");
1562                 return (EINVAL);
1563         }
1564
1565         return (0);
1566 }
1567
1568 /*********************************************************************
1569  *
1570  *  This routine maps the mbufs to tx descriptors, allowing the
1571  *  TX engine to transmit the packets.
1572  *      - return 0 on success, positive on failure
1573  *
1574  **********************************************************************/
1575
1576 static int
1577 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1578 {
1579         struct adapter  *adapter = txr->adapter;
1580         u32             olinfo_status = 0, cmd_type_len;
1581         u32             paylen = 0;
1582         int             i, j, error, nsegs, maxsegs;
1583         int             first, last = 0;
1584         struct mbuf     *m_head;
1585         bus_dma_segment_t segs[adapter->num_segs];
1586         bus_dmamap_t    map;
1587         struct ixgbe_tx_buf *txbuf;
1588         union ixgbe_adv_tx_desc *txd = NULL;
1589
1590         m_head = *m_headp;
1591
1592         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1593                 error = ixgbe_tso_pullup(txr, m_headp);
1594                 if (error)
1595                         return error;
1596                 m_head = *m_headp;
1597         }
1598
1599         /* Basic descriptor defines */
1600         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1601             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1602
1603         if (m_head->m_flags & M_VLANTAG)
1604                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1605
1606         /*
1607          * Important to capture the first descriptor
1608          * used because it will contain the index of
1609          * the one we tell the hardware to report back
1610          */
1611         first = txr->next_avail_desc;
1612         txbuf = &txr->tx_buffers[first];
1613         map = txbuf->map;
1614
1615         /*
1616          * Map the packet for DMA.
1617          */
1618         maxsegs = txr->tx_avail - IXGBE_TX_RESERVED;
1619         if (maxsegs > adapter->num_segs)
1620                 maxsegs = adapter->num_segs;
1621
1622         error = bus_dmamap_load_mbuf_defrag(txr->txtag, map, m_headp,
1623             segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1624         if (error) {
1625                 if (error == ENOBUFS)
1626                         adapter->mbuf_defrag_failed++;
1627                 else
1628                         adapter->no_tx_dma_setup++;
1629
1630                 m_freem(*m_headp);
1631                 *m_headp = NULL;
1632                 return (error);
1633         }
1634
1635         /* Make certain there are enough descriptors */
1636         if (nsegs > txr->tx_avail - 2) {
1637                 txr->no_desc_avail++;
1638                 error = ENOBUFS;
1639                 goto xmit_fail;
1640         }
1641         m_head = *m_headp;
1642
1643         /*
1644         ** Set up the appropriate offload context
1645         ** this becomes the first descriptor of
1646         ** a packet.
1647         */
1648         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1649                 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1650                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1651                         olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1652                         olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1653                         ++adapter->tso_tx;
1654                 } else
1655                         return (ENXIO);
1656         } else if (ixgbe_tx_ctx_setup(txr, m_head))
1657                 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1658
1659 #ifdef IXGBE_IEEE1588
1660         /* This is changing soon to an mtag detection */
1661         if (we detect this mbuf has a TSTAMP mtag)
1662                 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1663 #endif
1664
1665 #ifdef IXGBE_FDIR
1666         /* Do the flow director magic */
1667         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1668                 ++txr->atr_count;
1669                 if (txr->atr_count >= atr_sample_rate) {
1670                         ixgbe_atr(txr, m_head);
1671                         txr->atr_count = 0;
1672                 }
1673         }
1674 #endif
1675         /* Record payload length */
1676         if (paylen == 0)
1677                 olinfo_status |= m_head->m_pkthdr.len <<
1678                     IXGBE_ADVTXD_PAYLEN_SHIFT;
1679
1680         i = txr->next_avail_desc;
1681         for (j = 0; j < nsegs; j++) {
1682                 bus_size_t seglen;
1683                 bus_addr_t segaddr;
1684
1685                 txbuf = &txr->tx_buffers[i];
1686                 txd = &txr->tx_base[i];
1687                 seglen = segs[j].ds_len;
1688                 segaddr = htole64(segs[j].ds_addr);
1689
1690                 txd->read.buffer_addr = segaddr;
1691                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1692                     cmd_type_len |seglen);
1693                 txd->read.olinfo_status = htole32(olinfo_status);
1694                 last = i; /* descriptor that will get completion IRQ */
1695
1696                 if (++i == adapter->num_tx_desc)
1697                         i = 0;
1698
1699                 txbuf->m_head = NULL;
1700                 txbuf->eop_index = -1;
1701         }
1702
1703         txd->read.cmd_type_len |=
1704             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1705         txr->tx_avail -= nsegs;
1706         txr->next_avail_desc = i;
1707
1708         txbuf->m_head = m_head;
1709         /* Swap the dma map between the first and last descriptor */
1710         txr->tx_buffers[first].map = txbuf->map;
1711         txbuf->map = map;
1712         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1713
1714         /* Set the index of the descriptor that will be marked done */
1715         txbuf = &txr->tx_buffers[first];
1716         txbuf->eop_index = last;
1717
1718         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1719             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1720         /*
1721          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1722          * hardware that this frame is available to transmit.
1723          */
1724         ++txr->total_packets;
1725         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1726
1727         return (0);
1728
1729 xmit_fail:
1730         bus_dmamap_unload(txr->txtag, txbuf->map);
1731         return (error);
1732
1733 }
1734
1735 static void
1736 ixgbe_set_promisc(struct adapter *adapter)
1737 {
1738         u_int32_t       reg_rctl;
1739         struct ifnet   *ifp = adapter->ifp;
1740
1741         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1742         reg_rctl &= (~IXGBE_FCTRL_UPE);
1743         reg_rctl &= (~IXGBE_FCTRL_MPE);
1744         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1745
1746         if (ifp->if_flags & IFF_PROMISC) {
1747                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1748                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1749         } else if (ifp->if_flags & IFF_ALLMULTI) {
1750                 reg_rctl |= IXGBE_FCTRL_MPE;
1751                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1752                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1753         }
1754         return;
1755 }
1756
1757
1758 /*********************************************************************
1759  *  Multicast Update
1760  *
1761  *  This routine is called whenever multicast address list is updated.
1762  *
1763  **********************************************************************/
1764 #define IXGBE_RAR_ENTRIES 16
1765
1766 static void
1767 ixgbe_set_multi(struct adapter *adapter)
1768 {
1769         u32     fctrl;
1770         u8      *mta;
1771         u8      *update_ptr;
1772         struct  ifmultiaddr *ifma;
1773         int     mcnt = 0;
1774         struct ifnet   *ifp = adapter->ifp;
1775
1776         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1777
1778         mta = adapter->mta;
1779         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1780             MAX_NUM_MULTICAST_ADDRESSES);
1781
1782         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1783         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1784         if (ifp->if_flags & IFF_PROMISC)
1785                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1786         else if (ifp->if_flags & IFF_ALLMULTI) {
1787                 fctrl |= IXGBE_FCTRL_MPE;
1788                 fctrl &= ~IXGBE_FCTRL_UPE;
1789         } else
1790                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1791
1792         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1793
1794         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1795                 if (ifma->ifma_addr->sa_family != AF_LINK)
1796                         continue;
1797                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1798                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1799                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1800                 mcnt++;
1801         }
1802
1803         update_ptr = mta;
1804         ixgbe_update_mc_addr_list(&adapter->hw,
1805             update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1806
1807         return;
1808 }
1809
1810 /*
1811  * This is an iterator function now needed by the multicast
1812  * shared code. It simply feeds the shared code routine the
1813  * addresses in the array of ixgbe_set_multi() one by one.
1814  */
1815 static u8 *
1816 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1817 {
1818         u8 *addr = *update_ptr;
1819         u8 *newptr;
1820         *vmdq = 0;
1821
1822         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1823         *update_ptr = newptr;
1824         return addr;
1825 }
1826
1827
1828 /*********************************************************************
1829  *  Timer routine
1830  *
1831  *  This routine checks for link status,updates statistics,
1832  *  and runs the watchdog check.
1833  *
1834  **********************************************************************/
1835
1836 static void
1837 ixgbe_local_timer(void *arg)
1838 {
1839         struct adapter  *adapter = arg;
1840         device_t        dev = adapter->dev;
1841         struct ifnet    *ifp = adapter->ifp;
1842         struct ix_queue *que = adapter->queues;
1843         struct tx_ring  *txr = adapter->tx_rings;
1844         int             hung, busy, paused;
1845
1846         IXGBE_CORE_LOCK(adapter);
1847         hung = busy = paused = 0;
1848
1849         /* Check for pluggable optics */
1850         if (adapter->sfp_probe)
1851                 if (!ixgbe_sfp_probe(adapter))
1852                         goto out; /* Nothing to do */
1853
1854         ixgbe_update_link_status(adapter);
1855         ixgbe_update_stats_counters(adapter);
1856
1857         /*
1858          * If the interface has been paused
1859          * then don't do the watchdog check
1860          */
1861         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
1862                 paused = 1;
1863
1864         /*
1865         ** Check the TX queues status
1866         **      - central locked handling of OACTIVE
1867         **      - watchdog only if all queues show hung
1868         */
1869         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
1870                 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
1871                     (paused == 0))
1872                         ++hung;
1873                 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
1874                         ++busy;
1875         }
1876         /* Only truely watchdog if all queues show hung */
1877         if (hung == adapter->num_queues)
1878                 goto watchdog;
1879         /* Only turn off the stack flow when ALL are depleted */
1880         if (busy == adapter->num_queues)
1881                 ifp->if_flags |= IFF_OACTIVE;
1882         else if ((ifp->if_flags & IFF_OACTIVE) &&
1883             (busy < adapter->num_queues))
1884                 ifp->if_flags &= ~IFF_OACTIVE;
1885
1886 out:
1887         ixgbe_rearm_queues(adapter, adapter->que_mask);
1888         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1889         IXGBE_CORE_UNLOCK(adapter);
1890         return;
1891
1892 watchdog:
1893         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
1894         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
1895             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
1896             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
1897         device_printf(dev,"TX(%d) desc avail = %d,"
1898             "Next TX to Clean = %d\n",
1899             txr->me, txr->tx_avail, txr->next_to_clean);
1900         adapter->ifp->if_flags &= ~IFF_RUNNING;
1901         adapter->watchdog_events++;
1902         ixgbe_init_locked(adapter);
1903
1904         IXGBE_CORE_UNLOCK(adapter);
1905 }
1906
1907 /*
1908 ** Note: this routine updates the OS on the link state
1909 **      the real check of the hardware only happens with
1910 **      a link interrupt.
1911 */
1912 static void
1913 ixgbe_update_link_status(struct adapter *adapter)
1914 {
1915         struct ifnet    *ifp = adapter->ifp;
1916         struct tx_ring *txr = adapter->tx_rings;
1917         device_t dev = adapter->dev;
1918
1919
1920         if (adapter->link_up){
1921                 if (adapter->link_active == FALSE) {
1922                         if (bootverbose)
1923                                 device_printf(dev,"Link is up %d Gbps %s \n",
1924                                     ((adapter->link_speed == 128)? 10:1),
1925                                     "Full Duplex");
1926                         adapter->link_active = TRUE;
1927                         /* Update any Flow Control changes */
1928                         ixgbe_fc_enable(&adapter->hw);
1929                         ifp->if_link_state = LINK_STATE_UP;
1930                         if_link_state_change(ifp);
1931                 }
1932         } else { /* Link down */
1933                 if (adapter->link_active == TRUE) {
1934                         if (bootverbose)
1935                                 device_printf(dev,"Link is Down\n");
1936                         ifp->if_link_state = LINK_STATE_DOWN;
1937                         if_link_state_change(ifp);
1938                         adapter->link_active = FALSE;
1939                         for (int i = 0; i < adapter->num_queues;
1940                             i++, txr++)
1941                                 txr->queue_status = IXGBE_QUEUE_IDLE;
1942                 }
1943         }
1944
1945         return;
1946 }
1947
1948
1949 /*********************************************************************
1950  *
1951  *  This routine disables all traffic on the adapter by issuing a
1952  *  global reset on the MAC and deallocates TX/RX buffers.
1953  *
1954  **********************************************************************/
1955
1956 static void
1957 ixgbe_stop(void *arg)
1958 {
1959         struct ifnet   *ifp;
1960         struct adapter *adapter = arg;
1961         struct ixgbe_hw *hw = &adapter->hw;
1962         ifp = adapter->ifp;
1963
1964         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1965
1966         INIT_DEBUGOUT("ixgbe_stop: begin\n");
1967         ixgbe_disable_intr(adapter);
1968         callout_stop(&adapter->timer);
1969
1970         /* Let the stack know...*/
1971         ifp->if_flags &= ~IFF_RUNNING;
1972         ifp->if_flags |= IFF_OACTIVE;
1973
1974         ixgbe_reset_hw(hw);
1975         hw->adapter_stopped = FALSE;
1976         ixgbe_stop_adapter(hw);
1977         /* Turn off the laser */
1978         if (hw->phy.multispeed_fiber)
1979                 ixgbe_disable_tx_laser(hw);
1980
1981         /* reprogram the RAR[0] in case user changed it. */
1982         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1983
1984         return;
1985 }
1986
1987
1988 /*********************************************************************
1989  *
1990  *  Determine hardware revision.
1991  *
1992  **********************************************************************/
1993 static void
1994 ixgbe_identify_hardware(struct adapter *adapter)
1995 {
1996         device_t        dev = adapter->dev;
1997         struct ixgbe_hw *hw = &adapter->hw;
1998
1999         /* Save off the information about this board */
2000         hw->vendor_id = pci_get_vendor(dev);
2001         hw->device_id = pci_get_device(dev);
2002         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2003         hw->subsystem_vendor_id =
2004             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2005         hw->subsystem_device_id =
2006             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2007
2008         /* We need this here to set the num_segs below */
2009         ixgbe_set_mac_type(hw);
2010
2011         /* Pick up the 82599 and VF settings */
2012         if (hw->mac.type != ixgbe_mac_82598EB) {
2013                 hw->phy.smart_speed = ixgbe_smart_speed;
2014                 adapter->num_segs = IXGBE_82599_SCATTER;
2015         } else
2016                 adapter->num_segs = IXGBE_82598_SCATTER;
2017
2018         return;
2019 }
2020
2021 /*********************************************************************
2022  *
2023  *  Determine optic type
2024  *
2025  **********************************************************************/
2026 static void
2027 ixgbe_setup_optics(struct adapter *adapter)
2028 {
2029         struct ixgbe_hw *hw = &adapter->hw;
2030         int             layer;
2031
2032         layer = ixgbe_get_supported_physical_layer(hw);
2033
2034         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2035                 adapter->optics = IFM_10G_T;
2036                 return;
2037         }
2038
2039         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2040                 adapter->optics = IFM_1000_T;
2041                 return;
2042         }
2043
2044         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2045             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2046                 adapter->optics = IFM_10G_LR;
2047                 return;
2048         }
2049
2050         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2051                 adapter->optics = IFM_10G_SR;
2052                 return;
2053         }
2054
2055         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2056                 adapter->optics = IFM_10G_TWINAX;
2057                 return;
2058         }
2059
2060         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2061             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2062                 adapter->optics = IFM_10G_CX4;
2063                 return;
2064         }
2065
2066         /* If we get here just set the default */
2067         adapter->optics = IFM_ETHER | IFM_AUTO;
2068         return;
2069 }
2070
2071 /*********************************************************************
2072  *
2073  *  Setup the Legacy or MSI Interrupt handler
2074  *
2075  **********************************************************************/
2076 static int
2077 ixgbe_allocate_legacy(struct adapter *adapter)
2078 {
2079         device_t dev = adapter->dev;
2080         struct          ix_queue *que = adapter->queues;
2081         int error, rid = 0;
2082         unsigned int intr_flags;
2083
2084         /* MSI RID at 1 */
2085         if (adapter->msix == 1)
2086                 rid = 1;
2087
2088         /* Try allocating a MSI interrupt first */
2089         adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2090                 &rid, &intr_flags);
2091
2092         /* We allocate a single interrupt resource */
2093         adapter->res = bus_alloc_resource_any(dev,
2094             SYS_RES_IRQ, &rid, intr_flags);
2095         if (adapter->res == NULL) {
2096                 device_printf(dev, "Unable to allocate bus resource: "
2097                     "interrupt\n");
2098                 return (ENXIO);
2099         }
2100
2101         /* Tasklets for Link, SFP and Multispeed Fiber */
2102         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2103         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2104 #ifdef IXGBE_FDIR
2105         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2106 #endif
2107
2108         if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2109             ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2110                 device_printf(dev, "Failed to register fast interrupt "
2111                     "handler: %d\n", error);
2112                 taskqueue_free(adapter->tq);
2113                 adapter->tq = NULL;
2114                 return (error);
2115         }
2116         /* For simplicity in the handlers */
2117         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2118
2119         return (0);
2120 }
2121
2122
2123 /*********************************************************************
2124  *
2125  *  Setup MSIX Interrupt resources and handlers
2126  *
2127  **********************************************************************/
2128 static int
2129 ixgbe_allocate_msix(struct adapter *adapter)
2130 {
2131         device_t        dev = adapter->dev;
2132         struct          ix_queue *que = adapter->queues;
2133         int             error, rid, vector = 0;
2134         char            desc[16];
2135
2136         error = pci_setup_msix(dev);
2137         if (error) {
2138                 device_printf(dev, "MSI-X setup failed\n");
2139                 return (error);
2140         }
2141
2142         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2143                 rid = vector + 1;
2144
2145                 /*
2146                 ** Bind the msix vector, and thus the
2147                 ** ring to the corresponding cpu.
2148                 */
2149                 error = pci_alloc_msix_vector(dev, vector, &rid, i);
2150                 if (error) {
2151                         device_printf(dev, "pci_alloc_msix_vector failed\n");
2152                         return (error);
2153                 }
2154
2155                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2156                     RF_SHAREABLE | RF_ACTIVE);
2157                 if (que->res == NULL) {
2158                         device_printf(dev,"Unable to allocate"
2159                             " bus resource: que interrupt [%d]\n", vector);
2160                         return (ENXIO);
2161                 }
2162                 /* Set the handler function */
2163                 ksnprintf(desc, sizeof(desc), "%s que %d",
2164                     device_get_nameunit(dev), i);
2165                 error = bus_setup_intr_descr(dev, que->res, INTR_MPSAFE,
2166                     ixgbe_msix_que, que, &que->tag, &que->serializer, desc);
2167                 if (error) {
2168                         que->res = NULL;
2169                         device_printf(dev, "Failed to register QUE handler");
2170                         return (error);
2171                 }
2172                 que->msix = vector;
2173                 adapter->que_mask |= (u64)(1 << que->msix);
2174         }
2175
2176         /* and Link, bind vector to cpu #0 */
2177         rid = vector + 1;
2178         error = pci_alloc_msix_vector(dev, vector, &rid, 0);
2179         if (error) {
2180                 device_printf(dev, "pci_alloc_msix_vector failed\n");
2181                 return (error);
2182         }
2183         adapter->res = bus_alloc_resource_any(dev,
2184             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2185         if (!adapter->res) {
2186                 device_printf(dev,"Unable to allocate"
2187             " bus resource: Link interrupt [%d]\n", rid);
2188                 return (ENXIO);
2189         }
2190         /* Set the link handler function */
2191         error = bus_setup_intr_descr(dev, adapter->res, INTR_MPSAFE,
2192             ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer,
2193             "link");
2194         if (error) {
2195                 adapter->res = NULL;
2196                 device_printf(dev, "Failed to register LINK handler");
2197                 return (error);
2198         }
2199         pci_enable_msix(dev);
2200
2201         adapter->linkvec = vector;
2202         /* Tasklets for Link, SFP and Multispeed Fiber */
2203         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2204         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2205 #ifdef IXGBE_FDIR
2206         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2207 #endif
2208
2209         return (0);
2210 }
2211
2212 /*
2213  * Setup Either MSI/X or MSI
2214  */
2215 static int
2216 ixgbe_setup_msix(struct adapter *adapter)
2217 {
2218         device_t dev = adapter->dev;
2219         int rid, want, queues, msgs;
2220
2221         /* Override by tuneable */
2222         if (ixgbe_enable_msix == 0)
2223                 goto msi;
2224
2225         /* First try MSI/X */
2226         rid = PCIR_BAR(MSIX_82598_BAR);
2227         adapter->msix_mem = bus_alloc_resource_any(dev,
2228             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2229         if (!adapter->msix_mem) {
2230                 rid += 4;       /* 82599 maps in higher BAR */
2231                 adapter->msix_mem = bus_alloc_resource_any(dev,
2232                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2233         }
2234         if (!adapter->msix_mem) {
2235                 /* May not be enabled */
2236                 device_printf(adapter->dev,
2237                     "Unable to map MSIX table \n");
2238                 goto msi;
2239         }
2240
2241         msgs = pci_msix_count(dev);
2242         if (msgs == 0) { /* system has msix disabled */
2243                 bus_release_resource(dev, SYS_RES_MEMORY,
2244                     rid, adapter->msix_mem);
2245                 adapter->msix_mem = NULL;
2246                 goto msi;
2247         }
2248
2249         /* Figure out a reasonable auto config value */
2250         queues = (ncpus > (msgs-1)) ? (msgs-1) : ncpus;
2251
2252         if (ixgbe_num_queues != 0)
2253                 queues = ixgbe_num_queues;
2254         /* Set max queues to 8 when autoconfiguring */
2255         else if ((ixgbe_num_queues == 0) && (queues > 8))
2256                 queues = 8;
2257
2258         /*
2259         ** Want one vector (RX/TX pair) per queue
2260         ** plus an additional for Link.
2261         */
2262         want = queues + 1;
2263         if (msgs >= want)
2264                 msgs = want;
2265         else {
2266                 device_printf(adapter->dev,
2267                     "MSIX Configuration Problem, "
2268                     "%d vectors but %d queues wanted!\n",
2269                     msgs, want);
2270                 return (0); /* Will go to Legacy setup */
2271         }
2272         if (msgs) {
2273                 device_printf(adapter->dev,
2274                     "Using MSIX interrupts with %d vectors\n", msgs);
2275                 adapter->num_queues = queues;
2276                 return (msgs);
2277         }
2278 msi:
2279         msgs = pci_msi_count(dev);
2280         return (msgs);
2281 }
2282
2283
2284 static int
2285 ixgbe_allocate_pci_resources(struct adapter *adapter)
2286 {
2287         int             rid;
2288         device_t        dev = adapter->dev;
2289
2290         rid = PCIR_BAR(0);
2291         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2292             &rid, RF_ACTIVE);
2293
2294         if (!(adapter->pci_mem)) {
2295                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2296                 return (ENXIO);
2297         }
2298
2299         adapter->osdep.mem_bus_space_tag =
2300                 rman_get_bustag(adapter->pci_mem);
2301         adapter->osdep.mem_bus_space_handle =
2302                 rman_get_bushandle(adapter->pci_mem);
2303         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2304
2305         /* Legacy defaults */
2306         adapter->num_queues = 1;
2307         adapter->hw.back = &adapter->osdep;
2308
2309         /*
2310         ** Now setup MSI or MSI/X, should
2311         ** return us the number of supported
2312         ** vectors. (Will be 1 for MSI)
2313         */
2314         adapter->msix = ixgbe_setup_msix(adapter);
2315         return (0);
2316 }
2317
2318 static void
2319 ixgbe_free_pci_resources(struct adapter * adapter)
2320 {
2321         struct          ix_queue *que = adapter->queues;
2322         device_t        dev = adapter->dev;
2323         int             rid, memrid;
2324
2325         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2326                 memrid = PCIR_BAR(MSIX_82598_BAR);
2327         else
2328                 memrid = PCIR_BAR(MSIX_82599_BAR);
2329
2330         /*
2331         ** There is a slight possibility of a failure mode
2332         ** in attach that will result in entering this function
2333         ** before interrupt resources have been initialized, and
2334         ** in that case we do not want to execute the loops below
2335         ** We can detect this reliably by the state of the adapter
2336         ** res pointer.
2337         */
2338         if (adapter->res == NULL)
2339                 goto mem;
2340
2341         /*
2342         **  Release all msix queue resources:
2343         */
2344         for (int i = 0; i < adapter->num_queues; i++, que++) {
2345                 rid = que->msix + 1;
2346                 if (que->tag != NULL) {
2347                         bus_teardown_intr(dev, que->res, que->tag);
2348                         que->tag = NULL;
2349                 }
2350                 if (que->res != NULL)
2351                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2352         }
2353
2354
2355         /* Clean the Legacy or Link interrupt last */
2356         if (adapter->linkvec) /* we are doing MSIX */
2357                 rid = adapter->linkvec + 1;
2358         else
2359                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2360
2361         if (adapter->tag != NULL) {
2362                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2363                 adapter->tag = NULL;
2364         }
2365         if (adapter->res != NULL)
2366                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2367         if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2368                 pci_release_msi(adapter->dev);
2369
2370 mem:
2371         if (adapter->msix)
2372                 pci_release_msi(dev);
2373
2374         if (adapter->msix_mem != NULL)
2375                 bus_release_resource(dev, SYS_RES_MEMORY,
2376                     memrid, adapter->msix_mem);
2377
2378         if (adapter->pci_mem != NULL)
2379                 bus_release_resource(dev, SYS_RES_MEMORY,
2380                     PCIR_BAR(0), adapter->pci_mem);
2381
2382         return;
2383 }
2384
2385 /*********************************************************************
2386  *
2387  *  Setup networking device structure and register an interface.
2388  *
2389  **********************************************************************/
2390 static int
2391 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2392 {
2393         struct ixgbe_hw *hw = &adapter->hw;
2394         struct ifnet   *ifp;
2395
2396         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2397
2398         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2399         if (ifp == NULL) {
2400                 device_printf(dev, "can not allocate ifnet structure\n");
2401                 return (-1);
2402         }
2403         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2404         ifp->if_baudrate = 1000000000;
2405         ifp->if_init = ixgbe_init;
2406         ifp->if_softc = adapter;
2407         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2408         ifp->if_ioctl = ixgbe_ioctl;
2409         ifp->if_start = ixgbe_start;
2410 #if 0 /* __FreeBSD_version >= 800000 */
2411         ifp->if_transmit = ixgbe_mq_start;
2412         ifp->if_qflush = ixgbe_qflush;
2413 #endif
2414         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2415
2416         ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2417
2418         adapter->max_frame_size =
2419             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2420
2421         /*
2422          * Tell the upper layer(s) we support long frames.
2423          */
2424         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2425
2426         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2427         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2428         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2429 #if 0 /* NET_TSO */
2430                              |  IFCAP_VLAN_HWTSO
2431 #endif
2432                              |  IFCAP_VLAN_MTU;
2433         ifp->if_capenable = ifp->if_capabilities;
2434
2435         /* Don't enable LRO by default */
2436 #if 0 /* NET_LRO */
2437         ifp->if_capabilities |= IFCAP_LRO;
2438 #endif
2439
2440         /*
2441         ** Don't turn this on by default, if vlans are
2442         ** created on another pseudo device (eg. lagg)
2443         ** then vlan events are not passed thru, breaking
2444         ** operation, but with HW FILTER off it works. If
2445         ** using vlans directly on the ixgbe driver you can
2446         ** enable this and get full hardware tag filtering.
2447         */
2448         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2449
2450         /*
2451          * Specify the media types supported by this adapter and register
2452          * callbacks to update media and link information
2453          */
2454         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2455                      ixgbe_media_status);
2456         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2457         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2458         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2459                 ifmedia_add(&adapter->media,
2460                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2461                 ifmedia_add(&adapter->media,
2462                     IFM_ETHER | IFM_1000_T, 0, NULL);
2463         }
2464         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2465         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2466
2467         return (0);
2468 }
2469
2470 static void
2471 ixgbe_config_link(struct adapter *adapter)
2472 {
2473         struct ixgbe_hw *hw = &adapter->hw;
2474         u32     autoneg, err = 0;
2475         bool    sfp, negotiate;
2476
2477         sfp = ixgbe_is_sfp(hw);
2478
2479         if (sfp) {
2480                 if (hw->phy.multispeed_fiber) {
2481                         hw->mac.ops.setup_sfp(hw);
2482                         ixgbe_enable_tx_laser(hw);
2483                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2484                 } else
2485                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2486         } else {
2487                 if (hw->mac.ops.check_link)
2488                         err = ixgbe_check_link(hw, &autoneg,
2489                             &adapter->link_up, FALSE);
2490                 if (err)
2491                         goto out;
2492                 autoneg = hw->phy.autoneg_advertised;
2493                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2494                         err  = hw->mac.ops.get_link_capabilities(hw,
2495                             &autoneg, &negotiate);
2496                 if (err)
2497                         goto out;
2498                 if (hw->mac.ops.setup_link)
2499                         err = hw->mac.ops.setup_link(hw, autoneg,
2500                             negotiate, adapter->link_up);
2501         }
2502 out:
2503         return;
2504 }
2505
2506 /********************************************************************
2507  * Manage DMA'able memory.
2508  *******************************************************************/
2509 static void
2510 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2511 {
2512         if (error)
2513                 return;
2514         *(bus_addr_t *) arg = segs->ds_addr;
2515         return;
2516 }
2517
2518 static int
2519 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2520                 struct ixgbe_dma_alloc *dma, int mapflags)
2521 {
2522         device_t dev = adapter->dev;
2523         int             r;
2524
2525         r = bus_dma_tag_create(NULL,    /* parent */
2526                                DBA_ALIGN, 0,    /* alignment, bounds */
2527                                BUS_SPACE_MAXADDR,       /* lowaddr */
2528                                BUS_SPACE_MAXADDR,       /* highaddr */
2529                                NULL, NULL,      /* filter, filterarg */
2530                                size,    /* maxsize */
2531                                1,       /* nsegments */
2532                                size,    /* maxsegsize */
2533                                BUS_DMA_ALLOCNOW,        /* flags */
2534                                &dma->dma_tag);
2535         if (r != 0) {
2536                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2537                        "error %u\n", r);
2538                 goto fail_0;
2539         }
2540         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2541                              BUS_DMA_NOWAIT, &dma->dma_map);
2542         if (r != 0) {
2543                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2544                        "error %u\n", r);
2545                 goto fail_1;
2546         }
2547         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2548                             size,
2549                             ixgbe_dmamap_cb,
2550                             &dma->dma_paddr,
2551                             mapflags | BUS_DMA_NOWAIT);
2552         if (r != 0) {
2553                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2554                        "error %u\n", r);
2555                 goto fail_2;
2556         }
2557         dma->dma_size = size;
2558         return (0);
2559 fail_2:
2560         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2561 fail_1:
2562         bus_dma_tag_destroy(dma->dma_tag);
2563 fail_0:
2564         dma->dma_map = NULL;
2565         dma->dma_tag = NULL;
2566         return (r);
2567 }
2568
2569 static void
2570 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2571 {
2572         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2573             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2574         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2575         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2576         bus_dma_tag_destroy(dma->dma_tag);
2577 }
2578
2579
2580 /*********************************************************************
2581  *
2582  *  Allocate memory for the transmit and receive rings, and then
2583  *  the descriptors associated with each, called only once at attach.
2584  *
2585  **********************************************************************/
2586 static int
2587 ixgbe_allocate_queues(struct adapter *adapter)
2588 {
2589         device_t        dev = adapter->dev;
2590         struct ix_queue *que;
2591         struct tx_ring  *txr;
2592         struct rx_ring  *rxr;
2593         int rsize, tsize, error = IXGBE_SUCCESS;
2594         int txconf = 0, rxconf = 0;
2595
2596         /* First allocate the top level queue structs */
2597         if (!(adapter->queues =
2598             (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2599             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2600                 device_printf(dev, "Unable to allocate queue memory\n");
2601                 error = ENOMEM;
2602                 goto fail;
2603         }
2604
2605         /* First allocate the TX ring struct memory */
2606         if (!(adapter->tx_rings =
2607             (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2608             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2609                 device_printf(dev, "Unable to allocate TX ring memory\n");
2610                 error = ENOMEM;
2611                 goto tx_fail;
2612         }
2613
2614         /* Next allocate the RX */
2615         if (!(adapter->rx_rings =
2616             (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2617             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2618                 device_printf(dev, "Unable to allocate RX ring memory\n");
2619                 error = ENOMEM;
2620                 goto rx_fail;
2621         }
2622
2623         /* For the ring itself */
2624         tsize = roundup2(adapter->num_tx_desc *
2625             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2626
2627         /*
2628          * Now set up the TX queues, txconf is needed to handle the
2629          * possibility that things fail midcourse and we need to
2630          * undo memory gracefully
2631          */
2632         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2633                 /* Set up some basics */
2634                 txr = &adapter->tx_rings[i];
2635                 txr->adapter = adapter;
2636                 txr->me = i;
2637
2638                 /* Initialize the TX side lock */
2639                 IXGBE_TX_LOCK_INIT(txr);
2640
2641                 if (ixgbe_dma_malloc(adapter, tsize,
2642                         &txr->txdma, BUS_DMA_NOWAIT)) {
2643                         device_printf(dev,
2644                             "Unable to allocate TX Descriptor memory\n");
2645                         error = ENOMEM;
2646                         goto err_tx_desc;
2647                 }
2648                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2649                 bzero((void *)txr->tx_base, tsize);
2650
2651                 /* Now allocate transmit buffers for the ring */
2652                 if (ixgbe_allocate_transmit_buffers(txr)) {
2653                         device_printf(dev,
2654                             "Critical Failure setting up transmit buffers\n");
2655                         error = ENOMEM;
2656                         goto err_tx_desc;
2657                 }
2658 #if 0 /* __FreeBSD_version >= 800000 */
2659                 /* Allocate a buf ring */
2660                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2661                     M_WAITOK, &txr->tx_mtx);
2662                 if (txr->br == NULL) {
2663                         device_printf(dev,
2664                             "Critical Failure setting up buf ring\n");
2665                         error = ENOMEM;
2666                         goto err_tx_desc;
2667                 }
2668 #endif
2669         }
2670
2671         /*
2672          * Next the RX queues...
2673          */
2674         rsize = roundup2(adapter->num_rx_desc *
2675             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2676         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2677                 rxr = &adapter->rx_rings[i];
2678                 /* Set up some basics */
2679                 rxr->adapter = adapter;
2680                 rxr->me = i;
2681
2682                 /* Initialize the RX side lock */
2683                 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2684                     device_get_nameunit(dev), rxr->me);
2685                 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2686
2687                 if (ixgbe_dma_malloc(adapter, rsize,
2688                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2689                         device_printf(dev,
2690                             "Unable to allocate RxDescriptor memory\n");
2691                         error = ENOMEM;
2692                         goto err_rx_desc;
2693                 }
2694                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2695                 bzero((void *)rxr->rx_base, rsize);
2696
2697                 /* Allocate receive buffers for the ring*/
2698                 if (ixgbe_allocate_receive_buffers(rxr)) {
2699                         device_printf(dev,
2700                             "Critical Failure setting up receive buffers\n");
2701                         error = ENOMEM;
2702                         goto err_rx_desc;
2703                 }
2704         }
2705
2706         /*
2707         ** Finally set up the queue holding structs
2708         */
2709         for (int i = 0; i < adapter->num_queues; i++) {
2710                 que = &adapter->queues[i];
2711                 que->adapter = adapter;
2712                 que->txr = &adapter->tx_rings[i];
2713                 que->rxr = &adapter->rx_rings[i];
2714         }
2715
2716         return (0);
2717
2718 err_rx_desc:
2719         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2720                 ixgbe_dma_free(adapter, &rxr->rxdma);
2721 err_tx_desc:
2722         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2723                 ixgbe_dma_free(adapter, &txr->txdma);
2724         kfree(adapter->rx_rings, M_DEVBUF);
2725 rx_fail:
2726         kfree(adapter->tx_rings, M_DEVBUF);
2727 tx_fail:
2728         kfree(adapter->queues, M_DEVBUF);
2729 fail:
2730         return (error);
2731 }
2732
2733 /*********************************************************************
2734  *
2735  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2736  *  the information needed to transmit a packet on the wire. This is
2737  *  called only once at attach, setup is done every reset.
2738  *
2739  **********************************************************************/
2740 static int
2741 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2742 {
2743         struct adapter *adapter = txr->adapter;
2744         device_t dev = adapter->dev;
2745         struct ixgbe_tx_buf *txbuf;
2746         int error, i;
2747
2748         /*
2749          * Setup DMA descriptor areas.
2750          */
2751         if ((error = bus_dma_tag_create(
2752                                NULL,    /* parent */
2753                                1, 0,            /* alignment, bounds */
2754                                BUS_SPACE_MAXADDR,       /* lowaddr */
2755                                BUS_SPACE_MAXADDR,       /* highaddr */
2756                                NULL, NULL,              /* filter, filterarg */
2757                                IXGBE_TSO_SIZE,          /* maxsize */
2758                                adapter->num_segs,       /* nsegments */
2759                                PAGE_SIZE,               /* maxsegsize */
2760                                0,                       /* flags */
2761                                &txr->txtag))) {
2762                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2763                 goto fail;
2764         }
2765
2766         if (!(txr->tx_buffers =
2767             (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2768             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2769                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2770                 error = ENOMEM;
2771                 goto fail;
2772         }
2773
2774         /* Create the descriptor buffer dma maps */
2775         txbuf = txr->tx_buffers;
2776         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2777                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2778                 if (error != 0) {
2779                         device_printf(dev, "Unable to create TX DMA map\n");
2780                         goto fail;
2781                 }
2782         }
2783
2784         return 0;
2785 fail:
2786         /* We free all, it handles case where we are in the middle */
2787         ixgbe_free_transmit_structures(adapter);
2788         return (error);
2789 }
2790
2791 /*********************************************************************
2792  *
2793  *  Initialize a transmit ring.
2794  *
2795  **********************************************************************/
2796 static void
2797 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2798 {
2799         struct adapter *adapter = txr->adapter;
2800         struct ixgbe_tx_buf *txbuf;
2801         int i;
2802 #ifdef DEV_NETMAP
2803         struct netmap_adapter *na = NA(adapter->ifp);
2804         struct netmap_slot *slot;
2805 #endif /* DEV_NETMAP */
2806
2807         /* Clear the old ring contents */
2808         IXGBE_TX_LOCK(txr);
2809 #ifdef DEV_NETMAP
2810         /*
2811          * (under lock): if in netmap mode, do some consistency
2812          * checks and set slot to entry 0 of the netmap ring.
2813          */
2814         slot = netmap_reset(na, NR_TX, txr->me, 0);
2815 #endif /* DEV_NETMAP */
2816         bzero((void *)txr->tx_base,
2817               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
2818         /* Reset indices */
2819         txr->next_avail_desc = 0;
2820         txr->next_to_clean = 0;
2821
2822         /* Free any existing tx buffers. */
2823         txbuf = txr->tx_buffers;
2824         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2825                 if (txbuf->m_head != NULL) {
2826                         bus_dmamap_sync(txr->txtag, txbuf->map,
2827                             BUS_DMASYNC_POSTWRITE);
2828                         bus_dmamap_unload(txr->txtag, txbuf->map);
2829                         m_freem(txbuf->m_head);
2830                         txbuf->m_head = NULL;
2831                 }
2832 #ifdef DEV_NETMAP
2833                 /*
2834                  * In netmap mode, set the map for the packet buffer.
2835                  * NOTE: Some drivers (not this one) also need to set
2836                  * the physical buffer address in the NIC ring.
2837                  * Slots in the netmap ring (indexed by "si") are
2838                  * kring->nkr_hwofs positions "ahead" wrt the
2839                  * corresponding slot in the NIC ring. In some drivers
2840                  * (not here) nkr_hwofs can be negative. Function
2841                  * netmap_idx_n2k() handles wraparounds properly.
2842                  */
2843                 if (slot) {
2844                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
2845                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
2846                 }
2847 #endif /* DEV_NETMAP */
2848                 /* Clear the EOP index */
2849                 txbuf->eop_index = -1;
2850         }
2851
2852 #ifdef IXGBE_FDIR
2853         /* Set the rate at which we sample packets */
2854         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
2855                 txr->atr_sample = atr_sample_rate;
2856 #endif
2857
2858         /* Set number of descriptors available */
2859         txr->tx_avail = adapter->num_tx_desc;
2860
2861         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2862             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2863         IXGBE_TX_UNLOCK(txr);
2864 }
2865
2866 /*********************************************************************
2867  *
2868  *  Initialize all transmit rings.
2869  *
2870  **********************************************************************/
2871 static int
2872 ixgbe_setup_transmit_structures(struct adapter *adapter)
2873 {
2874         struct tx_ring *txr = adapter->tx_rings;
2875
2876         for (int i = 0; i < adapter->num_queues; i++, txr++)
2877                 ixgbe_setup_transmit_ring(txr);
2878
2879         return (0);
2880 }
2881
2882 /*********************************************************************
2883  *
2884  *  Enable transmit unit.
2885  *
2886  **********************************************************************/
2887 static void
2888 ixgbe_initialize_transmit_units(struct adapter *adapter)
2889 {
2890         struct tx_ring  *txr = adapter->tx_rings;
2891         struct ixgbe_hw *hw = &adapter->hw;
2892
2893         /* Setup the Base and Length of the Tx Descriptor Ring */
2894
2895         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2896                 u64     tdba = txr->txdma.dma_paddr;
2897                 u32     txctrl;
2898
2899                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
2900                        (tdba & 0x00000000ffffffffULL));
2901                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
2902                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
2903                     adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
2904
2905                 /* Setup the HW Tx Head and Tail descriptor pointers */
2906                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
2907                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
2908
2909                 /* Setup Transmit Descriptor Cmd Settings */
2910                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
2911                 txr->queue_status = IXGBE_QUEUE_IDLE;
2912
2913                 /* Disable Head Writeback */
2914                 switch (hw->mac.type) {
2915                 case ixgbe_mac_82598EB:
2916                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
2917                         break;
2918                 case ixgbe_mac_82599EB:
2919                 case ixgbe_mac_X540:
2920                 default:
2921                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
2922                         break;
2923                 }
2924                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2925                 switch (hw->mac.type) {
2926                 case ixgbe_mac_82598EB:
2927                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
2928                         break;
2929                 case ixgbe_mac_82599EB:
2930                 case ixgbe_mac_X540:
2931                 default:
2932                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
2933                         break;
2934                 }
2935
2936         }
2937
2938         if (hw->mac.type != ixgbe_mac_82598EB) {
2939                 u32 dmatxctl, rttdcs;
2940                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2941                 dmatxctl |= IXGBE_DMATXCTL_TE;
2942                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2943                 /* Disable arbiter to set MTQC */
2944                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2945                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
2946                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2947                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2948                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2949                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2950         }
2951
2952         return;
2953 }
2954
2955 /*********************************************************************
2956  *
2957  *  Free all transmit rings.
2958  *
2959  **********************************************************************/
2960 static void
2961 ixgbe_free_transmit_structures(struct adapter *adapter)
2962 {
2963         struct tx_ring *txr = adapter->tx_rings;
2964
2965         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2966                 IXGBE_TX_LOCK(txr);
2967                 ixgbe_free_transmit_buffers(txr);
2968                 ixgbe_dma_free(adapter, &txr->txdma);
2969                 IXGBE_TX_UNLOCK(txr);
2970                 IXGBE_TX_LOCK_DESTROY(txr);
2971         }
2972         kfree(adapter->tx_rings, M_DEVBUF);
2973 }
2974
2975 /*********************************************************************
2976  *
2977  *  Free transmit ring related data structures.
2978  *
2979  **********************************************************************/
2980 static void
2981 ixgbe_free_transmit_buffers(struct tx_ring *txr)
2982 {
2983         struct adapter *adapter = txr->adapter;
2984         struct ixgbe_tx_buf *tx_buffer;
2985         int             i;
2986
2987         INIT_DEBUGOUT("free_transmit_ring: begin");
2988
2989         if (txr->tx_buffers == NULL)
2990                 return;
2991
2992         tx_buffer = txr->tx_buffers;
2993         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2994                 if (tx_buffer->m_head != NULL) {
2995                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
2996                             BUS_DMASYNC_POSTWRITE);
2997                         bus_dmamap_unload(txr->txtag,
2998                             tx_buffer->map);
2999                         m_freem(tx_buffer->m_head);
3000                         tx_buffer->m_head = NULL;
3001                         if (tx_buffer->map != NULL) {
3002                                 bus_dmamap_destroy(txr->txtag,
3003                                     tx_buffer->map);
3004                                 tx_buffer->map = NULL;
3005                         }
3006                 } else if (tx_buffer->map != NULL) {
3007                         bus_dmamap_unload(txr->txtag,
3008                             tx_buffer->map);
3009                         bus_dmamap_destroy(txr->txtag,
3010                             tx_buffer->map);
3011                         tx_buffer->map = NULL;
3012                 }
3013         }
3014 #if 0 /* __FreeBSD_version >= 800000 */
3015         if (txr->br != NULL)
3016                 buf_ring_free(txr->br, M_DEVBUF);
3017 #endif
3018         if (txr->tx_buffers != NULL) {
3019                 kfree(txr->tx_buffers, M_DEVBUF);
3020                 txr->tx_buffers = NULL;
3021         }
3022         if (txr->txtag != NULL) {
3023                 bus_dma_tag_destroy(txr->txtag);
3024                 txr->txtag = NULL;
3025         }
3026         return;
3027 }
3028
3029 /*********************************************************************
3030  *
3031  *  Advanced Context Descriptor setup for VLAN or CSUM
3032  *
3033  **********************************************************************/
3034
3035 static bool
3036 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3037 {
3038         struct adapter *adapter = txr->adapter;
3039         struct ixgbe_adv_tx_context_desc *TXD;
3040         struct ixgbe_tx_buf        *tx_buffer;
3041         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3042         struct ether_vlan_header *eh;
3043         struct ip *ip;
3044         struct ip6_hdr *ip6;
3045         int  ehdrlen, ip_hlen = 0;
3046         u16     etype;
3047         u8      ipproto = 0;
3048         bool    offload = TRUE;
3049         int ctxd = txr->next_avail_desc;
3050         u16 vtag = 0;
3051
3052
3053         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3054                 offload = FALSE;
3055
3056         tx_buffer = &txr->tx_buffers[ctxd];
3057         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3058
3059         /*
3060         ** In advanced descriptors the vlan tag must
3061         ** be placed into the descriptor itself.
3062         */
3063         if (mp->m_flags & M_VLANTAG) {
3064                 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3065                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3066         } else if (offload == FALSE)
3067                 return FALSE;
3068
3069         /*
3070          * Determine where frame payload starts.
3071          * Jump over vlan headers if already present,
3072          * helpful for QinQ too.
3073          */
3074         eh = mtod(mp, struct ether_vlan_header *);
3075         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3076                 etype = ntohs(eh->evl_proto);
3077                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3078         } else {
3079                 etype = ntohs(eh->evl_encap_proto);
3080                 ehdrlen = ETHER_HDR_LEN;
3081         }
3082
3083         /* Set the ether header length */
3084         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3085
3086         switch (etype) {
3087                 case ETHERTYPE_IP:
3088                         ip = (struct ip *)(mp->m_data + ehdrlen);
3089                         ip_hlen = ip->ip_hl << 2;
3090                         ipproto = ip->ip_p;
3091                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3092                         break;
3093                 case ETHERTYPE_IPV6:
3094                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3095                         ip_hlen = sizeof(struct ip6_hdr);
3096                         /* XXX-BZ this will go badly in case of ext hdrs. */
3097                         ipproto = ip6->ip6_nxt;
3098                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3099                         break;
3100                 default:
3101                         offload = FALSE;
3102                         break;
3103         }
3104
3105         vlan_macip_lens |= ip_hlen;
3106         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3107
3108         switch (ipproto) {
3109                 case IPPROTO_TCP:
3110                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3111                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3112                         break;
3113
3114                 case IPPROTO_UDP:
3115                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3116                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3117                         break;
3118
3119 #if 0
3120                 case IPPROTO_SCTP:
3121                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3122                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3123                         break;
3124 #endif
3125                 default:
3126                         offload = FALSE;
3127                         break;
3128         }
3129
3130         /* Now copy bits into descriptor */
3131         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3132         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3133         TXD->seqnum_seed = htole32(0);
3134         TXD->mss_l4len_idx = htole32(0);
3135
3136         tx_buffer->m_head = NULL;
3137         tx_buffer->eop_index = -1;
3138
3139         /* We've consumed the first desc, adjust counters */
3140         if (++ctxd == adapter->num_tx_desc)
3141                 ctxd = 0;
3142         txr->next_avail_desc = ctxd;
3143         --txr->tx_avail;
3144
3145         return (offload);
3146 }
3147
3148 /**********************************************************************
3149  *
3150  *  Setup work for hardware segmentation offload (TSO) on
3151  *  adapters using advanced tx descriptors
3152  *
3153  **********************************************************************/
3154 static bool
3155 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3156     u32 *olinfo_status)
3157 {
3158         struct adapter *adapter = txr->adapter;
3159         struct ixgbe_adv_tx_context_desc *TXD;
3160         struct ixgbe_tx_buf        *tx_buffer;
3161         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3162         u16 vtag = 0, eh_type;
3163         u32 mss_l4len_idx = 0, len;
3164         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3165         struct ether_vlan_header *eh;
3166 #if 0 /* IPv6 TSO */
3167 #ifdef INET6
3168         struct ip6_hdr *ip6;
3169 #endif
3170 #endif
3171 #ifdef INET
3172         struct ip *ip;
3173 #endif
3174         struct tcphdr *th;
3175
3176
3177         /*
3178          * Determine where frame payload starts.
3179          * Jump over vlan headers if already present
3180          */
3181         eh = mtod(mp, struct ether_vlan_header *);
3182         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3183                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3184                 eh_type = eh->evl_proto;
3185         } else {
3186                 ehdrlen = ETHER_HDR_LEN;
3187                 eh_type = eh->evl_encap_proto;
3188         }
3189
3190         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3191         len = ehdrlen + sizeof(struct tcphdr);
3192         switch (ntohs(eh_type)) {
3193 #if 0 /* IPv6 TSO */
3194 #ifdef INET6
3195         case ETHERTYPE_IPV6:
3196                 if (mp->m_len < len + sizeof(struct ip6_hdr))
3197                         return FALSE;
3198                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3199                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3200                 if (ip6->ip6_nxt != IPPROTO_TCP)
3201                         return FALSE;
3202                 ip_hlen = sizeof(struct ip6_hdr);
3203                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3204                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3205                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3206                 break;
3207 #endif
3208 #endif
3209 #ifdef INET
3210         case ETHERTYPE_IP:
3211                 if (mp->m_len < len + sizeof(struct ip))
3212                         return FALSE;
3213                 ip = (struct ip *)(mp->m_data + ehdrlen);
3214                 if (ip->ip_p != IPPROTO_TCP)
3215                         return FALSE;
3216                 ip->ip_sum = 0;
3217                 ip_hlen = ip->ip_hl << 2;
3218                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3219                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3220                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3221                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3222                 /* Tell transmit desc to also do IPv4 checksum. */
3223                 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3224                 break;
3225 #endif
3226         default:
3227                 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3228                     __func__, ntohs(eh_type));
3229                 break;
3230         }
3231
3232         ctxd = txr->next_avail_desc;
3233         tx_buffer = &txr->tx_buffers[ctxd];
3234         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3235
3236         tcp_hlen = th->th_off << 2;
3237
3238         /* This is used in the transmit desc in encap */
3239         *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3240
3241         /* VLAN MACLEN IPLEN */
3242         if (mp->m_flags & M_VLANTAG) {
3243                 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3244                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3245         }
3246
3247         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3248         vlan_macip_lens |= ip_hlen;
3249         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3250
3251         /* ADV DTYPE TUCMD */
3252         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3253         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3254         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3255
3256         /* MSS L4LEN IDX */
3257         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3258         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3259         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3260
3261         TXD->seqnum_seed = htole32(0);
3262         tx_buffer->m_head = NULL;
3263         tx_buffer->eop_index = -1;
3264
3265         if (++ctxd == adapter->num_tx_desc)
3266                 ctxd = 0;
3267
3268         txr->tx_avail--;
3269         txr->next_avail_desc = ctxd;
3270         return TRUE;
3271 }
3272
3273 #ifdef IXGBE_FDIR
3274 /*
3275 ** This routine parses packet headers so that Flow
3276 ** Director can make a hashed filter table entry
3277 ** allowing traffic flows to be identified and kept
3278 ** on the same cpu.  This would be a performance
3279 ** hit, but we only do it at IXGBE_FDIR_RATE of
3280 ** packets.
3281 */
3282 static void
3283 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3284 {
3285         struct adapter                  *adapter = txr->adapter;
3286         struct ix_queue                 *que;
3287         struct ip                       *ip;
3288         struct tcphdr                   *th;
3289         struct udphdr                   *uh;
3290         struct ether_vlan_header        *eh;
3291         union ixgbe_atr_hash_dword      input = {.dword = 0};
3292         union ixgbe_atr_hash_dword      common = {.dword = 0};
3293         int                             ehdrlen, ip_hlen;
3294         u16                             etype;
3295
3296         eh = mtod(mp, struct ether_vlan_header *);
3297         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3298                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3299                 etype = eh->evl_proto;
3300         } else {
3301                 ehdrlen = ETHER_HDR_LEN;
3302                 etype = eh->evl_encap_proto;
3303         }
3304
3305         /* Only handling IPv4 */
3306         if (etype != htons(ETHERTYPE_IP))
3307                 return;
3308
3309         ip = (struct ip *)(mp->m_data + ehdrlen);
3310         ip_hlen = ip->ip_hl << 2;
3311
3312         /* check if we're UDP or TCP */
3313         switch (ip->ip_p) {
3314         case IPPROTO_TCP:
3315                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3316                 /* src and dst are inverted */
3317                 common.port.dst ^= th->th_sport;
3318                 common.port.src ^= th->th_dport;
3319                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3320                 break;
3321         case IPPROTO_UDP:
3322                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3323                 /* src and dst are inverted */
3324                 common.port.dst ^= uh->uh_sport;
3325                 common.port.src ^= uh->uh_dport;
3326                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3327                 break;
3328         default:
3329                 return;
3330         }
3331
3332         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3333         if (mp->m_pkthdr.ether_vtag)
3334                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3335         else
3336                 common.flex_bytes ^= etype;
3337         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3338
3339         que = &adapter->queues[txr->me];
3340         /*
3341         ** This assumes the Rx queue and Tx
3342         ** queue are bound to the same CPU
3343         */
3344         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3345             input, common, que->msix);
3346 }
3347 #endif /* IXGBE_FDIR */
3348
3349 /**********************************************************************
3350  *
3351  *  Examine each tx_buffer in the used queue. If the hardware is done
3352  *  processing the packet then free associated resources. The
3353  *  tx_buffer is put back on the free queue.
3354  *
3355  **********************************************************************/
3356 static void
3357 ixgbe_txeof(struct tx_ring *txr)
3358 {
3359         struct adapter  *adapter = txr->adapter;
3360         struct ifnet    *ifp = adapter->ifp;
3361         u32     first, last, done, processed;
3362         struct ixgbe_tx_buf *tx_buffer;
3363         struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3364
3365         IXGBE_TX_LOCK_ASSERT(txr);
3366
3367 #ifdef DEV_NETMAP
3368         if (ifp->if_capenable & IFCAP_NETMAP) {
3369                 struct netmap_adapter *na = NA(ifp);
3370                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3371
3372                 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3373
3374                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3375                     BUS_DMASYNC_POSTREAD);
3376                 /*
3377                  * In netmap mode, all the work is done in the context
3378                  * of the client thread. Interrupt handlers only wake up
3379                  * clients, which may be sleeping on individual rings
3380                  * or on a global resource for all rings.
3381                  * To implement tx interrupt mitigation, we wake up the client
3382                  * thread roughly every half ring, even if the NIC interrupts
3383                  * more frequently. This is implemented as follows:
3384                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3385                  *   the slot that should wake up the thread (nkr_num_slots
3386                  *   means the user thread should not be woken up);
3387                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3388                  *   or the slot has the DD bit set.
3389                  *
3390                  * When the driver has separate locks, we need to
3391                  * release and re-acquire txlock to avoid deadlocks.
3392                  * XXX see if we can find a better way.
3393                  */
3394                 if (!netmap_mitigate ||
3395                     (kring->nr_kflags < kring->nkr_num_slots &&
3396                      tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3397                         kring->nr_kflags = kring->nkr_num_slots;
3398                         selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3399                         IXGBE_TX_UNLOCK(txr);
3400                         IXGBE_CORE_LOCK(adapter);
3401                         selwakeuppri(&na->tx_si, PI_NET);
3402                         IXGBE_CORE_UNLOCK(adapter);
3403                         IXGBE_TX_LOCK(txr);
3404                 }
3405                 return FALSE;
3406         }
3407 #endif /* DEV_NETMAP */
3408
3409         if (txr->tx_avail == adapter->num_tx_desc) {
3410                 txr->queue_status = IXGBE_QUEUE_IDLE;
3411                 return;
3412         }
3413
3414         processed = 0;
3415         first = txr->next_to_clean;
3416         tx_buffer = &txr->tx_buffers[first];
3417         /* For cleanup we just use legacy struct */
3418         tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3419         last = tx_buffer->eop_index;
3420         if (last == -1)
3421                 return;
3422         eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3423
3424         /*
3425         ** Get the index of the first descriptor
3426         ** BEYOND the EOP and call that 'done'.
3427         ** I do this so the comparison in the
3428         ** inner while loop below can be simple
3429         */
3430         if (++last == adapter->num_tx_desc) last = 0;
3431         done = last;
3432
3433         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3434             BUS_DMASYNC_POSTREAD);
3435         /*
3436         ** Only the EOP descriptor of a packet now has the DD
3437         ** bit set, this is what we look for...
3438         */
3439         while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3440                 /* We clean the range of the packet */
3441                 while (first != done) {
3442                         tx_desc->upper.data = 0;
3443                         tx_desc->lower.data = 0;
3444                         tx_desc->buffer_addr = 0;
3445                         ++txr->tx_avail;
3446                         ++processed;
3447
3448                         if (tx_buffer->m_head) {
3449                                 txr->bytes +=
3450                                     tx_buffer->m_head->m_pkthdr.len;
3451                                 bus_dmamap_sync(txr->txtag,
3452                                     tx_buffer->map,
3453                                     BUS_DMASYNC_POSTWRITE);
3454                                 bus_dmamap_unload(txr->txtag,
3455                                     tx_buffer->map);
3456                                 m_freem(tx_buffer->m_head);
3457                                 tx_buffer->m_head = NULL;
3458                                 tx_buffer->map = NULL;
3459                         }
3460                         tx_buffer->eop_index = -1;
3461                         txr->watchdog_time = ticks;
3462
3463                         if (++first == adapter->num_tx_desc)
3464                                 first = 0;
3465
3466                         tx_buffer = &txr->tx_buffers[first];
3467                         tx_desc =
3468                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3469                 }
3470                 ++txr->packets;
3471                 ++ifp->if_opackets;
3472                 /* See if there is more work now */
3473                 last = tx_buffer->eop_index;
3474                 if (last != -1) {
3475                         eop_desc =
3476                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3477                         /* Get next done point */
3478                         if (++last == adapter->num_tx_desc) last = 0;
3479                         done = last;
3480                 } else
3481                         break;
3482         }
3483         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3484             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3485
3486         txr->next_to_clean = first;
3487
3488         /*
3489         ** Watchdog calculation, we know there's
3490         ** work outstanding or the first return
3491         ** would have been taken, so none processed
3492         ** for too long indicates a hang.
3493         */
3494         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3495                 txr->queue_status = IXGBE_QUEUE_HUNG;
3496
3497         /* With a minimum free clear the depleted state bit.  */
3498         if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3499                 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3500
3501         if (txr->tx_avail == adapter->num_tx_desc) {
3502                 txr->queue_status = IXGBE_QUEUE_IDLE;
3503         }
3504 }
3505
3506 /*********************************************************************
3507  *
3508  *  Refresh mbuf buffers for RX descriptor rings
3509  *   - now keeps its own state so discards due to resource
3510  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3511  *     it just returns, keeping its placeholder, thus it can simply
3512  *     be recalled to try again.
3513  *
3514  **********************************************************************/
3515 static void
3516 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3517 {
3518         struct adapter          *adapter = rxr->adapter;
3519         bus_dma_segment_t       hseg[1];
3520         bus_dma_segment_t       pseg[1];
3521         struct ixgbe_rx_buf     *rxbuf;
3522         struct mbuf             *mh, *mp;
3523         int                     i, j, nsegs, error;
3524         bool                    refreshed = FALSE;
3525
3526         i = j = rxr->next_to_refresh;
3527         /* Control the loop with one beyond */
3528         if (++j == adapter->num_rx_desc)
3529                 j = 0;
3530
3531         while (j != limit) {
3532                 rxbuf = &rxr->rx_buffers[i];
3533                 if (rxr->hdr_split == FALSE)
3534                         goto no_split;
3535
3536                 if (rxbuf->m_head == NULL) {
3537                         mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3538                         if (mh == NULL)
3539                                 goto update;
3540                 } else
3541                         mh = rxbuf->m_head;
3542
3543                 mh->m_pkthdr.len = mh->m_len = MHLEN;
3544                 mh->m_len = MHLEN;
3545                 mh->m_flags |= M_PKTHDR;
3546                 /* Get the memory mapping */
3547                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3548                     rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3549                 if (error != 0) {
3550                         kprintf("Refresh mbufs: hdr dmamap load"
3551                             " failure - %d\n", error);
3552                         m_free(mh);
3553                         rxbuf->m_head = NULL;
3554                         goto update;
3555                 }
3556                 rxbuf->m_head = mh;
3557                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3558                     BUS_DMASYNC_PREREAD);
3559                 rxr->rx_base[i].read.hdr_addr =
3560                     htole64(hseg[0].ds_addr);
3561
3562 no_split:
3563                 if (rxbuf->m_pack == NULL) {
3564                         mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3565                             M_PKTHDR, adapter->rx_mbuf_sz);
3566                         if (mp == NULL)
3567                                 goto update;
3568                 } else
3569                         mp = rxbuf->m_pack;
3570
3571                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3572                 /* Get the memory mapping */
3573                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3574                     rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3575                 if (error != 0) {
3576                         kprintf("Refresh mbufs: payload dmamap load"
3577                             " failure - %d\n", error);
3578                         m_free(mp);
3579                         rxbuf->m_pack = NULL;
3580                         goto update;
3581                 }
3582                 rxbuf->m_pack = mp;
3583                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3584                     BUS_DMASYNC_PREREAD);
3585                 rxr->rx_base[i].read.pkt_addr =
3586                     htole64(pseg[0].ds_addr);
3587
3588                 refreshed = TRUE;
3589                 /* Next is precalculated */
3590                 i = j;
3591                 rxr->next_to_refresh = i;
3592                 if (++j == adapter->num_rx_desc)
3593                         j = 0;
3594         }
3595 update:
3596         if (refreshed) /* Update hardware tail index */
3597                 IXGBE_WRITE_REG(&adapter->hw,
3598                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3599         return;
3600 }
3601
3602 /*********************************************************************
3603  *
3604  *  Allocate memory for rx_buffer structures. Since we use one
3605  *  rx_buffer per received packet, the maximum number of rx_buffer's
3606  *  that we'll need is equal to the number of receive descriptors
3607  *  that we've allocated.
3608  *
3609  **********************************************************************/
3610 static int
3611 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3612 {
3613         struct  adapter         *adapter = rxr->adapter;
3614         device_t                dev = adapter->dev;
3615         struct ixgbe_rx_buf     *rxbuf;
3616         int                     i, bsize, error;
3617
3618         bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3619         if (!(rxr->rx_buffers =
3620             (struct ixgbe_rx_buf *) kmalloc(bsize,
3621             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3622                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3623                 error = ENOMEM;
3624                 goto fail;
3625         }
3626
3627         if ((error = bus_dma_tag_create(NULL,   /* parent */
3628                                    1, 0,        /* alignment, bounds */
3629                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3630                                    BUS_SPACE_MAXADDR,   /* highaddr */
3631                                    NULL, NULL,          /* filter, filterarg */
3632                                    MSIZE,               /* maxsize */
3633                                    1,                   /* nsegments */
3634                                    MSIZE,               /* maxsegsize */
3635                                    0,                   /* flags */
3636                                    &rxr->htag))) {
3637                 device_printf(dev, "Unable to create RX DMA tag\n");
3638                 goto fail;
3639         }
3640
3641         if ((error = bus_dma_tag_create(NULL,   /* parent */
3642                                    1, 0,        /* alignment, bounds */
3643                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3644                                    BUS_SPACE_MAXADDR,   /* highaddr */
3645                                    NULL, NULL,          /* filter, filterarg */
3646                                    MJUM16BYTES,         /* maxsize */
3647                                    1,                   /* nsegments */
3648                                    MJUM16BYTES,         /* maxsegsize */
3649                                    0,                   /* flags */
3650                                    &rxr->ptag))) {
3651                 device_printf(dev, "Unable to create RX DMA tag\n");
3652                 goto fail;
3653         }
3654
3655         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3656                 rxbuf = &rxr->rx_buffers[i];
3657                 error = bus_dmamap_create(rxr->htag,
3658                     BUS_DMA_NOWAIT, &rxbuf->hmap);
3659                 if (error) {
3660                         device_printf(dev, "Unable to create RX head map\n");
3661                         goto fail;
3662                 }
3663                 error = bus_dmamap_create(rxr->ptag,
3664                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3665                 if (error) {
3666                         device_printf(dev, "Unable to create RX pkt map\n");
3667                         goto fail;
3668                 }
3669         }
3670
3671         return (0);
3672
3673 fail:
3674         /* Frees all, but can handle partial completion */
3675         ixgbe_free_receive_structures(adapter);
3676         return (error);
3677 }
3678
3679 /*
3680 ** Used to detect a descriptor that has
3681 ** been merged by Hardware RSC.
3682 */
3683 static inline u32
3684 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3685 {
3686         return (le32toh(rx->wb.lower.lo_dword.data) &
3687             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3688 }
3689
3690 /*********************************************************************
3691  *
3692  *  Initialize Hardware RSC (LRO) feature on 82599
3693  *  for an RX ring, this is toggled by the LRO capability
3694  *  even though it is transparent to the stack.
3695  *
3696  **********************************************************************/
3697 #if 0   /* NET_LRO */
3698 static void
3699 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3700 {
3701         struct  adapter         *adapter = rxr->adapter;
3702         struct  ixgbe_hw        *hw = &adapter->hw;
3703         u32                     rscctrl, rdrxctl;
3704
3705         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3706         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3707 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3708         if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3709 #endif /* DEV_NETMAP */
3710         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3711         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3712         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3713
3714         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3715         rscctrl |= IXGBE_RSCCTL_RSCEN;
3716         /*
3717         ** Limit the total number of descriptors that
3718         ** can be combined, so it does not exceed 64K
3719         */
3720         if (adapter->rx_mbuf_sz == MCLBYTES)
3721                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3722         else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3723                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3724         else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3725                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3726         else  /* Using 16K cluster */
3727                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3728
3729         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3730
3731         /* Enable TCP header recognition */
3732         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3733             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3734             IXGBE_PSRTYPE_TCPHDR));
3735
3736         /* Disable RSC for ACK packets */
3737         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3738             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3739
3740         rxr->hw_rsc = TRUE;
3741 }
3742 #endif
3743
3744 static void
3745 ixgbe_free_receive_ring(struct rx_ring *rxr)
3746 {
3747         struct  adapter         *adapter;
3748         struct ixgbe_rx_buf       *rxbuf;
3749         int i;
3750
3751         adapter = rxr->adapter;
3752         for (i = 0; i < adapter->num_rx_desc; i++) {
3753                 rxbuf = &rxr->rx_buffers[i];
3754                 if (rxbuf->m_head != NULL) {
3755                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3756                             BUS_DMASYNC_POSTREAD);
3757                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3758                         rxbuf->m_head->m_flags |= M_PKTHDR;
3759                         m_freem(rxbuf->m_head);
3760                 }
3761                 if (rxbuf->m_pack != NULL) {
3762                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3763                             BUS_DMASYNC_POSTREAD);
3764                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3765                         rxbuf->m_pack->m_flags |= M_PKTHDR;
3766                         m_freem(rxbuf->m_pack);
3767                 }
3768                 rxbuf->m_head = NULL;
3769                 rxbuf->m_pack = NULL;
3770         }
3771 }
3772
3773
3774 /*********************************************************************
3775  *
3776  *  Initialize a receive ring and its buffers.
3777  *
3778  **********************************************************************/
3779 static int
3780 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3781 {
3782         struct  adapter         *adapter;
3783         struct ifnet            *ifp;
3784         device_t                dev;
3785         struct ixgbe_rx_buf     *rxbuf;
3786         bus_dma_segment_t       pseg[1], hseg[1];
3787 #if 0   /* NET_LRO */
3788         struct lro_ctrl         *lro = &rxr->lro;
3789 #endif
3790         int                     rsize, nsegs, error = 0;
3791 #ifdef DEV_NETMAP
3792         struct netmap_adapter *na = NA(rxr->adapter->ifp);
3793         struct netmap_slot *slot;
3794 #endif /* DEV_NETMAP */
3795
3796         adapter = rxr->adapter;
3797         ifp = adapter->ifp;
3798         dev = adapter->dev;
3799
3800         /* Clear the ring contents */
3801         IXGBE_RX_LOCK(rxr);
3802 #ifdef DEV_NETMAP
3803         /* same as in ixgbe_setup_transmit_ring() */
3804         slot = netmap_reset(na, NR_RX, rxr->me, 0);
3805 #endif /* DEV_NETMAP */
3806         rsize = roundup2(adapter->num_rx_desc *
3807             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3808         bzero((void *)rxr->rx_base, rsize);
3809
3810         /* Free current RX buffer structs and their mbufs */
3811         ixgbe_free_receive_ring(rxr);
3812
3813         /* Configure header split? */
3814         if (ixgbe_header_split)
3815                 rxr->hdr_split = TRUE;
3816
3817         /* Now replenish the mbufs */
3818         for (int j = 0; j != adapter->num_rx_desc; ++j) {
3819                 struct mbuf     *mh, *mp;
3820
3821                 rxbuf = &rxr->rx_buffers[j];
3822 #ifdef DEV_NETMAP
3823                 /*
3824                  * In netmap mode, fill the map and set the buffer
3825                  * address in the NIC ring, considering the offset
3826                  * between the netmap and NIC rings (see comment in
3827                  * ixgbe_setup_transmit_ring() ). No need to allocate
3828                  * an mbuf, so end the block with a continue;
3829                  */
3830                 if (slot) {
3831                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
3832                         uint64_t paddr;
3833                         void *addr;
3834
3835                         addr = PNMB(slot + sj, &paddr);
3836                         netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
3837                         /* Update descriptor */
3838                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
3839                         continue;
3840                 }
3841 #endif /* DEV_NETMAP */
3842                 /*
3843                 ** Don't allocate mbufs if not
3844                 ** doing header split, its wasteful
3845                 */
3846                 if (rxr->hdr_split == FALSE)
3847                         goto skip_head;
3848
3849                 /* First the header */
3850                 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
3851                 if (rxbuf->m_head == NULL) {
3852                         error = ENOBUFS;
3853                         goto fail;
3854                 }
3855                 m_adj(rxbuf->m_head, ETHER_ALIGN);
3856                 mh = rxbuf->m_head;
3857                 mh->m_len = mh->m_pkthdr.len = MHLEN;
3858                 mh->m_flags |= M_PKTHDR;
3859                 /* Get the memory mapping */
3860                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3861                     rxbuf->hmap, rxbuf->m_head, hseg, 1,
3862                     &nsegs, BUS_DMA_NOWAIT);
3863
3864                 if (error != 0) /* Nothing elegant to do here */
3865                         goto fail;
3866                 bus_dmamap_sync(rxr->htag,
3867                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
3868                 /* Update descriptor */
3869                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
3870
3871 skip_head:
3872                 /* Now the payload cluster */
3873                 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
3874                     M_PKTHDR, adapter->rx_mbuf_sz);
3875                 if (rxbuf->m_pack == NULL) {
3876                         error = ENOBUFS;
3877                         goto fail;
3878                 }
3879                 mp = rxbuf->m_pack;
3880                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3881                 /* Get the memory mapping */
3882                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3883                     rxbuf->pmap, mp, hseg, 1,
3884                     &nsegs, BUS_DMA_NOWAIT);
3885                 if (error != 0)
3886                         goto fail;
3887                 bus_dmamap_sync(rxr->ptag,
3888                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
3889                 /* Update descriptor */
3890                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
3891         }
3892
3893
3894         /* Setup our descriptor indices */
3895         rxr->next_to_check = 0;
3896         rxr->next_to_refresh = 0;
3897         rxr->lro_enabled = FALSE;
3898         rxr->rx_split_packets = 0;
3899         rxr->rx_bytes = 0;
3900         rxr->discard = FALSE;
3901         rxr->vtag_strip = FALSE;
3902
3903         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3904             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3905
3906         /*
3907         ** Now set up the LRO interface:
3908         ** 82598 uses software LRO, the
3909         ** 82599 and X540 use a hardware assist.
3910         */
3911 #if 0 /* NET_LRO */
3912         if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
3913             (ifp->if_capenable & IFCAP_RXCSUM) &&
3914             (ifp->if_capenable & IFCAP_LRO))
3915                 ixgbe_setup_hw_rsc(rxr);
3916         else if (ifp->if_capenable & IFCAP_LRO) {
3917                 int err = tcp_lro_init(lro);
3918                 if (err) {
3919                         device_printf(dev, "LRO Initialization failed!\n");
3920                         goto fail;
3921                 }
3922                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
3923                 rxr->lro_enabled = TRUE;
3924                 lro->ifp = adapter->ifp;
3925         }
3926 #endif
3927
3928         IXGBE_RX_UNLOCK(rxr);
3929         return (0);
3930
3931 fail:
3932         ixgbe_free_receive_ring(rxr);
3933         IXGBE_RX_UNLOCK(rxr);
3934         return (error);
3935 }
3936
3937 /*********************************************************************
3938  *
3939  *  Initialize all receive rings.
3940  *
3941  **********************************************************************/
3942 static int
3943 ixgbe_setup_receive_structures(struct adapter *adapter)
3944 {
3945         struct rx_ring *rxr = adapter->rx_rings;
3946         int j;
3947
3948         for (j = 0; j < adapter->num_queues; j++, rxr++)
3949                 if (ixgbe_setup_receive_ring(rxr))
3950                         goto fail;
3951
3952         return (0);
3953 fail:
3954         /*
3955          * Free RX buffers allocated so far, we will only handle
3956          * the rings that completed, the failing case will have
3957          * cleaned up for itself. 'j' failed, so its the terminus.
3958          */
3959         for (int i = 0; i < j; ++i) {
3960                 rxr = &adapter->rx_rings[i];
3961                 ixgbe_free_receive_ring(rxr);
3962         }
3963
3964         return (ENOBUFS);
3965 }
3966
3967 /*********************************************************************
3968  *
3969  *  Setup receive registers and features.
3970  *
3971  **********************************************************************/
3972 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
3973
3974 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
3975
3976 static void
3977 ixgbe_initialize_receive_units(struct adapter *adapter)
3978 {
3979         struct  rx_ring *rxr = adapter->rx_rings;
3980         struct ixgbe_hw *hw = &adapter->hw;
3981         struct ifnet   *ifp = adapter->ifp;
3982         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
3983         u32             reta, mrqc = 0, hlreg, random[10];
3984
3985
3986         /*
3987          * Make sure receives are disabled while
3988          * setting up the descriptor ring
3989          */
3990         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
3991         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
3992             rxctrl & ~IXGBE_RXCTRL_RXEN);
3993
3994         /* Enable broadcasts */
3995         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3996         fctrl |= IXGBE_FCTRL_BAM;
3997         fctrl |= IXGBE_FCTRL_DPF;
3998         fctrl |= IXGBE_FCTRL_PMCF;
3999         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4000
4001         /* Set for Jumbo Frames? */
4002         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4003         if (ifp->if_mtu > ETHERMTU)
4004                 hlreg |= IXGBE_HLREG0_JUMBOEN;
4005         else
4006                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4007 #ifdef DEV_NETMAP
4008         /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4009         if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4010                 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4011         else
4012                 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4013 #endif /* DEV_NETMAP */
4014         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4015
4016         bufsz = (adapter->rx_mbuf_sz +
4017             BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4018
4019         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4020                 u64 rdba = rxr->rxdma.dma_paddr;
4021
4022                 /* Setup the Base and Length of the Rx Descriptor Ring */
4023                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4024                                (rdba & 0x00000000ffffffffULL));
4025                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4026                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4027                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4028
4029                 /* Set up the SRRCTL register */
4030                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4031                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4032                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4033                 srrctl |= bufsz;
4034                 if (rxr->hdr_split) {
4035                         /* Use a standard mbuf for the header */
4036                         srrctl |= ((IXGBE_RX_HDR <<
4037                             IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
4038                             & IXGBE_SRRCTL_BSIZEHDR_MASK);
4039                         srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4040                 } else
4041                         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4042                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4043
4044                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4045                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4046                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4047         }
4048
4049         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4050                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4051                               IXGBE_PSRTYPE_UDPHDR |
4052                               IXGBE_PSRTYPE_IPV4HDR |
4053                               IXGBE_PSRTYPE_IPV6HDR;
4054                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4055         }
4056
4057         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4058
4059         /* Setup RSS */
4060         if (adapter->num_queues > 1) {
4061                 int i, j;
4062                 reta = 0;
4063
4064                 /* set up random bits */
4065                 karc4rand(&random, sizeof(random));
4066
4067                 /* Set up the redirection table */
4068                 for (i = 0, j = 0; i < 128; i++, j++) {
4069                         if (j == adapter->num_queues) j = 0;
4070                         reta = (reta << 8) | (j * 0x11);
4071                         if ((i & 3) == 3)
4072                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4073                 }
4074
4075                 /* Now fill our hash function seeds */
4076                 for (int i = 0; i < 10; i++)
4077                         IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4078
4079                 /* Perform hash on these packet types */
4080                 mrqc = IXGBE_MRQC_RSSEN
4081                      | IXGBE_MRQC_RSS_FIELD_IPV4
4082                      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4083                      | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4084                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4085                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4086                      | IXGBE_MRQC_RSS_FIELD_IPV6
4087                      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4088                      | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4089                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4090                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4091
4092                 /* RSS and RX IPP Checksum are mutually exclusive */
4093                 rxcsum |= IXGBE_RXCSUM_PCSD;
4094         }
4095
4096         if (ifp->if_capenable & IFCAP_RXCSUM)
4097                 rxcsum |= IXGBE_RXCSUM_PCSD;
4098
4099         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4100                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4101
4102         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4103
4104         return;
4105 }
4106
4107 /*********************************************************************
4108  *
4109  *  Free all receive rings.
4110  *
4111  **********************************************************************/
4112 static void
4113 ixgbe_free_receive_structures(struct adapter *adapter)
4114 {
4115         struct rx_ring *rxr = adapter->rx_rings;
4116
4117         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4118 #if 0   /* NET_LRO */
4119                 struct lro_ctrl         *lro = &rxr->lro;
4120 #endif
4121                 ixgbe_free_receive_buffers(rxr);
4122                 /* Free LRO memory */
4123 #if 0   /* NET_LRO */
4124                 tcp_lro_free(lro);
4125 #endif
4126                 /* Free the ring memory as well */
4127                 ixgbe_dma_free(adapter, &rxr->rxdma);
4128         }
4129
4130         kfree(adapter->rx_rings, M_DEVBUF);
4131 }
4132
4133
4134 /*********************************************************************
4135  *
4136  *  Free receive ring data structures
4137  *
4138  **********************************************************************/
4139 static void
4140 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4141 {
4142         struct adapter          *adapter = rxr->adapter;
4143         struct ixgbe_rx_buf     *rxbuf;
4144
4145         INIT_DEBUGOUT("free_receive_structures: begin");
4146
4147         /* Cleanup any existing buffers */
4148         if (rxr->rx_buffers != NULL) {
4149                 for (int i = 0; i < adapter->num_rx_desc; i++) {
4150                         rxbuf = &rxr->rx_buffers[i];
4151                         if (rxbuf->m_head != NULL) {
4152                                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4153                                     BUS_DMASYNC_POSTREAD);
4154                                 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4155                                 rxbuf->m_head->m_flags |= M_PKTHDR;
4156                                 m_freem(rxbuf->m_head);
4157                         }
4158                         if (rxbuf->m_pack != NULL) {
4159                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4160                                     BUS_DMASYNC_POSTREAD);
4161                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4162                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
4163                                 m_freem(rxbuf->m_pack);
4164                         }
4165                         rxbuf->m_head = NULL;
4166                         rxbuf->m_pack = NULL;
4167                         if (rxbuf->hmap != NULL) {
4168                                 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4169                                 rxbuf->hmap = NULL;
4170                         }
4171                         if (rxbuf->pmap != NULL) {
4172                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4173                                 rxbuf->pmap = NULL;
4174                         }
4175                 }
4176                 if (rxr->rx_buffers != NULL) {
4177                         kfree(rxr->rx_buffers, M_DEVBUF);
4178                         rxr->rx_buffers = NULL;
4179                 }
4180         }
4181
4182         if (rxr->htag != NULL) {
4183                 bus_dma_tag_destroy(rxr->htag);
4184                 rxr->htag = NULL;
4185         }
4186         if (rxr->ptag != NULL) {
4187                 bus_dma_tag_destroy(rxr->ptag);
4188                 rxr->ptag = NULL;
4189         }
4190
4191         return;
4192 }
4193
4194 static __inline void
4195 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4196 {
4197
4198         /*
4199          * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4200          * should be computed by hardware. Also it should not have VLAN tag in
4201          * ethernet header.  In case of IPv6 we do not yet support ext. hdrs.
4202          */
4203 #if 0   /* NET_LRO */
4204         if (rxr->lro_enabled &&
4205             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4206             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4207             ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4208             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4209             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4210             (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4211             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4212             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4213                 /*
4214                  * Send to the stack if:
4215                  **  - LRO not enabled, or
4216                  **  - no LRO resources, or
4217                  **  - lro enqueue fails
4218                  */
4219                 if (rxr->lro.lro_cnt != 0)
4220                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4221                                 return;
4222         }
4223 #endif
4224         IXGBE_RX_UNLOCK(rxr);
4225         (*ifp->if_input)(ifp, m);
4226         IXGBE_RX_LOCK(rxr);
4227 }
4228
4229 static __inline void
4230 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4231 {
4232         struct ixgbe_rx_buf     *rbuf;
4233
4234         rbuf = &rxr->rx_buffers[i];
4235
4236         if (rbuf->fmp != NULL) {/* Partial chain ? */
4237                 rbuf->fmp->m_flags |= M_PKTHDR;
4238                 m_freem(rbuf->fmp);
4239                 rbuf->fmp = NULL;
4240         }
4241
4242         /*
4243         ** With advanced descriptors the writeback
4244         ** clobbers the buffer addrs, so its easier
4245         ** to just free the existing mbufs and take
4246         ** the normal refresh path to get new buffers
4247         ** and mapping.
4248         */
4249         if (rbuf->m_head) {
4250                 m_free(rbuf->m_head);
4251                 rbuf->m_head = NULL;
4252         }
4253
4254         if (rbuf->m_pack) {
4255                 m_free(rbuf->m_pack);
4256                 rbuf->m_pack = NULL;
4257         }
4258
4259         return;
4260 }
4261
4262 static void
4263 ixgbe_add_sysctl(struct adapter *adapter)
4264 {
4265         sysctl_ctx_init(&adapter->sysctl_ctx);
4266         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
4267             SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
4268             device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
4269         if (adapter->sysctl_tree == NULL) {
4270                 device_printf(adapter->dev, "can't add sysctl node\n");
4271                 return;
4272         }
4273         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4274                         SYSCTL_CHILDREN(adapter->sysctl_tree),
4275                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
4276                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
4277
4278         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4279             SYSCTL_CHILDREN(adapter->sysctl_tree),
4280             OID_AUTO, "intr_rate", CTLTYPE_INT | CTLFLAG_RW,
4281             adapter, 0, ixgbe_sysctl_intr_rate, "I", "interrupt rate");
4282
4283         /*
4284         ** Allow a kind of speed control by forcing the autoneg
4285         ** advertised speed list to only a certain value, this
4286         ** supports 1G on 82599 devices, and 100Mb on x540.
4287         */
4288         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4289                         SYSCTL_CHILDREN(adapter->sysctl_tree),
4290                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
4291                         adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
4292
4293         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4294                         SYSCTL_CHILDREN(adapter->sysctl_tree),
4295                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
4296                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
4297
4298         /* Sysctl for limiting the amount of work done in the taskqueue */
4299         ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
4300             "max number of rx packets to process", &adapter->rx_process_limit,
4301             ixgbe_rx_process_limit);
4302 }
4303
4304 /*********************************************************************
4305  *
4306  *  This routine executes in interrupt context. It replenishes
4307  *  the mbufs in the descriptor and sends data which has been
4308  *  dma'ed into host memory to upper layer.
4309  *
4310  *  We loop at most count times if count is > 0, or until done if
4311  *  count < 0.
4312  *
4313  *  Return TRUE for more work, FALSE for all clean.
4314  *********************************************************************/
4315 static void
4316 ixgbe_rxeof(struct ix_queue *que, int count)
4317 {
4318         struct adapter          *adapter = que->adapter;
4319         struct rx_ring          *rxr = que->rxr;
4320         struct ifnet            *ifp = adapter->ifp;
4321 #if 0   /* NET_LRO */
4322         struct lro_ctrl         *lro = &rxr->lro;
4323         struct lro_entry        *queued;
4324 #endif
4325         int                     i, nextp, processed = 0;
4326         u32                     staterr = 0;
4327         union ixgbe_adv_rx_desc *cur;
4328         struct ixgbe_rx_buf     *rbuf, *nbuf;
4329
4330         IXGBE_RX_LOCK(rxr);
4331
4332 #ifdef DEV_NETMAP
4333         if (ifp->if_capenable & IFCAP_NETMAP) {
4334                 /*
4335                  * Same as the txeof routine: only wakeup clients on intr.
4336                  * NKR_PENDINTR in nr_kflags is used to implement interrupt
4337                  * mitigation (ixgbe_rxsync() will not look for new packets
4338                  * unless NKR_PENDINTR is set).
4339                  */
4340                 struct netmap_adapter *na = NA(ifp);
4341
4342                 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4343                 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4344                 IXGBE_RX_UNLOCK(rxr);
4345                 IXGBE_CORE_LOCK(adapter);
4346                 selwakeuppri(&na->rx_si, PI_NET);
4347                 IXGBE_CORE_UNLOCK(adapter);
4348                 return (FALSE);
4349         }
4350 #endif /* DEV_NETMAP */
4351         for (i = rxr->next_to_check; count != 0;) {
4352                 struct mbuf     *sendmp, *mh, *mp;
4353                 u32             rsc, ptype;
4354                 u16             hlen, plen, hdr;
4355                 u16             vtag = 0;
4356                 bool            eop;
4357
4358                 /* Sync the ring. */
4359                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4360                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4361
4362                 cur = &rxr->rx_base[i];
4363                 staterr = le32toh(cur->wb.upper.status_error);
4364
4365                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4366                         break;
4367                 if ((ifp->if_flags & IFF_RUNNING) == 0)
4368                         break;
4369
4370                 count--;
4371                 sendmp = NULL;
4372                 nbuf = NULL;
4373                 rsc = 0;
4374                 cur->wb.upper.status_error = 0;
4375                 rbuf = &rxr->rx_buffers[i];
4376                 mh = rbuf->m_head;
4377                 mp = rbuf->m_pack;
4378
4379                 plen = le16toh(cur->wb.upper.length);
4380                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4381                     IXGBE_RXDADV_PKTTYPE_MASK;
4382                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4383                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4384
4385                 /* Process vlan info */
4386                 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
4387                         vtag = le16toh(cur->wb.upper.vlan);
4388
4389                 /* Make sure bad packets are discarded */
4390                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4391                     (rxr->discard)) {
4392                         ifp->if_ierrors++;
4393                         rxr->rx_discarded++;
4394                         if (eop)
4395                                 rxr->discard = FALSE;
4396                         else
4397                                 rxr->discard = TRUE;
4398                         ixgbe_rx_discard(rxr, i);
4399                         goto next_desc;
4400                 }
4401
4402                 /*
4403                 ** On 82599 which supports a hardware
4404                 ** LRO (called HW RSC), packets need
4405                 ** not be fragmented across sequential
4406                 ** descriptors, rather the next descriptor
4407                 ** is indicated in bits of the descriptor.
4408                 ** This also means that we might proceses
4409                 ** more than one packet at a time, something
4410                 ** that has never been true before, it
4411                 ** required eliminating global chain pointers
4412                 ** in favor of what we are doing here.  -jfv
4413                 */
4414                 if (!eop) {
4415                         /*
4416                         ** Figure out the next descriptor
4417                         ** of this frame.
4418                         */
4419                         if (rxr->hw_rsc == TRUE) {
4420                                 rsc = ixgbe_rsc_count(cur);
4421                                 rxr->rsc_num += (rsc - 1);
4422                         }
4423                         if (rsc) { /* Get hardware index */
4424                                 nextp = ((staterr &
4425                                     IXGBE_RXDADV_NEXTP_MASK) >>
4426                                     IXGBE_RXDADV_NEXTP_SHIFT);
4427                         } else { /* Just sequential */
4428                                 nextp = i + 1;
4429                                 if (nextp == adapter->num_rx_desc)
4430                                         nextp = 0;
4431                         }
4432                         nbuf = &rxr->rx_buffers[nextp];
4433                         prefetch(nbuf);
4434                 }
4435                 /*
4436                 ** The header mbuf is ONLY used when header
4437                 ** split is enabled, otherwise we get normal
4438                 ** behavior, ie, both header and payload
4439                 ** are DMA'd into the payload buffer.
4440                 **
4441                 ** Rather than using the fmp/lmp global pointers
4442                 ** we now keep the head of a packet chain in the
4443                 ** buffer struct and pass this along from one
4444                 ** descriptor to the next, until we get EOP.
4445                 */
4446                 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4447                         /* This must be an initial descriptor */
4448                         hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4449                             IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4450                         if (hlen > IXGBE_RX_HDR)
4451                                 hlen = IXGBE_RX_HDR;
4452                         mh->m_len = hlen;
4453                         mh->m_flags |= M_PKTHDR;
4454                         mh->m_next = NULL;
4455                         mh->m_pkthdr.len = mh->m_len;
4456                         /* Null buf pointer so it is refreshed */
4457                         rbuf->m_head = NULL;
4458                         /*
4459                         ** Check the payload length, this
4460                         ** could be zero if its a small
4461                         ** packet.
4462                         */
4463                         if (plen > 0) {
4464                                 mp->m_len = plen;
4465                                 mp->m_next = NULL;
4466                                 mp->m_flags &= ~M_PKTHDR;
4467                                 mh->m_next = mp;
4468                                 mh->m_pkthdr.len += mp->m_len;
4469                                 /* Null buf pointer so it is refreshed */
4470                                 rbuf->m_pack = NULL;
4471                                 rxr->rx_split_packets++;
4472                         }
4473                         /*
4474                         ** Now create the forward
4475                         ** chain so when complete
4476                         ** we wont have to.
4477                         */
4478                         if (eop == 0) {
4479                                 /* stash the chain head */
4480                                 nbuf->fmp = mh;
4481                                 /* Make forward chain */
4482                                 if (plen)
4483                                         mp->m_next = nbuf->m_pack;
4484                                 else
4485                                         mh->m_next = nbuf->m_pack;
4486                         } else {
4487                                 /* Singlet, prepare to send */
4488                                 sendmp = mh;
4489                                 /* If hardware handled vtag */
4490                                 if (vtag) {
4491                                         sendmp->m_pkthdr.ether_vlantag = vtag;
4492                                         sendmp->m_flags |= M_VLANTAG;
4493                                 }
4494                         }
4495                 } else {
4496                         /*
4497                         ** Either no header split, or a
4498                         ** secondary piece of a fragmented
4499                         ** split packet.
4500                         */
4501                         mp->m_len = plen;
4502                         /*
4503                         ** See if there is a stored head
4504                         ** that determines what we are
4505                         */
4506                         sendmp = rbuf->fmp;
4507                         rbuf->m_pack = rbuf->fmp = NULL;
4508
4509                         if (sendmp != NULL) {  /* secondary frag */
4510                                 mp->m_flags &= ~M_PKTHDR;
4511                                 sendmp->m_pkthdr.len += mp->m_len;
4512                         } else {
4513                                 /* first desc of a non-ps chain */
4514                                 sendmp = mp;
4515                                 sendmp->m_flags |= M_PKTHDR;
4516                                 sendmp->m_pkthdr.len = mp->m_len;
4517                                 if (staterr & IXGBE_RXD_STAT_VP) {
4518                                         sendmp->m_pkthdr.ether_vlantag = vtag;
4519                                         sendmp->m_flags |= M_VLANTAG;
4520                                 }
4521                         }
4522                         /* Pass the head pointer on */
4523                         if (eop == 0) {
4524                                 nbuf->fmp = sendmp;
4525                                 sendmp = NULL;
4526                                 mp->m_next = nbuf->m_pack;
4527                         }
4528                 }
4529                 ++processed;
4530                 /* Sending this frame? */
4531                 if (eop) {
4532                         sendmp->m_pkthdr.rcvif = ifp;
4533                         ifp->if_ipackets++;
4534                         rxr->rx_packets++;
4535                         /* capture data for AIM */
4536                         rxr->bytes += sendmp->m_pkthdr.len;
4537                         rxr->rx_bytes += sendmp->m_pkthdr.len;
4538                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4539                                 ixgbe_rx_checksum(staterr, sendmp, ptype);
4540 #if 0 /* __FreeBSD_version >= 800000 */
4541                         sendmp->m_pkthdr.flowid = que->msix;
4542                         sendmp->m_flags |= M_FLOWID;
4543 #endif
4544                 }
4545 next_desc:
4546                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4547                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4548
4549                 /* Advance our pointers to the next descriptor. */
4550                 if (++i == adapter->num_rx_desc)
4551                         i = 0;
4552
4553                 /* Now send to the stack or do LRO */
4554                 if (sendmp != NULL) {
4555                         rxr->next_to_check = i;
4556                         ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4557                         i = rxr->next_to_check;
4558                 }
4559
4560                /* Every 8 descriptors we go to refresh mbufs */
4561                 if (processed == 8) {
4562                         ixgbe_refresh_mbufs(rxr, i);
4563                         processed = 0;
4564                 }
4565         }
4566
4567         /* Refresh any remaining buf structs */
4568         if (ixgbe_rx_unrefreshed(rxr))
4569                 ixgbe_refresh_mbufs(rxr, i);
4570
4571         rxr->next_to_check = i;
4572
4573         /*
4574          * Flush any outstanding LRO work
4575          */
4576 #if 0   /* NET_LRO */
4577         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4578                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4579                 tcp_lro_flush(lro, queued);
4580         }
4581 #endif
4582
4583         IXGBE_RX_UNLOCK(rxr);
4584
4585         /*
4586         ** We still have cleaning to do?
4587         ** Schedule another interrupt if so.
4588         */
4589         if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4590                 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4591         }
4592 }
4593
4594
4595 /*********************************************************************
4596  *
4597  *  Verify that the hardware indicated that the checksum is valid.
4598  *  Inform the stack about the status of checksum so that stack
4599  *  doesn't spend time verifying the checksum.
4600  *
4601  *********************************************************************/
4602 static void
4603 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4604 {
4605         u16     status = (u16) staterr;
4606         u8      errors = (u8) (staterr >> 24);
4607         bool    sctp = FALSE;
4608
4609         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4610             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4611                 sctp = TRUE;
4612
4613         if (status & IXGBE_RXD_STAT_IPCS) {
4614                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4615                         /* IP Checksum Good */
4616                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4617                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4618
4619                 } else
4620                         mp->m_pkthdr.csum_flags = 0;
4621         }
4622         if (status & IXGBE_RXD_STAT_L4CS) {
4623                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4624 #if 0
4625                 if (sctp)
4626                         type = CSUM_SCTP_VALID;
4627 #endif
4628                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4629                         mp->m_pkthdr.csum_flags |= type;
4630                         if (!sctp)
4631                                 mp->m_pkthdr.csum_data = htons(0xffff);
4632                 }
4633         }
4634         return;
4635 }
4636
4637
4638 /*
4639 ** This routine is run via an vlan config EVENT,
4640 ** it enables us to use the HW Filter table since
4641 ** we can get the vlan id. This just creates the
4642 ** entry in the soft version of the VFTA, init will
4643 ** repopulate the real table.
4644 */
4645 static void
4646 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4647 {
4648         struct adapter  *adapter = ifp->if_softc;
4649         u16             index, bit;
4650
4651         if (ifp->if_softc !=  arg)   /* Not our event */
4652                 return;
4653
4654         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4655                 return;
4656
4657         IXGBE_CORE_LOCK(adapter);
4658         index = (vtag >> 5) & 0x7F;
4659         bit = vtag & 0x1F;
4660         adapter->shadow_vfta[index] |= (1 << bit);
4661         ++adapter->num_vlans;
4662         ixgbe_init_locked(adapter);
4663         IXGBE_CORE_UNLOCK(adapter);
4664 }
4665
4666 /*
4667 ** This routine is run via an vlan
4668 ** unconfig EVENT, remove our entry
4669 ** in the soft vfta.
4670 */
4671 static void
4672 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4673 {
4674         struct adapter  *adapter = ifp->if_softc;
4675         u16             index, bit;
4676
4677         if (ifp->if_softc !=  arg)
4678                 return;
4679
4680         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4681                 return;
4682
4683         IXGBE_CORE_LOCK(adapter);
4684         index = (vtag >> 5) & 0x7F;
4685         bit = vtag & 0x1F;
4686         adapter->shadow_vfta[index] &= ~(1 << bit);
4687         --adapter->num_vlans;
4688         /* Re-init to load the changes */
4689         ixgbe_init_locked(adapter);
4690         IXGBE_CORE_UNLOCK(adapter);
4691 }
4692
4693 static void
4694 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4695 {
4696         struct ifnet    *ifp = adapter->ifp;
4697         struct ixgbe_hw *hw = &adapter->hw;
4698         struct rx_ring  *rxr;
4699         u32             ctrl;
4700
4701         /*
4702         ** We get here thru init_locked, meaning
4703         ** a soft reset, this has already cleared
4704         ** the VFTA and other state, so if there
4705         ** have been no vlan's registered do nothing.
4706         */
4707         if (adapter->num_vlans == 0)
4708                 return;
4709
4710         /*
4711         ** A soft reset zero's out the VFTA, so
4712         ** we need to repopulate it now.
4713         */
4714         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4715                 if (adapter->shadow_vfta[i] != 0)
4716                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4717                             adapter->shadow_vfta[i]);
4718
4719         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4720         /* Enable the Filter Table if enabled */
4721         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4722                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4723                 ctrl |= IXGBE_VLNCTRL_VFE;
4724         }
4725         if (hw->mac.type == ixgbe_mac_82598EB)
4726                 ctrl |= IXGBE_VLNCTRL_VME;
4727         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4728
4729         /* Setup the queues for vlans */
4730         for (int i = 0; i < adapter->num_queues; i++) {
4731                 rxr = &adapter->rx_rings[i];
4732                 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4733                 if (hw->mac.type != ixgbe_mac_82598EB) {
4734                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4735                         ctrl |= IXGBE_RXDCTL_VME;
4736                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4737                 }
4738                 rxr->vtag_strip = TRUE;
4739         }
4740 }
4741
4742 static void
4743 ixgbe_enable_intr(struct adapter *adapter)
4744 {
4745         struct ixgbe_hw *hw = &adapter->hw;
4746         struct ix_queue *que = adapter->queues;
4747         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4748
4749
4750         /* Enable Fan Failure detection */
4751         if (hw->device_id == IXGBE_DEV_ID_82598AT)
4752                     mask |= IXGBE_EIMS_GPI_SDP1;
4753         else {
4754                     mask |= IXGBE_EIMS_ECC;
4755                     mask |= IXGBE_EIMS_GPI_SDP0;
4756                     mask |= IXGBE_EIMS_GPI_SDP1;
4757                     mask |= IXGBE_EIMS_GPI_SDP2;
4758 #ifdef IXGBE_FDIR
4759                     mask |= IXGBE_EIMS_FLOW_DIR;
4760 #endif
4761         }
4762
4763         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4764
4765         /* With RSS we use auto clear */
4766         if (adapter->msix_mem) {
4767                 mask = IXGBE_EIMS_ENABLE_MASK;
4768                 /* Don't autoclear Link */
4769                 mask &= ~IXGBE_EIMS_OTHER;
4770                 mask &= ~IXGBE_EIMS_LSC;
4771                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4772         }
4773
4774         /*
4775         ** Now enable all queues, this is done separately to
4776         ** allow for handling the extended (beyond 32) MSIX
4777         ** vectors that can be used by 82599
4778         */
4779         for (int i = 0; i < adapter->num_queues; i++, que++)
4780                 ixgbe_enable_queue(adapter, que->msix);
4781
4782         IXGBE_WRITE_FLUSH(hw);
4783
4784         return;
4785 }
4786
4787 static void
4788 ixgbe_disable_intr(struct adapter *adapter)
4789 {
4790         if (adapter->msix_mem)
4791                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4792         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4793                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4794         } else {
4795                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4796                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4797                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4798         }
4799         IXGBE_WRITE_FLUSH(&adapter->hw);
4800         return;
4801 }
4802
4803 u16
4804 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4805 {
4806         u16 value;
4807
4808         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4809             reg, 2);
4810
4811         return (value);
4812 }
4813
4814 void
4815 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4816 {
4817         pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4818             reg, value, 2);
4819
4820         return;
4821 }
4822
4823 /*
4824 ** Setup the correct IVAR register for a particular MSIX interrupt
4825 **   (yes this is all very magic and confusing :)
4826 **  - entry is the register array entry
4827 **  - vector is the MSIX vector for this queue
4828 **  - type is RX/TX/MISC
4829 */
4830 static void
4831 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4832 {
4833         struct ixgbe_hw *hw = &adapter->hw;
4834         u32 ivar, index;
4835
4836         vector |= IXGBE_IVAR_ALLOC_VAL;
4837
4838         switch (hw->mac.type) {
4839
4840         case ixgbe_mac_82598EB:
4841                 if (type == -1)
4842                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4843                 else
4844                         entry += (type * 64);
4845                 index = (entry >> 2) & 0x1F;
4846                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4847                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4848                 ivar |= (vector << (8 * (entry & 0x3)));
4849                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
4850                 break;
4851
4852         case ixgbe_mac_82599EB:
4853         case ixgbe_mac_X540:
4854                 if (type == -1) { /* MISC IVAR */
4855                         index = (entry & 1) * 8;
4856                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4857                         ivar &= ~(0xFF << index);
4858                         ivar |= (vector << index);
4859                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4860                 } else {        /* RX/TX IVARS */
4861                         index = (16 * (entry & 1)) + (8 * type);
4862                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4863                         ivar &= ~(0xFF << index);
4864                         ivar |= (vector << index);
4865                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4866                 }
4867
4868         default:
4869                 break;
4870         }
4871 }
4872
4873 static void
4874 ixgbe_configure_ivars(struct adapter *adapter)
4875 {
4876         struct  ix_queue *que = adapter->queues;
4877
4878         for (int i = 0; i < adapter->num_queues; i++, que++) {
4879                 /* First the RX queue entry */
4880                 ixgbe_set_ivar(adapter, i, que->msix, 0);
4881                 /* ... and the TX */
4882                 ixgbe_set_ivar(adapter, i, que->msix, 1);
4883                 /* Set an Initial EITR value */
4884                 ixgbe_set_eitr(adapter, que->msix, IXGBE_INTR_RATE);
4885         }
4886
4887         /* For the Link interrupt */
4888         ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
4889 }
4890
4891 /*
4892 ** ixgbe_sfp_probe - called in the local timer to
4893 ** determine if a port had optics inserted.
4894 */
4895 static bool ixgbe_sfp_probe(struct adapter *adapter)
4896 {
4897         struct ixgbe_hw *hw = &adapter->hw;
4898         device_t        dev = adapter->dev;
4899         bool            result = FALSE;
4900
4901         if ((hw->phy.type == ixgbe_phy_nl) &&
4902             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
4903                 s32 ret = hw->phy.ops.identify_sfp(hw);
4904                 if (ret)
4905                         goto out;
4906                 ret = hw->phy.ops.reset(hw);
4907                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4908                         device_printf(dev,"Unsupported SFP+ module detected!");
4909                         kprintf(" Reload driver with supported module.\n");
4910                         adapter->sfp_probe = FALSE;
4911                         goto out;
4912                 } else
4913                         device_printf(dev,"SFP+ module detected!\n");
4914                 /* We now have supported optics */
4915                 adapter->sfp_probe = FALSE;
4916                 /* Set the optics type so system reports correctly */
4917                 ixgbe_setup_optics(adapter);
4918                 result = TRUE;
4919         }
4920 out:
4921         return (result);
4922 }
4923
4924 /*
4925 ** Tasklet for handling SFP module interrupts
4926 */
4927 static void
4928 ixgbe_handle_mod(void *context, int pending)
4929 {
4930         struct adapter  *adapter = context;
4931         struct ixgbe_hw *hw = &adapter->hw;
4932         device_t        dev = adapter->dev;
4933         u32 err;
4934
4935         err = hw->phy.ops.identify_sfp(hw);
4936         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4937                 device_printf(dev,
4938                     "Unsupported SFP+ module type was detected.\n");
4939                 return;
4940         }
4941         err = hw->mac.ops.setup_sfp(hw);
4942         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4943                 device_printf(dev,
4944                     "Setup failure - unsupported SFP+ module type.\n");
4945                 return;
4946         }
4947         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
4948         return;
4949 }
4950
4951
4952 /*
4953 ** Tasklet for handling MSF (multispeed fiber) interrupts
4954 */
4955 static void
4956 ixgbe_handle_msf(void *context, int pending)
4957 {
4958         struct adapter  *adapter = context;
4959         struct ixgbe_hw *hw = &adapter->hw;
4960         u32 autoneg;
4961         bool negotiate;
4962
4963         autoneg = hw->phy.autoneg_advertised;
4964         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
4965                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
4966         if (hw->mac.ops.setup_link)
4967                 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
4968         return;
4969 }
4970
4971 #ifdef IXGBE_FDIR
4972 /*
4973 ** Tasklet for reinitializing the Flow Director filter table
4974 */
4975 static void
4976 ixgbe_reinit_fdir(void *context, int pending)
4977 {
4978         struct adapter  *adapter = context;
4979         struct ifnet   *ifp = adapter->ifp;
4980
4981         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
4982                 return;
4983         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
4984         adapter->fdir_reinit = 0;
4985         /* re-enable flow director interrupts */
4986         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
4987         /* Restart the interface */
4988         ifp->if_drv_flags |= IFF_DRV_RUNNING;
4989         return;
4990 }
4991 #endif
4992
4993 /**********************************************************************
4994  *
4995  *  Update the board statistics counters.
4996  *
4997  **********************************************************************/
4998 static void
4999 ixgbe_update_stats_counters(struct adapter *adapter)
5000 {
5001         struct ifnet   *ifp = adapter->ifp;
5002         struct ixgbe_hw *hw = &adapter->hw;
5003         u32  missed_rx = 0, bprc, lxon, lxoff, total;
5004         u64  total_missed_rx = 0;
5005
5006         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5007         adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5008         adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5009         adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5010
5011         for (int i = 0; i < 8; i++) {
5012                 u32 mp;
5013                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5014                 /* missed_rx tallies misses for the gprc workaround */
5015                 missed_rx += mp;
5016                 /* global total per queue */
5017                 adapter->stats.mpc[i] += mp;
5018                 /* Running comprehensive total for stats display */
5019                 total_missed_rx += adapter->stats.mpc[i];
5020                 if (hw->mac.type == ixgbe_mac_82598EB)
5021                         adapter->stats.rnbc[i] +=
5022                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5023                 adapter->stats.pxontxc[i] +=
5024                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5025                 adapter->stats.pxonrxc[i] +=
5026                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5027                 adapter->stats.pxofftxc[i] +=
5028                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5029                 adapter->stats.pxoffrxc[i] +=
5030                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5031                 adapter->stats.pxon2offc[i] +=
5032                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5033         }
5034         for (int i = 0; i < 16; i++) {
5035                 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5036                 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5037                 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5038                 adapter->stats.qbrc[i] +=
5039                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5040                 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5041                 adapter->stats.qbtc[i] +=
5042                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5043                 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5044         }
5045         adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5046         adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5047         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5048
5049         /* Hardware workaround, gprc counts missed packets */
5050         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5051         adapter->stats.gprc -= missed_rx;
5052
5053         if (hw->mac.type != ixgbe_mac_82598EB) {
5054                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5055                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5056                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5057                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5058                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5059                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5060                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5061                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5062         } else {
5063                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5064                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5065                 /* 82598 only has a counter in the high register */
5066                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5067                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5068                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5069         }
5070
5071         /*
5072          * Workaround: mprc hardware is incorrectly counting
5073          * broadcasts, so for now we subtract those.
5074          */
5075         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5076         adapter->stats.bprc += bprc;
5077         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5078         if (hw->mac.type == ixgbe_mac_82598EB)
5079                 adapter->stats.mprc -= bprc;
5080
5081         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5082         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5083         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5084         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5085         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5086         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5087
5088         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5089         adapter->stats.lxontxc += lxon;
5090         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5091         adapter->stats.lxofftxc += lxoff;
5092         total = lxon + lxoff;
5093
5094         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5095         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5096         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5097         adapter->stats.gptc -= total;
5098         adapter->stats.mptc -= total;
5099         adapter->stats.ptc64 -= total;
5100         adapter->stats.gotc -= total * ETHER_MIN_LEN;
5101
5102         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5103         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5104         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5105         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5106         adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5107         adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5108         adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5109         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5110         adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5111         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5112         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5113         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5114         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5115         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5116         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5117         adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5118         adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5119         adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5120         /* Only read FCOE on 82599 */
5121         if (hw->mac.type != ixgbe_mac_82598EB) {
5122                 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5123                 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5124                 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5125                 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5126                 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5127         }
5128
5129         /* Fill out the OS statistics structure */
5130         ifp->if_ipackets = adapter->stats.gprc;
5131         ifp->if_opackets = adapter->stats.gptc;
5132         ifp->if_ibytes = adapter->stats.gorc;
5133         ifp->if_obytes = adapter->stats.gotc;
5134         ifp->if_imcasts = adapter->stats.mprc;
5135         ifp->if_collisions = 0;
5136
5137         /* Rx Errors */
5138         ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs +
5139                 adapter->stats.rlec;
5140 }
5141
5142 /** ixgbe_sysctl_tdh_handler - Handler function
5143  *  Retrieves the TDH value from the hardware
5144  */
5145 static int
5146 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5147 {
5148         int error;
5149
5150         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5151         if (!txr) return 0;
5152
5153         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5154         error = sysctl_handle_int(oidp, &val, 0, req);
5155         if (error || !req->newptr)
5156                 return error;
5157         return 0;
5158 }
5159
5160 /** ixgbe_sysctl_tdt_handler - Handler function
5161  *  Retrieves the TDT value from the hardware
5162  */
5163 static int
5164 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5165 {
5166         int error;
5167
5168         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5169         if (!txr) return 0;
5170
5171         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5172         error = sysctl_handle_int(oidp, &val, 0, req);
5173         if (error || !req->newptr)
5174                 return error;
5175         return 0;
5176 }
5177
5178 /** ixgbe_sysctl_rdh_handler - Handler function
5179  *  Retrieves the RDH value from the hardware
5180  */
5181 static int
5182 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5183 {
5184         int error;
5185
5186         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5187         if (!rxr) return 0;
5188
5189         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5190         error = sysctl_handle_int(oidp, &val, 0, req);
5191         if (error || !req->newptr)
5192                 return error;
5193         return 0;
5194 }
5195
5196 /** ixgbe_sysctl_rdt_handler - Handler function
5197  *  Retrieves the RDT value from the hardware
5198  */
5199 static int
5200 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5201 {
5202         int error;
5203
5204         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5205         if (!rxr) return 0;
5206
5207         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5208         error = sysctl_handle_int(oidp, &val, 0, req);
5209         if (error || !req->newptr)
5210                 return error;
5211         return 0;
5212 }
5213
5214 /*
5215  * Add sysctl variables, one per statistic, to the system.
5216  */
5217 static void
5218 ixgbe_add_hw_stats(struct adapter *adapter)
5219 {
5220         struct tx_ring *txr = adapter->tx_rings;
5221         struct rx_ring *rxr = adapter->rx_rings;
5222
5223         struct sysctl_ctx_list *ctx = &adapter->sysctl_ctx;
5224         struct sysctl_oid *tree = adapter->sysctl_tree;
5225         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5226         struct ixgbe_hw_stats *stats = &adapter->stats;
5227
5228         struct sysctl_oid *stat_node, *queue_node;
5229         struct sysctl_oid_list *stat_list, *queue_list;
5230
5231 #define QUEUE_NAME_LEN 32
5232         char namebuf[QUEUE_NAME_LEN];
5233
5234         /* Driver Statistics */
5235         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5236                         CTLFLAG_RD, &adapter->dropped_pkts,
5237                         "Driver dropped packets");
5238         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5239                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5240                         "m_defrag() failed");
5241         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup",
5242                         CTLFLAG_RD, &adapter->no_tx_dma_setup,
5243                         "Driver tx dma failure in xmit");
5244         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5245                         CTLFLAG_RD, &adapter->watchdog_events,
5246                         "Watchdog timeouts");
5247         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx",
5248                         CTLFLAG_RD, &adapter->tso_tx,
5249                         "TSO");
5250         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5251                         CTLFLAG_RD, &adapter->link_irq,
5252                         "Link MSIX IRQ Handled");
5253
5254         for (int i = 0; i < adapter->num_queues; i++, txr++) {
5255         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5256                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5257                                             CTLFLAG_RD, NULL, "Queue Name");
5258                 queue_list = SYSCTL_CHILDREN(queue_node);
5259                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5260                                 CTLFLAG_RD, &(adapter->queues[i].irqs), 0,
5261                                 "irqs on this queue");
5262                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5263                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5264                                 ixgbe_sysctl_tdh_handler, "IU",
5265                                 "Transmit Descriptor Head");
5266                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5267                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5268                                 ixgbe_sysctl_tdt_handler, "IU",
5269                                 "Transmit Descriptor Tail");
5270                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5271                                 CTLFLAG_RD, &txr->no_desc_avail, 0,
5272                                 "Queue No Descriptor Available");
5273                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5274                                 CTLFLAG_RD, &txr->total_packets, 0,
5275                                 "Queue Packets Transmitted");
5276         }
5277
5278         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5279         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5280                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5281                                             CTLFLAG_RD, NULL, "Queue Name");
5282                 queue_list = SYSCTL_CHILDREN(queue_node);
5283
5284 #if 0   /* NET_LRO */
5285                 struct lro_ctrl *lro = &rxr->lro;
5286 #endif
5287
5288         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5289                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5290                                             CTLFLAG_RD, NULL, "Queue Name");
5291                 queue_list = SYSCTL_CHILDREN(queue_node);
5292
5293                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5294                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5295                                 ixgbe_sysctl_rdh_handler, "IU",
5296                                 "Receive Descriptor Head");
5297                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5298                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5299                                 ixgbe_sysctl_rdt_handler, "IU",
5300                                 "Receive Descriptor Tail");
5301                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5302                                 CTLFLAG_RD, &rxr->rx_packets, 0,
5303                                 "Queue Packets Received");
5304                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5305                                 CTLFLAG_RD, &rxr->rx_bytes, 0,
5306                                 "Queue Bytes Received");
5307 #if 0   /* NET_LRO */
5308                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5309                                 CTLFLAG_RD, &lro->lro_queued, 0,
5310                                 "LRO Queued");
5311                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5312                                 CTLFLAG_RD, &lro->lro_flushed, 0,
5313                                 "LRO Flushed");
5314 #endif
5315         }
5316
5317         /* MAC stats get the own sub node */
5318
5319         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5320                                     CTLFLAG_RD, NULL, "MAC Statistics");
5321         stat_list = SYSCTL_CHILDREN(stat_node);
5322
5323         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5324                         CTLFLAG_RD, &stats->crcerrs, 0,
5325                         "CRC Errors");
5326         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5327                         CTLFLAG_RD, &stats->illerrc, 0,
5328                         "Illegal Byte Errors");
5329         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5330                         CTLFLAG_RD, &stats->errbc, 0,
5331                         "Byte Errors");
5332         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5333                         CTLFLAG_RD, &stats->mspdc, 0,
5334                         "MAC Short Packets Discarded");
5335         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5336                         CTLFLAG_RD, &stats->mlfc, 0,
5337                         "MAC Local Faults");
5338         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5339                         CTLFLAG_RD, &stats->mrfc, 0,
5340                         "MAC Remote Faults");
5341         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5342                         CTLFLAG_RD, &stats->rlec, 0,
5343                         "Receive Length Errors");
5344         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd",
5345                         CTLFLAG_RD, &stats->lxontxc, 0,
5346                         "Link XON Transmitted");
5347         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd",
5348                         CTLFLAG_RD, &stats->lxonrxc, 0,
5349                         "Link XON Received");
5350         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd",
5351                         CTLFLAG_RD, &stats->lxofftxc, 0,
5352                         "Link XOFF Transmitted");
5353         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd",
5354                         CTLFLAG_RD, &stats->lxoffrxc, 0,
5355                         "Link XOFF Received");
5356
5357         /* Packet Reception Stats */
5358         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5359                         CTLFLAG_RD, &stats->tor, 0,
5360                         "Total Octets Received");
5361         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5362                         CTLFLAG_RD, &stats->gorc, 0,
5363                         "Good Octets Received");
5364         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5365                         CTLFLAG_RD, &stats->tpr, 0,
5366                         "Total Packets Received");
5367         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5368                         CTLFLAG_RD, &stats->gprc, 0,
5369                         "Good Packets Received");
5370         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5371                         CTLFLAG_RD, &stats->mprc, 0,
5372                         "Multicast Packets Received");
5373         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5374                         CTLFLAG_RD, &stats->bprc, 0,
5375                         "Broadcast Packets Received");
5376         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5377                         CTLFLAG_RD, &stats->prc64, 0,
5378                         "64 byte frames received ");
5379         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5380                         CTLFLAG_RD, &stats->prc127, 0,
5381                         "65-127 byte frames received");
5382         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5383                         CTLFLAG_RD, &stats->prc255, 0,
5384                         "128-255 byte frames received");
5385         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5386                         CTLFLAG_RD, &stats->prc511, 0,
5387                         "256-511 byte frames received");
5388         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5389                         CTLFLAG_RD, &stats->prc1023, 0,
5390                         "512-1023 byte frames received");
5391         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5392                         CTLFLAG_RD, &stats->prc1522, 0,
5393                         "1023-1522 byte frames received");
5394         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5395                         CTLFLAG_RD, &stats->ruc, 0,
5396                         "Receive Undersized");
5397         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5398                         CTLFLAG_RD, &stats->rfc, 0,
5399                         "Fragmented Packets Received ");
5400         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5401                         CTLFLAG_RD, &stats->roc, 0,
5402                         "Oversized Packets Received");
5403         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5404                         CTLFLAG_RD, &stats->rjc, 0,
5405                         "Received Jabber");
5406         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5407                         CTLFLAG_RD, &stats->mngprc, 0,
5408                         "Management Packets Received");
5409         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5410                         CTLFLAG_RD, &stats->mngptc, 0,
5411                         "Management Packets Dropped");
5412         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5413                         CTLFLAG_RD, &stats->xec, 0,
5414                         "Checksum Errors");
5415
5416         /* Packet Transmission Stats */
5417         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5418                         CTLFLAG_RD, &stats->gotc, 0,
5419                         "Good Octets Transmitted");
5420         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5421                         CTLFLAG_RD, &stats->tpt, 0,
5422                         "Total Packets Transmitted");
5423         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5424                         CTLFLAG_RD, &stats->gptc, 0,
5425                         "Good Packets Transmitted");
5426         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5427                         CTLFLAG_RD, &stats->bptc, 0,
5428                         "Broadcast Packets Transmitted");
5429         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5430                         CTLFLAG_RD, &stats->mptc, 0,
5431                         "Multicast Packets Transmitted");
5432         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5433                         CTLFLAG_RD, &stats->mngptc, 0,
5434                         "Management Packets Transmitted");
5435         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5436                         CTLFLAG_RD, &stats->ptc64, 0,
5437                         "64 byte frames transmitted ");
5438         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5439                         CTLFLAG_RD, &stats->ptc127, 0,
5440                         "65-127 byte frames transmitted");
5441         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5442                         CTLFLAG_RD, &stats->ptc255, 0,
5443                         "128-255 byte frames transmitted");
5444         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5445                         CTLFLAG_RD, &stats->ptc511, 0,
5446                         "256-511 byte frames transmitted");
5447         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5448                         CTLFLAG_RD, &stats->ptc1023, 0,
5449                         "512-1023 byte frames transmitted");
5450         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5451                         CTLFLAG_RD, &stats->ptc1522, 0,
5452                         "1024-1522 byte frames transmitted");
5453
5454         /* FC Stats */
5455         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc",
5456                 CTLFLAG_RD, &stats->fccrc, 0,
5457                 "FC CRC Errors");
5458         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last",
5459                 CTLFLAG_RD, &stats->fclast, 0,
5460                 "FC Last Error");
5461         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd",
5462                 CTLFLAG_RD, &stats->fcoerpdc, 0,
5463                 "FCoE Packets Dropped");
5464         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd",
5465                 CTLFLAG_RD, &stats->fcoeprc, 0,
5466                 "FCoE Packets Received");
5467         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd",
5468                 CTLFLAG_RD, &stats->fcoeptc, 0,
5469                 "FCoE Packets Transmitted");
5470         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd",
5471                 CTLFLAG_RD, &stats->fcoedwrc, 0,
5472                 "FCoE DWords Received");
5473         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd",
5474                 CTLFLAG_RD, &stats->fcoedwtc, 0,
5475                 "FCoE DWords Transmitted");
5476 }
5477
5478 /*
5479 ** Set flow control using sysctl:
5480 ** Flow control values:
5481 **      0 - off
5482 **      1 - rx pause
5483 **      2 - tx pause
5484 **      3 - full
5485 */
5486 static int
5487 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5488 {
5489         int error, last;
5490         struct adapter *adapter = (struct adapter *) arg1;
5491
5492         last = adapter->fc;
5493         error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5494         if ((error) || (req->newptr == NULL))
5495                 return (error);
5496
5497         /* Don't bother if it's not changed */
5498         if (adapter->fc == last)
5499                 return (0);
5500
5501         switch (adapter->fc) {
5502                 case ixgbe_fc_rx_pause:
5503                 case ixgbe_fc_tx_pause:
5504                 case ixgbe_fc_full:
5505                         adapter->hw.fc.requested_mode = adapter->fc;
5506                         break;
5507                 case ixgbe_fc_none:
5508                 default:
5509                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
5510         }
5511         /* Don't autoneg if forcing a value */
5512         adapter->hw.fc.disable_fc_autoneg = TRUE;
5513         ixgbe_fc_enable(&adapter->hw);
5514         return error;
5515 }
5516
5517 static void
5518 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5519         const char *description, int *limit, int value)
5520 {
5521         *limit = value;
5522         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
5523             SYSCTL_CHILDREN(adapter->sysctl_tree),
5524             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5525 }
5526
5527 /*
5528 ** Control link advertise speed:
5529 **      1 - advertise only 1G
5530 **      2 - advertise 100Mb
5531 **      3 - advertise normal
5532 */
5533 static int
5534 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5535 {
5536         int                     error = 0;
5537         struct adapter          *adapter;
5538         device_t                dev;
5539         struct ixgbe_hw         *hw;
5540         ixgbe_link_speed        speed, last;
5541
5542         adapter = (struct adapter *) arg1;
5543         dev = adapter->dev;
5544         hw = &adapter->hw;
5545         last = adapter->advertise;
5546
5547         error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5548         if ((error) || (adapter->advertise == -1))
5549                 return (error);
5550
5551         if (adapter->advertise == last) /* no change */
5552                 return (0);
5553
5554         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5555             (hw->phy.multispeed_fiber)))
5556                 return (error);
5557
5558         if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5559                 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5560                 return (error);
5561         }
5562
5563         if (adapter->advertise == 1)
5564                 speed = IXGBE_LINK_SPEED_1GB_FULL;
5565         else if (adapter->advertise == 2)
5566                 speed = IXGBE_LINK_SPEED_100_FULL;
5567         else if (adapter->advertise == 3)
5568                 speed = IXGBE_LINK_SPEED_1GB_FULL |
5569                         IXGBE_LINK_SPEED_10GB_FULL;
5570         else /* bogus value */
5571                 return (error);
5572
5573         hw->mac.autotry_restart = TRUE;
5574         hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5575
5576         return (error);
5577 }
5578
5579 /*
5580 ** Thermal Shutdown Trigger
5581 **   - cause a Thermal Overtemp IRQ
5582 */
5583 static int
5584 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5585 {
5586         int             error, fire = 0;
5587         struct adapter  *adapter = (struct adapter *) arg1;
5588         struct ixgbe_hw *hw = &adapter->hw;
5589
5590
5591         if (hw->mac.type != ixgbe_mac_X540)
5592                 return (0);
5593
5594         error = sysctl_handle_int(oidp, &fire, 0, req);
5595         if ((error) || (req->newptr == NULL))
5596                 return (error);
5597
5598         if (fire) {
5599                 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5600                 reg |= IXGBE_EICR_TS;
5601                 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5602         }
5603
5604         return (0);
5605 }
5606
5607 static void
5608 ixgbe_set_eitr(struct adapter *sc, int idx, int rate)
5609 {
5610         uint32_t eitr = 0;
5611
5612         /* convert rate in intr/s to hw representation */
5613         if (rate > 0) {
5614                 eitr = 1000000 / rate;
5615                 eitr <<= IXGBE_EITR_INTVL_SHIFT;
5616
5617                 if (eitr == 0) {
5618                         /* Don't disable it */
5619                         eitr = 1 << IXGBE_EITR_INTVL_SHIFT;
5620                 } else if (eitr > IXGBE_EITR_INTVL_MASK) {
5621                         /* Don't allow it to be too large */
5622                         eitr = IXGBE_EITR_INTVL_MASK;
5623                 }
5624         }
5625         IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(idx), eitr);
5626 }
5627
5628 static int
5629 ixgbe_sysctl_intr_rate(SYSCTL_HANDLER_ARGS)
5630 {
5631         struct adapter *sc = (void *)arg1;
5632         struct ifnet *ifp = sc->ifp;
5633         int error, intr_rate, running;
5634         struct ix_queue *que = sc->queues;
5635
5636         intr_rate = sc->intr_rate;
5637         error = sysctl_handle_int(oidp, &intr_rate, 0, req);
5638         if (error || req->newptr == NULL)
5639                 return error;
5640         if (intr_rate < 0)
5641                 return EINVAL;
5642
5643         ifnet_serialize_all(ifp);
5644
5645         sc->intr_rate = intr_rate;
5646         running = ifp->if_flags & IFF_RUNNING;
5647         if (running)
5648                 ixgbe_set_eitr(sc, 0, sc->intr_rate);
5649
5650         if (running && (sc->intr_type == PCI_INTR_TYPE_MSIX)) {
5651                 for (int i = 0; i < sc->num_queues; i++, que++)
5652                         ixgbe_set_eitr(sc, que->msix, sc->intr_rate);
5653         }
5654
5655         if (bootverbose)
5656                 if_printf(ifp, "interrupt rate set to %d/sec\n", sc->intr_rate);
5657
5658         ifnet_deserialize_all(ifp);
5659
5660         return 0;
5661 }
5662
5663 /* rearrange mbuf chain to get contiguous bytes */
5664 static int
5665 ixgbe_tso_pullup(struct tx_ring *txr, struct mbuf **mp)
5666 {
5667         int hoff, iphlen, thoff;
5668         struct mbuf *m;
5669
5670         m = *mp;
5671         KASSERT(M_WRITABLE(m), ("TSO mbuf not writable"));
5672
5673         iphlen = m->m_pkthdr.csum_iphlen;
5674         thoff = m->m_pkthdr.csum_thlen;
5675         hoff = m->m_pkthdr.csum_lhlen;
5676
5677         KASSERT(iphlen > 0, ("invalid ip hlen"));
5678         KASSERT(thoff > 0, ("invalid tcp hlen"));
5679         KASSERT(hoff > 0, ("invalid ether hlen"));
5680
5681         if (__predict_false(m->m_len < hoff + iphlen + thoff)) {
5682                 m = m_pullup(m, hoff + iphlen + thoff);
5683                 if (m == NULL) {
5684                         *mp = NULL;
5685                         return ENOBUFS;
5686                 }
5687                 *mp = m;
5688         }
5689
5690         return 0;
5691 }