sys/dev/netif/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2012, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
  34
  35 #include "opt_inet.h"
  36 #include "opt_inet6.h"
  37
  38 #include "ixgbe.h"
  39
  40 /*********************************************************************
  41  *  Set this to one to display debug statistics
  42  *********************************************************************/
  43 int             ixgbe_display_debug_stats = 0;
  44
  45 /*********************************************************************
  46  *  Driver version
  47  *********************************************************************/
  48 char ixgbe_driver_version[] = "2.4.8";
  49
  50 /*********************************************************************
  51  *  PCI Device ID Table
  52  *
  53  *  Used by probe to select devices to load on
  54  *  Last field stores an index into ixgbe_strings
  55  *  Last entry must be all 0s
  56  *
  57  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  58  *********************************************************************/
  59
  60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  61 {
  62         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
  81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
  82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
  83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
  84         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
  85         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
  86         /* required last entry */
  87         {0, 0, 0, 0, 0}
  88 };
  89
  90 /*********************************************************************
  91  *  Table of branding strings
  92  *********************************************************************/
  93
  94 static char    *ixgbe_strings[] = {
  95         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  96 };
  97
  98 /*********************************************************************
  99  *  Function prototypes
 100  *********************************************************************/
 101 static int      ixgbe_probe(device_t);
 102 static int      ixgbe_attach(device_t);
 103 static int      ixgbe_detach(device_t);
 104 static int      ixgbe_shutdown(device_t);
 105 static void     ixgbe_start(struct ifnet *, struct ifaltq_subque *);
 106 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
 107 #if 0 /* __FreeBSD_version >= 800000 */
 108 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
 109 static int      ixgbe_mq_start_locked(struct ifnet *,
 110                     struct tx_ring *, struct mbuf *);
 111 static void     ixgbe_qflush(struct ifnet *);
 112 #endif
 113 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 114 static void     ixgbe_init(void *);
 115 static void     ixgbe_init_locked(struct adapter *);
 116 static void     ixgbe_stop(void *);
 117 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 118 static int      ixgbe_media_change(struct ifnet *);
 119 static void     ixgbe_identify_hardware(struct adapter *);
 120 static int      ixgbe_allocate_pci_resources(struct adapter *);
 121 static int      ixgbe_allocate_msix(struct adapter *);
 122 static int      ixgbe_allocate_legacy(struct adapter *);
 123 static int      ixgbe_allocate_queues(struct adapter *);
 124 static int      ixgbe_setup_msix(struct adapter *);
 125 static void     ixgbe_free_pci_resources(struct adapter *);
 126 static void     ixgbe_local_timer(void *);
 127 static int      ixgbe_setup_interface(device_t, struct adapter *);
 128 static void     ixgbe_config_link(struct adapter *);
 129
 130 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 131 static int      ixgbe_setup_transmit_structures(struct adapter *);
 132 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 133 static void     ixgbe_initialize_transmit_units(struct adapter *);
 134 static void     ixgbe_free_transmit_structures(struct adapter *);
 135 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 136
 137 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 138 static int      ixgbe_setup_receive_structures(struct adapter *);
 139 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 140 static void     ixgbe_initialize_receive_units(struct adapter *);
 141 static void     ixgbe_free_receive_structures(struct adapter *);
 142 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 143 #if 0   /* NET_LRO */
 144 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
 145 #endif
 146
 147 static void     ixgbe_enable_intr(struct adapter *);
 148 static void     ixgbe_disable_intr(struct adapter *);
 149 static void     ixgbe_update_stats_counters(struct adapter *);
 150 static void     ixgbe_txeof(struct tx_ring *);
 151 static void     ixgbe_rxeof(struct ix_queue *, int);
 152 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
 153 static void     ixgbe_set_promisc(struct adapter *);
 154 static void     ixgbe_set_multi(struct adapter *);
 155 static void     ixgbe_update_link_status(struct adapter *);
 156 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
 157 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 158 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 159 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
 160 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
 161 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 162                     struct ixgbe_dma_alloc *, int);
 163 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 164 static void     ixgbe_add_rx_process_limit(struct adapter *, const char *,
 165                     const char *, int *, int);
 166 static bool     ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 167 static bool     ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
 168 static int      ixgbe_tso_pullup(struct tx_ring *, struct mbuf **);
 169 static void     ixgbe_add_sysctl(struct adapter *);
 170 static void     ixgbe_set_eitr(struct adapter *, int, int);
 171 static int      ixgbe_sysctl_intr_rate(SYSCTL_HANDLER_ARGS);
 172 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 173 static void     ixgbe_configure_ivars(struct adapter *);
 174 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 175
 176 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 177 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 178 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 179
 180 static void     ixgbe_add_hw_stats(struct adapter *adapter);
 181
 182 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
 183 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
 184                     struct mbuf *, u32);
 185
 186 /* Support for pluggable optic modules */
 187 static bool     ixgbe_sfp_probe(struct adapter *);
 188 static void     ixgbe_setup_optics(struct adapter *);
 189
 190 /* Legacy (single vector interrupt handler */
 191 static void     ixgbe_legacy_irq(void *);
 192
 193 /* The MSI/X Interrupt handlers */
 194 static void     ixgbe_msix_que(void *);
 195 static void     ixgbe_msix_link(void *);
 196
 197 /* Deferred interrupt tasklets */
 198 static void     ixgbe_handle_msf(void *, int);
 199 static void     ixgbe_handle_mod(void *, int);
 200
 201 #ifdef IXGBE_FDIR
 202 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
 203 static void     ixgbe_reinit_fdir(void *, int);
 204 #endif
 205
 206 /*********************************************************************
 207  *  FreeBSD Device Interface Entry Points
 208  *********************************************************************/
 209
 210 static device_method_t ixgbe_methods[] = {
 211         /* Device interface */
 212         DEVMETHOD(device_probe, ixgbe_probe),
 213         DEVMETHOD(device_attach, ixgbe_attach),
 214         DEVMETHOD(device_detach, ixgbe_detach),
 215         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 216         DEVMETHOD_END
 217 };
 218
 219 static driver_t ixgbe_driver = {
 220         "ix", ixgbe_methods, sizeof(struct adapter),
 221 };
 222
 223 devclass_t ixgbe_devclass;
 224 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, NULL, NULL);
 225
 226 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 227 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 228
 229 /*
 230 ** TUNEABLE PARAMETERS:
 231 */
 232
 233 /* How many packets rxeof tries to clean at a time */
 234 static int ixgbe_rx_process_limit = 128;
 235 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 236
 237 /*
 238 ** Smart speed setting, default to on
 239 ** this only works as a compile option
 240 ** right now as its during attach, set
 241 ** this to 'ixgbe_smart_speed_off' to
 242 ** disable.
 243 */
 244 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 245
 246 static int ixgbe_msi_enable = 1;
 247 TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
 248
 249 /*
 250  * MSIX should be the default for best performance,
 251  * but this allows it to be forced off for testing.
 252  */
 253 static int ixgbe_enable_msix = 1;
 254 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 255
 256 /*
 257  * Header split: this causes the hardware to DMA
 258  * the header into a separate mbuf from the payload,
 259  * it can be a performance win in some workloads, but
 260  * in others it actually hurts, its off by default.
 261  */
 262 static int ixgbe_header_split = FALSE;
 263 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
 264
 265 /*
 266  * Number of Queues, can be set to 0,
 267  * it then autoconfigures based on the
 268  * number of cpus with a max of 8. This
 269  * can be overriden manually here.
 270  */
 271 static int ixgbe_num_queues = 0;
 272 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
 273
 274 /*
 275 ** Number of TX descriptors per ring,
 276 ** setting higher than RX as this seems
 277 ** the better performing choice.
 278 */
 279 static int ixgbe_txd = PERFORM_TXD;
 280 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 281
 282 /* Number of RX descriptors per ring */
 283 static int ixgbe_rxd = PERFORM_RXD;
 284 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 285
 286 /* Keep running tab on them for sanity check */
 287 static int ixgbe_total_ports;
 288
 289 #ifdef IXGBE_FDIR
 290 /*
 291 ** For Flow Director: this is the
 292 ** number of TX packets we sample
 293 ** for the filter pool, this means
 294 ** every 20th packet will be probed.
 295 **
 296 ** This feature can be disabled by
 297 ** setting this to 0.
 298 */
 299 static int atr_sample_rate = 20;
 300 /*
 301 ** Flow Director actually 'steals'
 302 ** part of the packet buffer as its
 303 ** filter pool, this variable controls
 304 ** how much it uses:
 305 **  0 = 64K, 1 = 128K, 2 = 256K
 306 */
 307 static int fdir_pballoc = 1;
 308 #endif
 309
 310 #ifdef DEV_NETMAP
 311 /*
 312  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
 313  * be a reference on how to implement netmap support in a driver.
 314  * Additional comments are in ixgbe_netmap.h .
 315  *
 316  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
 317  * that extend the standard driver.
 318  */
 319 #include <dev/netmap/ixgbe_netmap.h>
 320 #endif /* DEV_NETMAP */
 321
 322 /*********************************************************************
 323  *  Device identification routine
 324  *
 325  *  ixgbe_probe determines if the driver should be loaded on
 326  *  adapter based on PCI vendor/device id of the adapter.
 327  *
 328  *  return BUS_PROBE_DEFAULT on success, positive on failure
 329  *********************************************************************/
 330
 331 static int
 332 ixgbe_probe(device_t dev)
 333 {
 334         ixgbe_vendor_info_t *ent;
 335
 336         u16     pci_vendor_id = 0;
 337         u16     pci_device_id = 0;
 338         u16     pci_subvendor_id = 0;
 339         u16     pci_subdevice_id = 0;
 340         char    adapter_name[256];
 341
 342         INIT_DEBUGOUT("ixgbe_probe: begin");
 343
 344         pci_vendor_id = pci_get_vendor(dev);
 345         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 346                 return (ENXIO);
 347
 348         pci_device_id = pci_get_device(dev);
 349         pci_subvendor_id = pci_get_subvendor(dev);
 350         pci_subdevice_id = pci_get_subdevice(dev);
 351
 352         ent = ixgbe_vendor_info_array;
 353         while (ent->vendor_id != 0) {
 354                 if ((pci_vendor_id == ent->vendor_id) &&
 355                     (pci_device_id == ent->device_id) &&
 356
 357                     ((pci_subvendor_id == ent->subvendor_id) ||
 358                      (ent->subvendor_id == 0)) &&
 359
 360                     ((pci_subdevice_id == ent->subdevice_id) ||
 361                      (ent->subdevice_id == 0))) {
 362                         ksprintf(adapter_name, "%s, Version - %s",
 363                                 ixgbe_strings[ent->index],
 364                                 ixgbe_driver_version);
 365                         device_set_desc_copy(dev, adapter_name);
 366                         ++ixgbe_total_ports;
 367                         return (BUS_PROBE_DEFAULT);
 368                 }
 369                 ent++;
 370         }
 371         return (ENXIO);
 372 }
 373
 374 /*********************************************************************
 375  *  Device initialization routine
 376  *
 377  *  The attach entry point is called when the driver is being loaded.
 378  *  This routine identifies the type of hardware, allocates all resources
 379  *  and initializes the hardware.
 380  *
 381  *  return 0 on success, positive on failure
 382  *********************************************************************/
 383
 384 static int
 385 ixgbe_attach(device_t dev)
 386 {
 387         struct adapter *adapter;
 388         struct ixgbe_hw *hw;
 389         int             error = 0;
 390         u16             csum;
 391         u32             ctrl_ext;
 392
 393         INIT_DEBUGOUT("ixgbe_attach: begin");
 394
 395         if (resource_disabled("ixgbe", device_get_unit(dev))) {
 396                 device_printf(dev, "Disabled by device hint\n");
 397                 return (ENXIO);
 398         }
 399
 400         /* Allocate, clear, and link in our adapter structure */
 401         adapter = device_get_softc(dev);
 402         adapter->dev = adapter->osdep.dev = dev;
 403         hw = &adapter->hw;
 404
 405         /* Core Lock Init*/
 406         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 407
 408         /* Set up the timer callout */
 409         callout_init_mp(&adapter->timer);
 410
 411         /* Determine hardware revision */
 412         ixgbe_identify_hardware(adapter);
 413
 414         /* Enable bus mastering */
 415         pci_enable_busmaster(dev);
 416
 417         /* Do base PCI setup - map BAR0 */
 418         if (ixgbe_allocate_pci_resources(adapter)) {
 419                 device_printf(dev, "Allocation of PCI resources failed\n");
 420                 error = ENXIO;
 421                 goto err_out;
 422         }
 423
 424         /* Do descriptor calc and sanity checks */
 425         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 426             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 427                 device_printf(dev, "TXD config issue, using default!\n");
 428                 adapter->num_tx_desc = DEFAULT_TXD;
 429         } else
 430                 adapter->num_tx_desc = ixgbe_txd;
 431
 432         /*
 433         ** With many RX rings it is easy to exceed the
 434         ** system mbuf allocation. Tuning nmbclusters
 435         ** can alleviate this.
 436         */
 437         if (nmbclusters > 0 ) {
 438                 int s;
 439                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 440                 if (s > nmbclusters) {
 441                         device_printf(dev, "RX Descriptors exceed "
 442                             "system mbuf max, using default instead!\n");
 443                         ixgbe_rxd = DEFAULT_RXD;
 444                 }
 445         }
 446
 447         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 448             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 449                 device_printf(dev, "RXD config issue, using default!\n");
 450                 adapter->num_rx_desc = DEFAULT_RXD;
 451         } else
 452                 adapter->num_rx_desc = ixgbe_rxd;
 453
 454         /* Allocate our TX/RX Queues */
 455         if (ixgbe_allocate_queues(adapter)) {
 456                 error = ENOMEM;
 457                 goto err_out;
 458         }
 459
 460         /* Allocate multicast array memory. */
 461         adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
 462             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 463         if (adapter->mta == NULL) {
 464                 device_printf(dev, "Can not allocate multicast setup array\n");
 465                 error = ENOMEM;
 466                 goto err_late;
 467         }
 468
 469         /* Initialize the shared code */
 470         error = ixgbe_init_shared_code(hw);
 471         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 472                 /*
 473                 ** No optics in this port, set up
 474                 ** so the timer routine will probe
 475                 ** for later insertion.
 476                 */
 477                 adapter->sfp_probe = TRUE;
 478                 error = 0;
 479         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 480                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 481                 error = EIO;
 482                 goto err_late;
 483         } else if (error) {
 484                 device_printf(dev,"Unable to initialize the shared code\n");
 485                 error = EIO;
 486                 goto err_late;
 487         }
 488
 489         /* Make sure we have a good EEPROM before we read from it */
 490         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 491                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 492                 error = EIO;
 493                 goto err_late;
 494         }
 495
 496         error = ixgbe_init_hw(hw);
 497         switch (error) {
 498         case IXGBE_ERR_EEPROM_VERSION:
 499                 device_printf(dev, "This device is a pre-production adapter/"
 500                     "LOM.  Please be aware there may be issues associated "
 501                     "with your hardware.\n If you are experiencing problems "
 502                     "please contact your Intel or hardware representative "
 503                     "who provided you with this hardware.\n");
 504                 break;
 505         case IXGBE_ERR_SFP_NOT_SUPPORTED:
 506                 device_printf(dev,"Unsupported SFP+ Module\n");
 507                 error = EIO;
 508                 device_printf(dev,"Hardware Initialization Failure\n");
 509                 goto err_late;
 510         case IXGBE_ERR_SFP_NOT_PRESENT:
 511                 device_printf(dev,"No SFP+ Module found\n");
 512                 /* falls thru */
 513         default:
 514                 break;
 515         }
 516
 517         /* Detect and set physical type */
 518         ixgbe_setup_optics(adapter);
 519
 520         if ((adapter->msix > 1) && (ixgbe_enable_msix)) {
 521                 adapter->intr_type = PCI_INTR_TYPE_MSIX;
 522                 error = ixgbe_allocate_msix(adapter);
 523         } else {
 524                 error = ixgbe_allocate_legacy(adapter);
 525         }
 526         if (error)
 527                 goto err_late;
 528
 529         /* Setup OS specific network interface */
 530         if (ixgbe_setup_interface(dev, adapter) != 0)
 531                 goto err_late;
 532
 533         /* Add sysctl tree */
 534         ixgbe_add_sysctl(adapter);
 535
 536         /* Initialize statistics */
 537         ixgbe_update_stats_counters(adapter);
 538
 539         /* Register for VLAN events */
 540         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 541             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 542         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 543             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 544
 545         /* Print PCIE bus type/speed/width info */
 546         ixgbe_get_bus_info(hw);
 547         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
 548             ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
 549             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
 550             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
 551             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
 552             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
 553             ("Unknown"));
 554
 555         if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
 556             (hw->bus.speed == ixgbe_bus_speed_2500)) {
 557                 device_printf(dev, "PCI-Express bandwidth available"
 558                     " for this card\n     is not sufficient for"
 559                     " optimal performance.\n");
 560                 device_printf(dev, "For optimal performance a x8 "
 561                     "PCIE, or x4 PCIE 2 slot is required.\n");
 562         }
 563
 564         /* let hardware know driver is loaded */
 565         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 566         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 567         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 568
 569         ixgbe_add_hw_stats(adapter);
 570
 571 #ifdef DEV_NETMAP
 572         ixgbe_netmap_attach(adapter);
 573 #endif /* DEV_NETMAP */
 574         INIT_DEBUGOUT("ixgbe_attach: end");
 575         return (0);
 576 err_late:
 577         ixgbe_free_transmit_structures(adapter);
 578         ixgbe_free_receive_structures(adapter);
 579 err_out:
 580         if (adapter->ifp != NULL)
 581                 if_free(adapter->ifp);
 582         ixgbe_free_pci_resources(adapter);
 583         kfree(adapter->mta, M_DEVBUF);
 584         return (error);
 585
 586 }
 587
 588 /*********************************************************************
 589  *  Device removal routine
 590  *
 591  *  The detach entry point is called when the driver is being removed.
 592  *  This routine stops the adapter and deallocates all the resources
 593  *  that were allocated for driver operation.
 594  *
 595  *  return 0 on success, positive on failure
 596  *********************************************************************/
 597
 598 static int
 599 ixgbe_detach(device_t dev)
 600 {
 601         struct adapter *adapter = device_get_softc(dev);
 602         u32     ctrl_ext;
 603
 604         INIT_DEBUGOUT("ixgbe_detach: begin");
 605
 606         /* Make sure VLANS are not using driver */
 607         if (adapter->ifp->if_vlantrunks != NULL) {
 608                 device_printf(dev,"Vlan in use, detach first\n");
 609                 return (EBUSY);
 610         }
 611
 612         IXGBE_CORE_LOCK(adapter);
 613         ixgbe_stop(adapter);
 614         IXGBE_CORE_UNLOCK(adapter);
 615
 616         /* Drain the Link queue */
 617         if (adapter->tq) {
 618                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 619                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 620 #ifdef IXGBE_FDIR
 621                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 622 #endif
 623                 taskqueue_free(adapter->tq);
 624         }
 625
 626         /* let hardware know driver is unloading */
 627         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 628         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 629         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 630
 631         /* Unregister VLAN events */
 632         if (adapter->vlan_attach != NULL)
 633                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 634         if (adapter->vlan_detach != NULL)
 635                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 636
 637         ether_ifdetach(adapter->ifp);
 638         callout_stop(&adapter->timer);
 639 #ifdef DEV_NETMAP
 640         netmap_detach(adapter->ifp);
 641 #endif /* DEV_NETMAP */
 642         ixgbe_free_pci_resources(adapter);
 643         bus_generic_detach(dev);
 644         if_free(adapter->ifp);
 645
 646         ixgbe_free_transmit_structures(adapter);
 647         ixgbe_free_receive_structures(adapter);
 648         kfree(adapter->mta, M_DEVBUF);
 649         sysctl_ctx_free(&adapter->sysctl_ctx);
 650
 651         IXGBE_CORE_LOCK_DESTROY(adapter);
 652         return (0);
 653 }
 654
 655 /*********************************************************************
 656  *
 657  *  Shutdown entry point
 658  *
 659  **********************************************************************/
 660
 661 static int
 662 ixgbe_shutdown(device_t dev)
 663 {
 664         struct adapter *adapter = device_get_softc(dev);
 665         IXGBE_CORE_LOCK(adapter);
 666         ixgbe_stop(adapter);
 667         IXGBE_CORE_UNLOCK(adapter);
 668         return (0);
 669 }
 670
 671
 672 /*********************************************************************
 673  *  Transmit entry point
 674  *
 675  *  ixgbe_start is called by the stack to initiate a transmit.
 676  *  The driver will remain in this routine as long as there are
 677  *  packets to transmit and transmit resources are available.
 678  *  In case resources are not available stack is notified and
 679  *  the packet is requeued.
 680  **********************************************************************/
 681
 682 static void
 683 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 684 {
 685         struct mbuf    *m_head;
 686         struct adapter *adapter = txr->adapter;
 687
 688         IXGBE_TX_LOCK_ASSERT(txr);
 689
 690         if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd))
 691                 return;
 692
 693         if (!adapter->link_active) {
 694                 ifq_purge(&ifp->if_snd);
 695                 return;
 696         }
 697
 698         while (!ifq_is_empty(&ifp->if_snd)) {
 699                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
 700                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 701                         break;
 702                 }
 703
 704                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
 705                 if (m_head == NULL)
 706                         break;
 707
 708                 if (ixgbe_xmit(txr, &m_head)) {
 709 #if 0 /* XXX: prepend to an ALTQ queue ? */
 710                         if (m_head != NULL)
 711                                 IF_PREPEND(&ifp->if_snd, m_head);
 712 #endif
 713                         if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
 714                                 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 715                         break;
 716                 }
 717                 /* Send a copy of the frame to the BPF listener */
 718                 ETHER_BPF_MTAP(ifp, m_head);
 719
 720                 /* Set watchdog on */
 721                 txr->watchdog_time = ticks;
 722                 txr->queue_status = IXGBE_QUEUE_WORKING;
 723
 724         }
 725         return;
 726 }
 727
 728 /*
 729  * Legacy TX start - called by the stack, this
 730  * always uses the first tx ring, and should
 731  * not be used with multiqueue tx enabled.
 732  */
 733 static void
 734 ixgbe_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
 735 {
 736         struct adapter *adapter = ifp->if_softc;
 737         struct tx_ring  *txr = adapter->tx_rings;
 738
 739         ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq);
 740
 741         if (ifp->if_flags & IFF_RUNNING) {
 742                 IXGBE_TX_LOCK(txr);
 743                 ixgbe_start_locked(txr, ifp);
 744                 IXGBE_TX_UNLOCK(txr);
 745         }
 746         return;
 747 }
 748
 749 #if 0 /* __FreeBSD_version >= 800000 */
 750 /*
 751 ** Multiqueue Transmit driver
 752 **
 753 */
 754 static int
 755 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
 756 {
 757         struct adapter  *adapter = ifp->if_softc;
 758         struct ix_queue *que;
 759         struct tx_ring  *txr;
 760         int             i = 0, err = 0;
 761
 762         /* Which queue to use */
 763         if ((m->m_flags & M_FLOWID) != 0)
 764                 i = m->m_pkthdr.flowid % adapter->num_queues;
 765         else
 766                 i = curcpu % adapter->num_queues;
 767
 768         txr = &adapter->tx_rings[i];
 769         que = &adapter->queues[i];
 770
 771         if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
 772             IXGBE_TX_TRYLOCK(txr)) {
 773                 err = ixgbe_mq_start_locked(ifp, txr, m);
 774                 IXGBE_TX_UNLOCK(txr);
 775         } else {
 776                 err = drbr_enqueue(ifp, txr->br, m);
 777                 taskqueue_enqueue(que->tq, &que->que_task);
 778         }
 779
 780         return (err);
 781 }
 782
 783 static int
 784 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 785 {
 786         struct adapter  *adapter = txr->adapter;
 787         struct mbuf     *next;
 788         int             enqueued, err = 0;
 789
 790         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
 791             (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
 792             adapter->link_active == 0) {
 793                 if (m != NULL)
 794                         err = drbr_enqueue(ifp, txr->br, m);
 795                 return (err);
 796         }
 797
 798         enqueued = 0;
 799         if (m == NULL) {
 800                 next = drbr_dequeue(ifp, txr->br);
 801         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 802                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 803                         return (err);
 804                 next = drbr_dequeue(ifp, txr->br);
 805         } else
 806                 next = m;
 807
 808         /* Process the queue */
 809         while (next != NULL) {
 810                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 811                         if (next != NULL)
 812                                 err = drbr_enqueue(ifp, txr->br, next);
 813                         break;
 814                 }
 815                 enqueued++;
 816                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 817                 /* Send a copy of the frame to the BPF listener */
 818                 ETHER_BPF_MTAP(ifp, next);
 819                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 820                         break;
 821                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
 822                         ixgbe_txeof(txr);
 823                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
 824                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 825                         break;
 826                 }
 827                 next = drbr_dequeue(ifp, txr->br);
 828         }
 829
 830         if (enqueued > 0) {
 831                 /* Set watchdog on */
 832                 txr->queue_status |= IXGBE_QUEUE_WORKING;
 833                 txr->watchdog_time = ticks;
 834         }
 835
 836         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
 837                 ixgbe_txeof(txr);
 838
 839         return (err);
 840 }
 841
 842 /*
 843 ** Flush all ring buffers
 844 */
 845 static void
 846 ixgbe_qflush(struct ifnet *ifp)
 847 {
 848         struct adapter  *adapter = ifp->if_softc;
 849         struct tx_ring  *txr = adapter->tx_rings;
 850         struct mbuf     *m;
 851
 852         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 853                 IXGBE_TX_LOCK(txr);
 854                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 855                         m_freem(m);
 856                 IXGBE_TX_UNLOCK(txr);
 857         }
 858         if_qflush(ifp);
 859 }
 860 #endif /* __FreeBSD_version >= 800000 */
 861
 862 /*********************************************************************
 863  *  Ioctl entry point
 864  *
 865  *  ixgbe_ioctl is called when the user wants to configure the
 866  *  interface.
 867  *
 868  *  return 0 on success, positive on failure
 869  **********************************************************************/
 870
 871 static int
 872 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
 873 {
 874         struct adapter  *adapter = ifp->if_softc;
 875         struct ifreq    *ifr = (struct ifreq *) data;
 876 #if defined(INET) || defined(INET6)
 877         struct ifaddr *ifa = (struct ifaddr *)data;
 878         bool            avoid_reset = FALSE;
 879 #endif
 880         int             error = 0;
 881
 882         switch (command) {
 883
 884         case SIOCSIFADDR:
 885 #ifdef INET
 886                 if (ifa->ifa_addr->sa_family == AF_INET)
 887                         avoid_reset = TRUE;
 888 #endif
 889 #ifdef INET6
 890                 if (ifa->ifa_addr->sa_family == AF_INET6)
 891                         avoid_reset = TRUE;
 892 #endif
 893 #if defined(INET) || defined(INET6)
 894                 /*
 895                 ** Calling init results in link renegotiation,
 896                 ** so we avoid doing it when possible.
 897                 */
 898                 if (avoid_reset) {
 899                         ifp->if_flags |= IFF_UP;
 900                         if (!(ifp->if_flags & IFF_RUNNING))
 901                                 ixgbe_init(adapter);
 902                         if (!(ifp->if_flags & IFF_NOARP))
 903                                 arp_ifinit(ifp, ifa);
 904                 } else
 905                         error = ether_ioctl(ifp, command, data);
 906 #endif
 907                 break;
 908         case SIOCSIFMTU:
 909                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 910                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 911                         error = EINVAL;
 912                 } else {
 913                         IXGBE_CORE_LOCK(adapter);
 914                         ifp->if_mtu = ifr->ifr_mtu;
 915                         adapter->max_frame_size =
 916                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 917                         ixgbe_init_locked(adapter);
 918                         IXGBE_CORE_UNLOCK(adapter);
 919                 }
 920                 break;
 921         case SIOCSIFFLAGS:
 922                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 923                 IXGBE_CORE_LOCK(adapter);
 924                 if (ifp->if_flags & IFF_UP) {
 925                         if ((ifp->if_flags & IFF_RUNNING)) {
 926                                 if ((ifp->if_flags ^ adapter->if_flags) &
 927                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 928                                         ixgbe_set_promisc(adapter);
 929                                 }
 930                         } else
 931                                 ixgbe_init_locked(adapter);
 932                 } else
 933                         if (ifp->if_flags & IFF_RUNNING)
 934                                 ixgbe_stop(adapter);
 935                 adapter->if_flags = ifp->if_flags;
 936                 IXGBE_CORE_UNLOCK(adapter);
 937                 break;
 938         case SIOCADDMULTI:
 939         case SIOCDELMULTI:
 940                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 941                 if (ifp->if_flags & IFF_RUNNING) {
 942                         IXGBE_CORE_LOCK(adapter);
 943                         ixgbe_disable_intr(adapter);
 944                         ixgbe_set_multi(adapter);
 945                         ixgbe_enable_intr(adapter);
 946                         IXGBE_CORE_UNLOCK(adapter);
 947                 }
 948                 break;
 949         case SIOCSIFMEDIA:
 950         case SIOCGIFMEDIA:
 951                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
 952                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 953                 break;
 954         case SIOCSIFCAP:
 955         {
 956                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 957                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
 958                 if (mask & IFCAP_HWCSUM)
 959                         ifp->if_capenable ^= IFCAP_HWCSUM;
 960                 if (mask & IFCAP_TSO4)
 961                         ifp->if_capenable ^= IFCAP_TSO4;
 962                 if (mask & IFCAP_TSO6)
 963                         ifp->if_capenable ^= IFCAP_TSO6;
 964 #if 0 /* NET_LRO */
 965                 if (mask & IFCAP_LRO)
 966                         ifp->if_capenable ^= IFCAP_LRO;
 967 #endif
 968                 if (mask & IFCAP_VLAN_HWTAGGING)
 969                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 970                 if (mask & IFCAP_VLAN_HWFILTER)
 971                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
 972 #if 0 /* NET_TSO */
 973                 if (mask & IFCAP_VLAN_HWTSO)
 974                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
 975 #endif
 976                 if (ifp->if_flags & IFF_RUNNING) {
 977                         IXGBE_CORE_LOCK(adapter);
 978                         ixgbe_init_locked(adapter);
 979                         IXGBE_CORE_UNLOCK(adapter);
 980                 }
 981 #if 0
 982                 VLAN_CAPABILITIES(ifp);
 983 #endif
 984                 break;
 985         }
 986
 987         default:
 988                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
 989                 error = ether_ioctl(ifp, command, data);
 990                 break;
 991         }
 992
 993         return (error);
 994 }
 995
 996 /*********************************************************************
 997  *  Init entry point
 998  *
 999  *  This routine is used in two ways. It is used by the stack as
1000  *  init entry point in network interface structure. It is also used
1001  *  by the driver as a hw/sw initialization routine to get to a
1002  *  consistent state.
1003  *
1004  *  return 0 on success, positive on failure
1005  **********************************************************************/
1006 #define IXGBE_MHADD_MFS_SHIFT 16
1007
1008 static void
1009 ixgbe_init_locked(struct adapter *adapter)
1010 {
1011         struct ifnet   *ifp = adapter->ifp;
1012         device_t        dev = adapter->dev;
1013         struct ixgbe_hw *hw = &adapter->hw;
1014         u32             k, txdctl, mhadd, gpie;
1015         u32             rxdctl, rxctrl;
1016
1017         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1018         INIT_DEBUGOUT("ixgbe_init: begin");
1019         hw->adapter_stopped = FALSE;
1020         ixgbe_stop_adapter(hw);
1021         callout_stop(&adapter->timer);
1022
1023         /* reprogram the RAR[0] in case user changed it. */
1024         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1025
1026         /* Get the latest mac address, User can use a LAA */
1027         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1028               IXGBE_ETH_LENGTH_OF_ADDRESS);
1029         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1030         hw->addr_ctrl.rar_used_count = 1;
1031
1032         /* Set the various hardware offload abilities */
1033         ifp->if_hwassist = 0;
1034         if (ifp->if_capenable & IFCAP_TSO)
1035                 ifp->if_hwassist |= CSUM_TSO;
1036         if (ifp->if_capenable & IFCAP_TXCSUM) {
1037                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1038 #if 0
1039                 if (hw->mac.type != ixgbe_mac_82598EB)
1040                         ifp->if_hwassist |= CSUM_SCTP;
1041 #endif
1042         }
1043
1044         /* Prepare transmit descriptors and buffers */
1045         if (ixgbe_setup_transmit_structures(adapter)) {
1046                 device_printf(dev,"Could not setup transmit structures\n");
1047                 ixgbe_stop(adapter);
1048                 return;
1049         }
1050
1051         ixgbe_init_hw(hw);
1052         ixgbe_initialize_transmit_units(adapter);
1053
1054         /* Setup Multicast table */
1055         ixgbe_set_multi(adapter);
1056
1057         /*
1058         ** Determine the correct mbuf pool
1059         ** for doing jumbo/headersplit
1060         */
1061 #if 0 /* XXX */
1062         if (adapter->max_frame_size <= 2048)
1063                 adapter->rx_mbuf_sz = MCLBYTES;
1064         else if (adapter->max_frame_size <= 4096)
1065                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1066         else if (adapter->max_frame_size <= 9216)
1067                 adapter->rx_mbuf_sz = MJUM9BYTES;
1068         else
1069                 adapter->rx_mbuf_sz = MJUM16BYTES;
1070 #else
1071         adapter->rx_mbuf_sz = MCLBYTES;
1072 #endif
1073
1074         /* Prepare receive descriptors and buffers */
1075         if (ixgbe_setup_receive_structures(adapter)) {
1076                 device_printf(dev,"Could not setup receive structures\n");
1077                 ixgbe_stop(adapter);
1078                 return;
1079         }
1080
1081         /* Configure RX settings */
1082         ixgbe_initialize_receive_units(adapter);
1083
1084         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1085
1086         /* Enable Fan Failure Interrupt */
1087         gpie |= IXGBE_SDP1_GPIEN;
1088
1089         /* Add for Module detection */
1090         if (hw->mac.type == ixgbe_mac_82599EB)
1091                 gpie |= IXGBE_SDP2_GPIEN;
1092
1093         /* Thermal Failure Detection */
1094         if (hw->mac.type == ixgbe_mac_X540)
1095                 gpie |= IXGBE_SDP0_GPIEN;
1096
1097         if (adapter->msix > 1) {
1098                 /* Enable Enhanced MSIX mode */
1099                 gpie |= IXGBE_GPIE_MSIX_MODE;
1100                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1101                     IXGBE_GPIE_OCD;
1102         }
1103         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1104
1105         /* Set MTU size */
1106         if (ifp->if_mtu > ETHERMTU) {
1107                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1108                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1109                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1110                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1111         }
1112
1113         /* Now enable all the queues */
1114
1115         for (int i = 0; i < adapter->num_queues; i++) {
1116                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1117                 txdctl |= IXGBE_TXDCTL_ENABLE;
1118                 /* Set WTHRESH to 8, burst writeback */
1119                 txdctl |= (8 << 16);
1120                 /*
1121                  * When the internal queue falls below PTHRESH (32),
1122                  * start prefetching as long as there are at least
1123                  * HTHRESH (1) buffers ready. The values are taken
1124                  * from the Intel linux driver 3.8.21.
1125                  * Prefetching enables tx line rate even with 1 queue.
1126                  */
1127                 txdctl |= (32 << 0) | (1 << 8);
1128                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1129         }
1130
1131         for (int i = 0; i < adapter->num_queues; i++) {
1132                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1133                 if (hw->mac.type == ixgbe_mac_82598EB) {
1134                         /*
1135                         ** PTHRESH = 21
1136                         ** HTHRESH = 4
1137                         ** WTHRESH = 8
1138                         */
1139                         rxdctl &= ~0x3FFFFF;
1140                         rxdctl |= 0x080420;
1141                 }
1142                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1143                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1144                 for (k = 0; k < 10; k++) {
1145                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1146                             IXGBE_RXDCTL_ENABLE)
1147                                 break;
1148                         else
1149                                 msec_delay(1);
1150                 }
1151                 wmb();
1152 #ifdef DEV_NETMAP
1153                 /*
1154                  * In netmap mode, we must preserve the buffers made
1155                  * available to userspace before the if_init()
1156                  * (this is true by default on the TX side, because
1157                  * init makes all buffers available to userspace).
1158                  *
1159                  * netmap_reset() and the device specific routines
1160                  * (e.g. ixgbe_setup_receive_rings()) map these
1161                  * buffers at the end of the NIC ring, so here we
1162                  * must set the RDT (tail) register to make sure
1163                  * they are not overwritten.
1164                  *
1165                  * In this driver the NIC ring starts at RDH = 0,
1166                  * RDT points to the last slot available for reception (?),
1167                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1168                  */
1169                 if (ifp->if_capenable & IFCAP_NETMAP) {
1170                         struct netmap_adapter *na = NA(adapter->ifp);
1171                         struct netmap_kring *kring = &na->rx_rings[i];
1172                         int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1173
1174                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1175                 } else
1176 #endif /* DEV_NETMAP */
1177                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1178         }
1179
1180         /* Set up VLAN support and filter */
1181         ixgbe_setup_vlan_hw_support(adapter);
1182
1183         /* Enable Receive engine */
1184         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1185         if (hw->mac.type == ixgbe_mac_82598EB)
1186                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1187         rxctrl |= IXGBE_RXCTRL_RXEN;
1188         ixgbe_enable_rx_dma(hw, rxctrl);
1189
1190         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1191
1192         /* Set up MSI/X routing */
1193         if (ixgbe_enable_msix)  {
1194                 ixgbe_configure_ivars(adapter);
1195                 /* Set up auto-mask */
1196                 if (hw->mac.type == ixgbe_mac_82598EB)
1197                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1198                 else {
1199                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1200                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1201                 }
1202         } else {  /* Simple settings for Legacy/MSI */
1203                 ixgbe_set_ivar(adapter, 0, 0, 0);
1204                 ixgbe_set_ivar(adapter, 0, 0, 1);
1205                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1206         }
1207
1208 #ifdef IXGBE_FDIR
1209         /* Init Flow director */
1210         if (hw->mac.type != ixgbe_mac_82598EB) {
1211                 u32 hdrm = 32 << fdir_pballoc;
1212
1213                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1214                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1215         }
1216 #endif
1217
1218         /*
1219         ** Check on any SFP devices that
1220         ** need to be kick-started
1221         */
1222         if (hw->phy.type == ixgbe_phy_none) {
1223                 int err = hw->phy.ops.identify(hw);
1224                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1225                         device_printf(dev,
1226                             "Unsupported SFP+ module type was detected.\n");
1227                         return;
1228                 }
1229         }
1230
1231         /* Set moderation on the Link interrupt */
1232         ixgbe_set_eitr(adapter, adapter->linkvec, IXGBE_LINK_ITR);
1233
1234         /* Config/Enable Link */
1235         ixgbe_config_link(adapter);
1236
1237         /* Hardware Packet Buffer & Flow Control setup */
1238         {
1239                 u32 rxpb, frame, size, tmp;
1240
1241                 frame = adapter->max_frame_size;
1242
1243                 /* Calculate High Water */
1244                 if (hw->mac.type == ixgbe_mac_X540)
1245                         tmp = IXGBE_DV_X540(frame, frame);
1246                 else
1247                         tmp = IXGBE_DV(frame, frame);
1248                 size = IXGBE_BT2KB(tmp);
1249                 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1250                 hw->fc.high_water[0] = rxpb - size;
1251
1252                 /* Now calculate Low Water */
1253                 if (hw->mac.type == ixgbe_mac_X540)
1254                         tmp = IXGBE_LOW_DV_X540(frame);
1255                 else
1256                         tmp = IXGBE_LOW_DV(frame);
1257                 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1258
1259                 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1260                 hw->fc.pause_time = IXGBE_FC_PAUSE;
1261                 hw->fc.send_xon = TRUE;
1262         }
1263         /* Initialize the FC settings */
1264         ixgbe_start_hw(hw);
1265
1266         /* And now turn on interrupts */
1267         ixgbe_enable_intr(adapter);
1268
1269         /* Now inform the stack we're ready */
1270         ifp->if_flags |= IFF_RUNNING;
1271         ifq_clr_oactive(&ifp->if_snd);
1272
1273         return;
1274 }
1275
1276 static void
1277 ixgbe_init(void *arg)
1278 {
1279         struct adapter *adapter = arg;
1280
1281         IXGBE_CORE_LOCK(adapter);
1282         ixgbe_init_locked(adapter);
1283         IXGBE_CORE_UNLOCK(adapter);
1284         return;
1285 }
1286
1287
1288 /*
1289 **
1290 ** MSIX Interrupt Handlers and Tasklets
1291 **
1292 */
1293
1294 static inline void
1295 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1296 {
1297         struct ixgbe_hw *hw = &adapter->hw;
1298         u64     queue = (u64)(1 << vector);
1299         u32     mask;
1300
1301         if (hw->mac.type == ixgbe_mac_82598EB) {
1302                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1303                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1304         } else {
1305                 mask = (queue & 0xFFFFFFFF);
1306                 if (mask)
1307                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1308                 mask = (queue >> 32);
1309                 if (mask)
1310                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1311         }
1312 }
1313
1314 static inline void
1315 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1316 {
1317         struct ixgbe_hw *hw = &adapter->hw;
1318         u64     queue = (u64)(1 << vector);
1319         u32     mask;
1320
1321         if (hw->mac.type == ixgbe_mac_82598EB) {
1322                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1323                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1324         } else {
1325                 mask = (queue & 0xFFFFFFFF);
1326                 if (mask)
1327                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1328                 mask = (queue >> 32);
1329                 if (mask)
1330                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1331         }
1332 }
1333
1334 static inline void
1335 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1336 {
1337         u32 mask;
1338
1339         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1340                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1341                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1342         } else {
1343                 mask = (queues & 0xFFFFFFFF);
1344                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1345                 mask = (queues >> 32);
1346                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1347         }
1348 }
1349
1350 /*********************************************************************
1351  *
1352  *  Legacy Interrupt Service routine
1353  *
1354  **********************************************************************/
1355
1356 static void
1357 ixgbe_legacy_irq(void *arg)
1358 {
1359         struct ix_queue *que = arg;
1360         struct adapter  *adapter = que->adapter;
1361         struct ixgbe_hw *hw = &adapter->hw;
1362         struct          tx_ring *txr = adapter->tx_rings;
1363         u32             reg_eicr;
1364
1365
1366         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1367
1368         ++que->irqs;
1369         if (reg_eicr == 0) {
1370                 ixgbe_enable_intr(adapter);
1371                 return;
1372         }
1373
1374         ixgbe_rxeof(que, adapter->rx_process_limit);
1375
1376         IXGBE_TX_LOCK(txr);
1377         ixgbe_txeof(txr);
1378         if (!ifq_is_empty(&adapter->ifp->if_snd))
1379                 ixgbe_start_locked(txr, adapter->ifp);
1380         IXGBE_TX_UNLOCK(txr);
1381
1382         /* Check for fan failure */
1383         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1384             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1385                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1386                     "REPLACE IMMEDIATELY!!\n");
1387                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1388         }
1389
1390         /* Link status change */
1391         if (reg_eicr & IXGBE_EICR_LSC) {
1392                 ixgbe_check_link(&adapter->hw,
1393                     &adapter->link_speed, &adapter->link_up, 0);
1394                 ixgbe_update_link_status(adapter);
1395         }
1396
1397         ixgbe_enable_intr(adapter);
1398 }
1399
1400
1401 /*********************************************************************
1402  *
1403  *  MSIX Queue Interrupt Service routine
1404  *
1405  **********************************************************************/
1406 void
1407 ixgbe_msix_que(void *arg)
1408 {
1409         struct ix_queue *que = arg;
1410         struct adapter  *adapter = que->adapter;
1411         struct tx_ring  *txr = que->txr;
1412
1413         ixgbe_disable_queue(adapter, que->msix);
1414         ++que->irqs;
1415
1416         ixgbe_rxeof(que, adapter->rx_process_limit);
1417
1418         IXGBE_TX_LOCK(txr);
1419         ixgbe_txeof(txr);
1420         if (!ifq_is_empty(&adapter->ifp->if_snd))
1421                 ixgbe_start_locked(txr, adapter->ifp);
1422         IXGBE_TX_UNLOCK(txr);
1423
1424         /* Reenable this interrupt */
1425         ixgbe_enable_queue(adapter, que->msix);
1426 }
1427
1428
1429 static void
1430 ixgbe_msix_link(void *arg)
1431 {
1432         struct adapter  *adapter = arg;
1433         struct ixgbe_hw *hw = &adapter->hw;
1434         u32             reg_eicr;
1435
1436         ++adapter->link_irq;
1437
1438         /* First get the cause */
1439         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1440         /* Clear interrupt with write */
1441         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1442
1443         /* Link status change */
1444         if (reg_eicr & IXGBE_EICR_LSC) {
1445                 ixgbe_check_link(&adapter->hw,
1446                     &adapter->link_speed, &adapter->link_up, 0);
1447                 ixgbe_update_link_status(adapter);
1448         }
1449
1450         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1451 #ifdef IXGBE_FDIR
1452                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1453                         /* This is probably overkill :) */
1454                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1455                                 return;
1456                         /* Disable the interrupt */
1457                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1458                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1459                 } else
1460 #endif
1461                 if (reg_eicr & IXGBE_EICR_ECC) {
1462                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1463                             "Please Reboot!!\n");
1464                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1465                 } else
1466
1467                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1468                         /* Clear the interrupt */
1469                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1470                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1471                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1472                         /* Clear the interrupt */
1473                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1474                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1475                 }
1476         }
1477
1478         /* Check for fan failure */
1479         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1480             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1481                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1482                     "REPLACE IMMEDIATELY!!\n");
1483                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1484         }
1485
1486         /* Check for over temp condition */
1487         if ((hw->mac.type == ixgbe_mac_X540) &&
1488             (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1489                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1490                     "PHY IS SHUT DOWN!!\n");
1491                 device_printf(adapter->dev, "System shutdown required\n");
1492                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1493         }
1494
1495         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1496         return;
1497 }
1498
1499 /*********************************************************************
1500  *
1501  *  Media Ioctl callback
1502  *
1503  *  This routine is called whenever the user queries the status of
1504  *  the interface using ifconfig.
1505  *
1506  **********************************************************************/
1507 static void
1508 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1509 {
1510         struct adapter *adapter = ifp->if_softc;
1511
1512         ASSERT_IFNET_SERIALIZED_ALL(ifp);
1513
1514         INIT_DEBUGOUT("ixgbe_media_status: begin");
1515         ixgbe_update_link_status(adapter);
1516
1517         ifmr->ifm_status = IFM_AVALID;
1518         ifmr->ifm_active = IFM_ETHER;
1519
1520         if (!adapter->link_active)
1521                 return;
1522
1523         ifmr->ifm_status |= IFM_ACTIVE;
1524
1525         switch (adapter->link_speed) {
1526                 case IXGBE_LINK_SPEED_100_FULL:
1527                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1528                         break;
1529                 case IXGBE_LINK_SPEED_1GB_FULL:
1530                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1531                         break;
1532                 case IXGBE_LINK_SPEED_10GB_FULL:
1533                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1534                         break;
1535         }
1536
1537         return;
1538 }
1539
1540 /*********************************************************************
1541  *
1542  *  Media Ioctl callback
1543  *
1544  *  This routine is called when the user changes speed/duplex using
1545  *  media/mediopt option with ifconfig.
1546  *
1547  **********************************************************************/
1548 static int
1549 ixgbe_media_change(struct ifnet * ifp)
1550 {
1551         struct adapter *adapter = ifp->if_softc;
1552         struct ifmedia *ifm = &adapter->media;
1553
1554         INIT_DEBUGOUT("ixgbe_media_change: begin");
1555
1556         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1557                 return (EINVAL);
1558
1559         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1560         case IFM_AUTO:
1561                 adapter->hw.phy.autoneg_advertised =
1562                     IXGBE_LINK_SPEED_100_FULL |
1563                     IXGBE_LINK_SPEED_1GB_FULL |
1564                     IXGBE_LINK_SPEED_10GB_FULL;
1565                 break;
1566         default:
1567                 device_printf(adapter->dev, "Only auto media type\n");
1568                 return (EINVAL);
1569         }
1570
1571         return (0);
1572 }
1573
1574 /*********************************************************************
1575  *
1576  *  This routine maps the mbufs to tx descriptors, allowing the
1577  *  TX engine to transmit the packets.
1578  *      - return 0 on success, positive on failure
1579  *
1580  **********************************************************************/
1581
1582 static int
1583 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1584 {
1585         struct adapter  *adapter = txr->adapter;
1586         u32             olinfo_status = 0, cmd_type_len;
1587         u32             paylen = 0;
1588         int             i, j, error, nsegs, maxsegs;
1589         int             first, last = 0;
1590         struct mbuf     *m_head;
1591         bus_dma_segment_t segs[adapter->num_segs];
1592         bus_dmamap_t    map;
1593         struct ixgbe_tx_buf *txbuf;
1594         union ixgbe_adv_tx_desc *txd = NULL;
1595
1596         m_head = *m_headp;
1597
1598         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1599                 error = ixgbe_tso_pullup(txr, m_headp);
1600                 if (error)
1601                         return error;
1602                 m_head = *m_headp;
1603         }
1604
1605         /* Basic descriptor defines */
1606         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1607             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1608
1609         if (m_head->m_flags & M_VLANTAG)
1610                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1611
1612         /*
1613          * Important to capture the first descriptor
1614          * used because it will contain the index of
1615          * the one we tell the hardware to report back
1616          */
1617         first = txr->next_avail_desc;
1618         txbuf = &txr->tx_buffers[first];
1619         map = txbuf->map;
1620
1621         /*
1622          * Map the packet for DMA.
1623          */
1624         maxsegs = txr->tx_avail - IXGBE_TX_RESERVED;
1625         if (maxsegs > adapter->num_segs)
1626                 maxsegs = adapter->num_segs;
1627
1628         error = bus_dmamap_load_mbuf_defrag(txr->txtag, map, m_headp,
1629             segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1630         if (error) {
1631                 if (error == ENOBUFS)
1632                         adapter->mbuf_defrag_failed++;
1633                 else
1634                         adapter->no_tx_dma_setup++;
1635
1636                 m_freem(*m_headp);
1637                 *m_headp = NULL;
1638                 return (error);
1639         }
1640
1641         /* Make certain there are enough descriptors */
1642         if (nsegs > txr->tx_avail - 2) {
1643                 txr->no_desc_avail++;
1644                 error = ENOBUFS;
1645                 goto xmit_fail;
1646         }
1647         m_head = *m_headp;
1648
1649         /*
1650         ** Set up the appropriate offload context
1651         ** this becomes the first descriptor of
1652         ** a packet.
1653         */
1654         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1655                 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1656                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1657                         olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1658                         olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1659                         ++adapter->tso_tx;
1660                 } else
1661                         return (ENXIO);
1662         } else if (ixgbe_tx_ctx_setup(txr, m_head))
1663                 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1664
1665 #ifdef IXGBE_IEEE1588
1666         /* This is changing soon to an mtag detection */
1667         if (we detect this mbuf has a TSTAMP mtag)
1668                 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1669 #endif
1670
1671 #ifdef IXGBE_FDIR
1672         /* Do the flow director magic */
1673         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1674                 ++txr->atr_count;
1675                 if (txr->atr_count >= atr_sample_rate) {
1676                         ixgbe_atr(txr, m_head);
1677                         txr->atr_count = 0;
1678                 }
1679         }
1680 #endif
1681         /* Record payload length */
1682         if (paylen == 0)
1683                 olinfo_status |= m_head->m_pkthdr.len <<
1684                     IXGBE_ADVTXD_PAYLEN_SHIFT;
1685
1686         i = txr->next_avail_desc;
1687         for (j = 0; j < nsegs; j++) {
1688                 bus_size_t seglen;
1689                 bus_addr_t segaddr;
1690
1691                 txbuf = &txr->tx_buffers[i];
1692                 txd = &txr->tx_base[i];
1693                 seglen = segs[j].ds_len;
1694                 segaddr = htole64(segs[j].ds_addr);
1695
1696                 txd->read.buffer_addr = segaddr;
1697                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1698                     cmd_type_len |seglen);
1699                 txd->read.olinfo_status = htole32(olinfo_status);
1700                 last = i; /* descriptor that will get completion IRQ */
1701
1702                 if (++i == adapter->num_tx_desc)
1703                         i = 0;
1704
1705                 txbuf->m_head = NULL;
1706                 txbuf->eop_index = -1;
1707         }
1708
1709         txd->read.cmd_type_len |=
1710             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1711         txr->tx_avail -= nsegs;
1712         txr->next_avail_desc = i;
1713
1714         txbuf->m_head = m_head;
1715         /* Swap the dma map between the first and last descriptor */
1716         txr->tx_buffers[first].map = txbuf->map;
1717         txbuf->map = map;
1718         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1719
1720         /* Set the index of the descriptor that will be marked done */
1721         txbuf = &txr->tx_buffers[first];
1722         txbuf->eop_index = last;
1723
1724         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1725             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1726         /*
1727          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1728          * hardware that this frame is available to transmit.
1729          */
1730         ++txr->total_packets;
1731         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1732
1733         return (0);
1734
1735 xmit_fail:
1736         bus_dmamap_unload(txr->txtag, txbuf->map);
1737         return (error);
1738
1739 }
1740
1741 static void
1742 ixgbe_set_promisc(struct adapter *adapter)
1743 {
1744         u_int32_t       reg_rctl;
1745         struct ifnet   *ifp = adapter->ifp;
1746
1747         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1748         reg_rctl &= (~IXGBE_FCTRL_UPE);
1749         reg_rctl &= (~IXGBE_FCTRL_MPE);
1750         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1751
1752         if (ifp->if_flags & IFF_PROMISC) {
1753                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1754                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1755         } else if (ifp->if_flags & IFF_ALLMULTI) {
1756                 reg_rctl |= IXGBE_FCTRL_MPE;
1757                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1758                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1759         }
1760         return;
1761 }
1762
1763
1764 /*********************************************************************
1765  *  Multicast Update
1766  *
1767  *  This routine is called whenever multicast address list is updated.
1768  *
1769  **********************************************************************/
1770 #define IXGBE_RAR_ENTRIES 16
1771
1772 static void
1773 ixgbe_set_multi(struct adapter *adapter)
1774 {
1775         u32     fctrl;
1776         u8      *mta;
1777         u8      *update_ptr;
1778         struct  ifmultiaddr *ifma;
1779         int     mcnt = 0;
1780         struct ifnet   *ifp = adapter->ifp;
1781
1782         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1783
1784         mta = adapter->mta;
1785         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1786             MAX_NUM_MULTICAST_ADDRESSES);
1787
1788         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1789         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1790         if (ifp->if_flags & IFF_PROMISC)
1791                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1792         else if (ifp->if_flags & IFF_ALLMULTI) {
1793                 fctrl |= IXGBE_FCTRL_MPE;
1794                 fctrl &= ~IXGBE_FCTRL_UPE;
1795         } else
1796                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1797
1798         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1799
1800         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1801                 if (ifma->ifma_addr->sa_family != AF_LINK)
1802                         continue;
1803                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1804                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1805                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1806                 mcnt++;
1807         }
1808
1809         update_ptr = mta;
1810         ixgbe_update_mc_addr_list(&adapter->hw,
1811             update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1812
1813         return;
1814 }
1815
1816 /*
1817  * This is an iterator function now needed by the multicast
1818  * shared code. It simply feeds the shared code routine the
1819  * addresses in the array of ixgbe_set_multi() one by one.
1820  */
1821 static u8 *
1822 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1823 {
1824         u8 *addr = *update_ptr;
1825         u8 *newptr;
1826         *vmdq = 0;
1827
1828         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1829         *update_ptr = newptr;
1830         return addr;
1831 }
1832
1833
1834 /*********************************************************************
1835  *  Timer routine
1836  *
1837  *  This routine checks for link status,updates statistics,
1838  *  and runs the watchdog check.
1839  *
1840  **********************************************************************/
1841
1842 static void
1843 ixgbe_local_timer(void *arg)
1844 {
1845         struct adapter  *adapter = arg;
1846         device_t        dev = adapter->dev;
1847         struct ifnet    *ifp = adapter->ifp;
1848         struct ix_queue *que = adapter->queues;
1849         struct tx_ring  *txr = adapter->tx_rings;
1850         int             hung, busy, paused;
1851
1852         IXGBE_CORE_LOCK(adapter);
1853         hung = busy = paused = 0;
1854
1855         /* Check for pluggable optics */
1856         if (adapter->sfp_probe)
1857                 if (!ixgbe_sfp_probe(adapter))
1858                         goto out; /* Nothing to do */
1859
1860         ixgbe_update_link_status(adapter);
1861         ixgbe_update_stats_counters(adapter);
1862
1863         /*
1864          * If the interface has been paused
1865          * then don't do the watchdog check
1866          */
1867         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
1868                 paused = 1;
1869
1870         /*
1871         ** Check the TX queues status
1872         **      - central locked handling of OACTIVE
1873         **      - watchdog only if all queues show hung
1874         */
1875         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
1876                 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
1877                     (paused == 0))
1878                         ++hung;
1879                 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
1880                         ++busy;
1881         }
1882         /* Only truely watchdog if all queues show hung */
1883         if (hung == adapter->num_queues)
1884                 goto watchdog;
1885         /* Only turn off the stack flow when ALL are depleted */
1886         if (busy == adapter->num_queues)
1887                 ifq_set_oactive(&ifp->if_snd);
1888         else if (ifq_is_oactive(&ifp->if_snd) && (busy < adapter->num_queues))
1889                 ifq_clr_oactive(&ifp->if_snd);
1890
1891 out:
1892         ixgbe_rearm_queues(adapter, adapter->que_mask);
1893         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1894         IXGBE_CORE_UNLOCK(adapter);
1895         return;
1896
1897 watchdog:
1898         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
1899         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
1900             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
1901             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
1902         device_printf(dev,"TX(%d) desc avail = %d,"
1903             "Next TX to Clean = %d\n",
1904             txr->me, txr->tx_avail, txr->next_to_clean);
1905         adapter->ifp->if_flags &= ~IFF_RUNNING;
1906         adapter->watchdog_events++;
1907         ixgbe_init_locked(adapter);
1908
1909         IXGBE_CORE_UNLOCK(adapter);
1910 }
1911
1912 /*
1913 ** Note: this routine updates the OS on the link state
1914 **      the real check of the hardware only happens with
1915 **      a link interrupt.
1916 */
1917 static void
1918 ixgbe_update_link_status(struct adapter *adapter)
1919 {
1920         struct ifnet    *ifp = adapter->ifp;
1921         struct tx_ring *txr = adapter->tx_rings;
1922         device_t dev = adapter->dev;
1923
1924
1925         if (adapter->link_up){
1926                 if (adapter->link_active == FALSE) {
1927                         if (bootverbose)
1928                                 device_printf(dev,"Link is up %d Gbps %s \n",
1929                                     ((adapter->link_speed == 128)? 10:1),
1930                                     "Full Duplex");
1931                         adapter->link_active = TRUE;
1932                         /* Update any Flow Control changes */
1933                         ixgbe_fc_enable(&adapter->hw);
1934                         ifp->if_link_state = LINK_STATE_UP;
1935                         if_link_state_change(ifp);
1936                 }
1937         } else { /* Link down */
1938                 if (adapter->link_active == TRUE) {
1939                         if (bootverbose)
1940                                 device_printf(dev,"Link is Down\n");
1941                         ifp->if_link_state = LINK_STATE_DOWN;
1942                         if_link_state_change(ifp);
1943                         adapter->link_active = FALSE;
1944                         for (int i = 0; i < adapter->num_queues;
1945                             i++, txr++)
1946                                 txr->queue_status = IXGBE_QUEUE_IDLE;
1947                 }
1948         }
1949
1950         return;
1951 }
1952
1953
1954 /*********************************************************************
1955  *
1956  *  This routine disables all traffic on the adapter by issuing a
1957  *  global reset on the MAC and deallocates TX/RX buffers.
1958  *
1959  **********************************************************************/
1960
1961 static void
1962 ixgbe_stop(void *arg)
1963 {
1964         struct ifnet   *ifp;
1965         struct adapter *adapter = arg;
1966         struct ixgbe_hw *hw = &adapter->hw;
1967         ifp = adapter->ifp;
1968
1969         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1970
1971         INIT_DEBUGOUT("ixgbe_stop: begin\n");
1972         ixgbe_disable_intr(adapter);
1973         callout_stop(&adapter->timer);
1974
1975         /* Let the stack know...*/
1976         ifp->if_flags &= ~IFF_RUNNING;
1977         ifq_clr_oactive(&ifp->if_snd);
1978
1979         ixgbe_reset_hw(hw);
1980         hw->adapter_stopped = FALSE;
1981         ixgbe_stop_adapter(hw);
1982         /* Turn off the laser */
1983         if (hw->phy.multispeed_fiber)
1984                 ixgbe_disable_tx_laser(hw);
1985
1986         /* reprogram the RAR[0] in case user changed it. */
1987         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1988
1989         return;
1990 }
1991
1992
1993 /*********************************************************************
1994  *
1995  *  Determine hardware revision.
1996  *
1997  **********************************************************************/
1998 static void
1999 ixgbe_identify_hardware(struct adapter *adapter)
2000 {
2001         device_t        dev = adapter->dev;
2002         struct ixgbe_hw *hw = &adapter->hw;
2003
2004         /* Save off the information about this board */
2005         hw->vendor_id = pci_get_vendor(dev);
2006         hw->device_id = pci_get_device(dev);
2007         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2008         hw->subsystem_vendor_id =
2009             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2010         hw->subsystem_device_id =
2011             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2012
2013         /* We need this here to set the num_segs below */
2014         ixgbe_set_mac_type(hw);
2015
2016         /* Pick up the 82599 and VF settings */
2017         if (hw->mac.type != ixgbe_mac_82598EB) {
2018                 hw->phy.smart_speed = ixgbe_smart_speed;
2019                 adapter->num_segs = IXGBE_82599_SCATTER;
2020         } else
2021                 adapter->num_segs = IXGBE_82598_SCATTER;
2022
2023         return;
2024 }
2025
2026 /*********************************************************************
2027  *
2028  *  Determine optic type
2029  *
2030  **********************************************************************/
2031 static void
2032 ixgbe_setup_optics(struct adapter *adapter)
2033 {
2034         struct ixgbe_hw *hw = &adapter->hw;
2035         int             layer;
2036
2037         layer = ixgbe_get_supported_physical_layer(hw);
2038
2039         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2040                 adapter->optics = IFM_10G_T;
2041                 return;
2042         }
2043
2044         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2045                 adapter->optics = IFM_1000_T;
2046                 return;
2047         }
2048
2049         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2050             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2051                 adapter->optics = IFM_10G_LR;
2052                 return;
2053         }
2054
2055         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2056                 adapter->optics = IFM_10G_SR;
2057                 return;
2058         }
2059
2060         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2061                 adapter->optics = IFM_10G_TWINAX;
2062                 return;
2063         }
2064
2065         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2066             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2067                 adapter->optics = IFM_10G_CX4;
2068                 return;
2069         }
2070
2071         /* If we get here just set the default */
2072         adapter->optics = IFM_ETHER | IFM_AUTO;
2073         return;
2074 }
2075
2076 /*********************************************************************
2077  *
2078  *  Setup the Legacy or MSI Interrupt handler
2079  *
2080  **********************************************************************/
2081 static int
2082 ixgbe_allocate_legacy(struct adapter *adapter)
2083 {
2084         device_t dev = adapter->dev;
2085         struct          ix_queue *que = adapter->queues;
2086         int error, rid = 0;
2087         unsigned int intr_flags;
2088
2089         /* MSI RID at 1 */
2090         if (adapter->msix == 1)
2091                 rid = 1;
2092
2093         /* Try allocating a MSI interrupt first */
2094         adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2095                 &rid, &intr_flags);
2096
2097         /* We allocate a single interrupt resource */
2098         adapter->res = bus_alloc_resource_any(dev,
2099             SYS_RES_IRQ, &rid, intr_flags);
2100         if (adapter->res == NULL) {
2101                 device_printf(dev, "Unable to allocate bus resource: "
2102                     "interrupt\n");
2103                 return (ENXIO);
2104         }
2105
2106         /* Tasklets for Link, SFP and Multispeed Fiber */
2107         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2108         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2109 #ifdef IXGBE_FDIR
2110         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2111 #endif
2112
2113         if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2114             ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2115                 device_printf(dev, "Failed to register fast interrupt "
2116                     "handler: %d\n", error);
2117                 taskqueue_free(adapter->tq);
2118                 adapter->tq = NULL;
2119                 return (error);
2120         }
2121         /* For simplicity in the handlers */
2122         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2123
2124         return (0);
2125 }
2126
2127
2128 /*********************************************************************
2129  *
2130  *  Setup MSIX Interrupt resources and handlers
2131  *
2132  **********************************************************************/
2133 static int
2134 ixgbe_allocate_msix(struct adapter *adapter)
2135 {
2136         device_t        dev = adapter->dev;
2137         struct          ix_queue *que = adapter->queues;
2138         int             error, rid, vector = 0;
2139         char            desc[16];
2140
2141         error = pci_setup_msix(dev);
2142         if (error) {
2143                 device_printf(dev, "MSI-X setup failed\n");
2144                 return (error);
2145         }
2146
2147         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2148                 rid = vector + 1;
2149
2150                 /*
2151                 ** Bind the msix vector, and thus the
2152                 ** ring to the corresponding cpu.
2153                 */
2154                 error = pci_alloc_msix_vector(dev, vector, &rid, i);
2155                 if (error) {
2156                         device_printf(dev, "pci_alloc_msix_vector failed\n");
2157                         return (error);
2158                 }
2159
2160                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2161                     RF_SHAREABLE | RF_ACTIVE);
2162                 if (que->res == NULL) {
2163                         device_printf(dev,"Unable to allocate"
2164                             " bus resource: que interrupt [%d]\n", vector);
2165                         return (ENXIO);
2166                 }
2167                 /* Set the handler function */
2168                 ksnprintf(desc, sizeof(desc), "%s que %d",
2169                     device_get_nameunit(dev), i);
2170                 error = bus_setup_intr_descr(dev, que->res, INTR_MPSAFE,
2171                     ixgbe_msix_que, que, &que->tag, &que->serializer, desc);
2172                 if (error) {
2173                         que->res = NULL;
2174                         device_printf(dev, "Failed to register QUE handler");
2175                         return (error);
2176                 }
2177                 que->msix = vector;
2178                 adapter->que_mask |= (u64)(1 << que->msix);
2179         }
2180
2181         /* and Link, bind vector to cpu #0 */
2182         rid = vector + 1;
2183         error = pci_alloc_msix_vector(dev, vector, &rid, 0);
2184         if (error) {
2185                 device_printf(dev, "pci_alloc_msix_vector failed\n");
2186                 return (error);
2187         }
2188         adapter->res = bus_alloc_resource_any(dev,
2189             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2190         if (!adapter->res) {
2191                 device_printf(dev,"Unable to allocate"
2192             " bus resource: Link interrupt [%d]\n", rid);
2193                 return (ENXIO);
2194         }
2195         /* Set the link handler function */
2196         error = bus_setup_intr_descr(dev, adapter->res, INTR_MPSAFE,
2197             ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer,
2198             "link");
2199         if (error) {
2200                 adapter->res = NULL;
2201                 device_printf(dev, "Failed to register LINK handler");
2202                 return (error);
2203         }
2204         pci_enable_msix(dev);
2205
2206         adapter->linkvec = vector;
2207         /* Tasklets for Link, SFP and Multispeed Fiber */
2208         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2209         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2210 #ifdef IXGBE_FDIR
2211         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2212 #endif
2213
2214         return (0);
2215 }
2216
2217 /*
2218  * Setup Either MSI/X or MSI
2219  */
2220 static int
2221 ixgbe_setup_msix(struct adapter *adapter)
2222 {
2223         device_t dev = adapter->dev;
2224         int rid, want, queues, msgs;
2225
2226         /* Override by tuneable */
2227         if (ixgbe_enable_msix == 0)
2228                 goto msi;
2229
2230         /* First try MSI/X */
2231         rid = PCIR_BAR(MSIX_82598_BAR);
2232         adapter->msix_mem = bus_alloc_resource_any(dev,
2233             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2234         if (!adapter->msix_mem) {
2235                 rid += 4;       /* 82599 maps in higher BAR */
2236                 adapter->msix_mem = bus_alloc_resource_any(dev,
2237                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2238         }
2239         if (!adapter->msix_mem) {
2240                 /* May not be enabled */
2241                 device_printf(adapter->dev,
2242                     "Unable to map MSIX table \n");
2243                 goto msi;
2244         }
2245
2246         msgs = pci_msix_count(dev);
2247         if (msgs == 0) { /* system has msix disabled */
2248                 bus_release_resource(dev, SYS_RES_MEMORY,
2249                     rid, adapter->msix_mem);
2250                 adapter->msix_mem = NULL;
2251                 goto msi;
2252         }
2253
2254         /* Figure out a reasonable auto config value */
2255         queues = (ncpus > (msgs-1)) ? (msgs-1) : ncpus;
2256
2257         if (ixgbe_num_queues != 0)
2258                 queues = ixgbe_num_queues;
2259         /* Set max queues to 8 when autoconfiguring */
2260         else if ((ixgbe_num_queues == 0) && (queues > 8))
2261                 queues = 8;
2262
2263         /*
2264         ** Want one vector (RX/TX pair) per queue
2265         ** plus an additional for Link.
2266         */
2267         want = queues + 1;
2268         if (msgs >= want)
2269                 msgs = want;
2270         else {
2271                 device_printf(adapter->dev,
2272                     "MSIX Configuration Problem, "
2273                     "%d vectors but %d queues wanted!\n",
2274                     msgs, want);
2275                 return (0); /* Will go to Legacy setup */
2276         }
2277         if (msgs) {
2278                 device_printf(adapter->dev,
2279                     "Using MSIX interrupts with %d vectors\n", msgs);
2280                 adapter->num_queues = queues;
2281                 return (msgs);
2282         }
2283 msi:
2284         msgs = pci_msi_count(dev);
2285         return (msgs);
2286 }
2287
2288
2289 static int
2290 ixgbe_allocate_pci_resources(struct adapter *adapter)
2291 {
2292         int             rid;
2293         device_t        dev = adapter->dev;
2294
2295         rid = PCIR_BAR(0);
2296         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2297             &rid, RF_ACTIVE);
2298
2299         if (!(adapter->pci_mem)) {
2300                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2301                 return (ENXIO);
2302         }
2303
2304         adapter->osdep.mem_bus_space_tag =
2305                 rman_get_bustag(adapter->pci_mem);
2306         adapter->osdep.mem_bus_space_handle =
2307                 rman_get_bushandle(adapter->pci_mem);
2308         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2309
2310         /* Legacy defaults */
2311         adapter->num_queues = 1;
2312         adapter->hw.back = &adapter->osdep;
2313
2314         /*
2315         ** Now setup MSI or MSI/X, should
2316         ** return us the number of supported
2317         ** vectors. (Will be 1 for MSI)
2318         */
2319         adapter->msix = ixgbe_setup_msix(adapter);
2320         return (0);
2321 }
2322
2323 static void
2324 ixgbe_free_pci_resources(struct adapter * adapter)
2325 {
2326         struct          ix_queue *que = adapter->queues;
2327         device_t        dev = adapter->dev;
2328         int             rid, memrid;
2329
2330         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2331                 memrid = PCIR_BAR(MSIX_82598_BAR);
2332         else
2333                 memrid = PCIR_BAR(MSIX_82599_BAR);
2334
2335         /*
2336         ** There is a slight possibility of a failure mode
2337         ** in attach that will result in entering this function
2338         ** before interrupt resources have been initialized, and
2339         ** in that case we do not want to execute the loops below
2340         ** We can detect this reliably by the state of the adapter
2341         ** res pointer.
2342         */
2343         if (adapter->res == NULL)
2344                 goto mem;
2345
2346         /*
2347         **  Release all msix queue resources:
2348         */
2349         for (int i = 0; i < adapter->num_queues; i++, que++) {
2350                 rid = que->msix + 1;
2351                 if (que->tag != NULL) {
2352                         bus_teardown_intr(dev, que->res, que->tag);
2353                         que->tag = NULL;
2354                 }
2355                 if (que->res != NULL)
2356                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2357         }
2358
2359
2360         /* Clean the Legacy or Link interrupt last */
2361         if (adapter->linkvec) /* we are doing MSIX */
2362                 rid = adapter->linkvec + 1;
2363         else
2364                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2365
2366         if (adapter->tag != NULL) {
2367                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2368                 adapter->tag = NULL;
2369         }
2370         if (adapter->res != NULL)
2371                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2372         if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2373                 pci_release_msi(adapter->dev);
2374
2375 mem:
2376         if (adapter->msix)
2377                 pci_release_msi(dev);
2378
2379         if (adapter->msix_mem != NULL)
2380                 bus_release_resource(dev, SYS_RES_MEMORY,
2381                     memrid, adapter->msix_mem);
2382
2383         if (adapter->pci_mem != NULL)
2384                 bus_release_resource(dev, SYS_RES_MEMORY,
2385                     PCIR_BAR(0), adapter->pci_mem);
2386
2387         return;
2388 }
2389
2390 /*********************************************************************
2391  *
2392  *  Setup networking device structure and register an interface.
2393  *
2394  **********************************************************************/
2395 static int
2396 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2397 {
2398         struct ixgbe_hw *hw = &adapter->hw;
2399         struct ifnet   *ifp;
2400
2401         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2402
2403         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2404         if (ifp == NULL) {
2405                 device_printf(dev, "can not allocate ifnet structure\n");
2406                 return (-1);
2407         }
2408         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2409         ifp->if_baudrate = 1000000000;
2410         ifp->if_init = ixgbe_init;
2411         ifp->if_softc = adapter;
2412         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2413         ifp->if_ioctl = ixgbe_ioctl;
2414         ifp->if_start = ixgbe_start;
2415 #if 0 /* __FreeBSD_version >= 800000 */
2416         ifp->if_transmit = ixgbe_mq_start;
2417         ifp->if_qflush = ixgbe_qflush;
2418 #endif
2419         ifq_set_maxlen(&ifp->if_snd, adapter->num_tx_desc - 2);
2420
2421         ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2422
2423         adapter->max_frame_size =
2424             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2425
2426         /*
2427          * Tell the upper layer(s) we support long frames.
2428          */
2429         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2430
2431         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2432         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2433         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2434 #if 0 /* NET_TSO */
2435                              |  IFCAP_VLAN_HWTSO
2436 #endif
2437                              |  IFCAP_VLAN_MTU;
2438         ifp->if_capenable = ifp->if_capabilities;
2439
2440         /* Don't enable LRO by default */
2441 #if 0 /* NET_LRO */
2442         ifp->if_capabilities |= IFCAP_LRO;
2443 #endif
2444
2445         /*
2446         ** Don't turn this on by default, if vlans are
2447         ** created on another pseudo device (eg. lagg)
2448         ** then vlan events are not passed thru, breaking
2449         ** operation, but with HW FILTER off it works. If
2450         ** using vlans directly on the ixgbe driver you can
2451         ** enable this and get full hardware tag filtering.
2452         */
2453         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2454
2455         /*
2456          * Specify the media types supported by this adapter and register
2457          * callbacks to update media and link information
2458          */
2459         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2460                      ixgbe_media_status);
2461         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2462         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2463         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2464                 ifmedia_add(&adapter->media,
2465                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2466                 ifmedia_add(&adapter->media,
2467                     IFM_ETHER | IFM_1000_T, 0, NULL);
2468         }
2469         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2470         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2471
2472         return (0);
2473 }
2474
2475 static void
2476 ixgbe_config_link(struct adapter *adapter)
2477 {
2478         struct ixgbe_hw *hw = &adapter->hw;
2479         u32     autoneg, err = 0;
2480         bool    sfp, negotiate;
2481
2482         sfp = ixgbe_is_sfp(hw);
2483
2484         if (sfp) {
2485                 if (hw->phy.multispeed_fiber) {
2486                         hw->mac.ops.setup_sfp(hw);
2487                         ixgbe_enable_tx_laser(hw);
2488                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2489                 } else
2490                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2491         } else {
2492                 if (hw->mac.ops.check_link)
2493                         err = ixgbe_check_link(hw, &autoneg,
2494                             &adapter->link_up, FALSE);
2495                 if (err)
2496                         goto out;
2497                 autoneg = hw->phy.autoneg_advertised;
2498                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2499                         err  = hw->mac.ops.get_link_capabilities(hw,
2500                             &autoneg, &negotiate);
2501                 if (err)
2502                         goto out;
2503                 if (hw->mac.ops.setup_link)
2504                         err = hw->mac.ops.setup_link(hw, autoneg,
2505                             negotiate, adapter->link_up);
2506         }
2507 out:
2508         return;
2509 }
2510
2511 /********************************************************************
2512  * Manage DMA'able memory.
2513  *******************************************************************/
2514 static void
2515 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2516 {
2517         if (error)
2518                 return;
2519         *(bus_addr_t *) arg = segs->ds_addr;
2520         return;
2521 }
2522
2523 static int
2524 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2525                 struct ixgbe_dma_alloc *dma, int mapflags)
2526 {
2527         device_t dev = adapter->dev;
2528         int             r;
2529
2530         r = bus_dma_tag_create(NULL,    /* parent */
2531                                DBA_ALIGN, 0,    /* alignment, bounds */
2532                                BUS_SPACE_MAXADDR,       /* lowaddr */
2533                                BUS_SPACE_MAXADDR,       /* highaddr */
2534                                NULL, NULL,      /* filter, filterarg */
2535                                size,    /* maxsize */
2536                                1,       /* nsegments */
2537                                size,    /* maxsegsize */
2538                                BUS_DMA_ALLOCNOW,        /* flags */
2539                                &dma->dma_tag);
2540         if (r != 0) {
2541                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2542                        "error %u\n", r);
2543                 goto fail_0;
2544         }
2545         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2546                              BUS_DMA_NOWAIT, &dma->dma_map);
2547         if (r != 0) {
2548                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2549                        "error %u\n", r);
2550                 goto fail_1;
2551         }
2552         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2553                             size,
2554                             ixgbe_dmamap_cb,
2555                             &dma->dma_paddr,
2556                             mapflags | BUS_DMA_NOWAIT);
2557         if (r != 0) {
2558                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2559                        "error %u\n", r);
2560                 goto fail_2;
2561         }
2562         dma->dma_size = size;
2563         return (0);
2564 fail_2:
2565         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2566 fail_1:
2567         bus_dma_tag_destroy(dma->dma_tag);
2568 fail_0:
2569         dma->dma_map = NULL;
2570         dma->dma_tag = NULL;
2571         return (r);
2572 }
2573
2574 static void
2575 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2576 {
2577         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2578             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2579         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2580         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2581         bus_dma_tag_destroy(dma->dma_tag);
2582 }
2583
2584
2585 /*********************************************************************
2586  *
2587  *  Allocate memory for the transmit and receive rings, and then
2588  *  the descriptors associated with each, called only once at attach.
2589  *
2590  **********************************************************************/
2591 static int
2592 ixgbe_allocate_queues(struct adapter *adapter)
2593 {
2594         device_t        dev = adapter->dev;
2595         struct ix_queue *que;
2596         struct tx_ring  *txr;
2597         struct rx_ring  *rxr;
2598         int rsize, tsize, error = IXGBE_SUCCESS;
2599         int txconf = 0, rxconf = 0;
2600
2601         /* First allocate the top level queue structs */
2602         if (!(adapter->queues =
2603             (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2604             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2605                 device_printf(dev, "Unable to allocate queue memory\n");
2606                 error = ENOMEM;
2607                 goto fail;
2608         }
2609
2610         /* First allocate the TX ring struct memory */
2611         if (!(adapter->tx_rings =
2612             (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2613             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2614                 device_printf(dev, "Unable to allocate TX ring memory\n");
2615                 error = ENOMEM;
2616                 goto tx_fail;
2617         }
2618
2619         /* Next allocate the RX */
2620         if (!(adapter->rx_rings =
2621             (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2622             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2623                 device_printf(dev, "Unable to allocate RX ring memory\n");
2624                 error = ENOMEM;
2625                 goto rx_fail;
2626         }
2627
2628         /* For the ring itself */
2629         tsize = roundup2(adapter->num_tx_desc *
2630             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2631
2632         /*
2633          * Now set up the TX queues, txconf is needed to handle the
2634          * possibility that things fail midcourse and we need to
2635          * undo memory gracefully
2636          */
2637         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2638                 /* Set up some basics */
2639                 txr = &adapter->tx_rings[i];
2640                 txr->adapter = adapter;
2641                 txr->me = i;
2642
2643                 /* Initialize the TX side lock */
2644                 IXGBE_TX_LOCK_INIT(txr);
2645
2646                 if (ixgbe_dma_malloc(adapter, tsize,
2647                         &txr->txdma, BUS_DMA_NOWAIT)) {
2648                         device_printf(dev,
2649                             "Unable to allocate TX Descriptor memory\n");
2650                         error = ENOMEM;
2651                         goto err_tx_desc;
2652                 }
2653                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2654                 bzero((void *)txr->tx_base, tsize);
2655
2656                 /* Now allocate transmit buffers for the ring */
2657                 if (ixgbe_allocate_transmit_buffers(txr)) {
2658                         device_printf(dev,
2659                             "Critical Failure setting up transmit buffers\n");
2660                         error = ENOMEM;
2661                         goto err_tx_desc;
2662                 }
2663 #if 0 /* __FreeBSD_version >= 800000 */
2664                 /* Allocate a buf ring */
2665                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2666                     M_WAITOK, &txr->tx_mtx);
2667                 if (txr->br == NULL) {
2668                         device_printf(dev,
2669                             "Critical Failure setting up buf ring\n");
2670                         error = ENOMEM;
2671                         goto err_tx_desc;
2672                 }
2673 #endif
2674         }
2675
2676         /*
2677          * Next the RX queues...
2678          */
2679         rsize = roundup2(adapter->num_rx_desc *
2680             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2681         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2682                 rxr = &adapter->rx_rings[i];
2683                 /* Set up some basics */
2684                 rxr->adapter = adapter;
2685                 rxr->me = i;
2686
2687                 /* Initialize the RX side lock */
2688                 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2689                     device_get_nameunit(dev), rxr->me);
2690                 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2691
2692                 if (ixgbe_dma_malloc(adapter, rsize,
2693                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2694                         device_printf(dev,
2695                             "Unable to allocate RxDescriptor memory\n");
2696                         error = ENOMEM;
2697                         goto err_rx_desc;
2698                 }
2699                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2700                 bzero((void *)rxr->rx_base, rsize);
2701
2702                 /* Allocate receive buffers for the ring*/
2703                 if (ixgbe_allocate_receive_buffers(rxr)) {
2704                         device_printf(dev,
2705                             "Critical Failure setting up receive buffers\n");
2706                         error = ENOMEM;
2707                         goto err_rx_desc;
2708                 }
2709         }
2710
2711         /*
2712         ** Finally set up the queue holding structs
2713         */
2714         for (int i = 0; i < adapter->num_queues; i++) {
2715                 que = &adapter->queues[i];
2716                 que->adapter = adapter;
2717                 que->txr = &adapter->tx_rings[i];
2718                 que->rxr = &adapter->rx_rings[i];
2719         }
2720
2721         return (0);
2722
2723 err_rx_desc:
2724         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2725                 ixgbe_dma_free(adapter, &rxr->rxdma);
2726 err_tx_desc:
2727         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2728                 ixgbe_dma_free(adapter, &txr->txdma);
2729         kfree(adapter->rx_rings, M_DEVBUF);
2730 rx_fail:
2731         kfree(adapter->tx_rings, M_DEVBUF);
2732 tx_fail:
2733         kfree(adapter->queues, M_DEVBUF);
2734 fail:
2735         return (error);
2736 }
2737
2738 /*********************************************************************
2739  *
2740  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2741  *  the information needed to transmit a packet on the wire. This is
2742  *  called only once at attach, setup is done every reset.
2743  *
2744  **********************************************************************/
2745 static int
2746 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2747 {
2748         struct adapter *adapter = txr->adapter;
2749         device_t dev = adapter->dev;
2750         struct ixgbe_tx_buf *txbuf;
2751         int error, i;
2752
2753         /*
2754          * Setup DMA descriptor areas.
2755          */
2756         if ((error = bus_dma_tag_create(
2757                                NULL,    /* parent */
2758                                1, 0,            /* alignment, bounds */
2759                                BUS_SPACE_MAXADDR,       /* lowaddr */
2760                                BUS_SPACE_MAXADDR,       /* highaddr */
2761                                NULL, NULL,              /* filter, filterarg */
2762                                IXGBE_TSO_SIZE,          /* maxsize */
2763                                adapter->num_segs,       /* nsegments */
2764                                PAGE_SIZE,               /* maxsegsize */
2765                                0,                       /* flags */
2766                                &txr->txtag))) {
2767                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2768                 goto fail;
2769         }
2770
2771         if (!(txr->tx_buffers =
2772             (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2773             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2774                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2775                 error = ENOMEM;
2776                 goto fail;
2777         }
2778
2779         /* Create the descriptor buffer dma maps */
2780         txbuf = txr->tx_buffers;
2781         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2782                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2783                 if (error != 0) {
2784                         device_printf(dev, "Unable to create TX DMA map\n");
2785                         goto fail;
2786                 }
2787         }
2788
2789         return 0;
2790 fail:
2791         /* We free all, it handles case where we are in the middle */
2792         ixgbe_free_transmit_structures(adapter);
2793         return (error);
2794 }
2795
2796 /*********************************************************************
2797  *
2798  *  Initialize a transmit ring.
2799  *
2800  **********************************************************************/
2801 static void
2802 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2803 {
2804         struct adapter *adapter = txr->adapter;
2805         struct ixgbe_tx_buf *txbuf;
2806         int i;
2807 #ifdef DEV_NETMAP
2808         struct netmap_adapter *na = NA(adapter->ifp);
2809         struct netmap_slot *slot;
2810 #endif /* DEV_NETMAP */
2811
2812         /* Clear the old ring contents */
2813         IXGBE_TX_LOCK(txr);
2814 #ifdef DEV_NETMAP
2815         /*
2816          * (under lock): if in netmap mode, do some consistency
2817          * checks and set slot to entry 0 of the netmap ring.
2818          */
2819         slot = netmap_reset(na, NR_TX, txr->me, 0);
2820 #endif /* DEV_NETMAP */
2821         bzero((void *)txr->tx_base,
2822               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
2823         /* Reset indices */
2824         txr->next_avail_desc = 0;
2825         txr->next_to_clean = 0;
2826
2827         /* Free any existing tx buffers. */
2828         txbuf = txr->tx_buffers;
2829         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2830                 if (txbuf->m_head != NULL) {
2831                         bus_dmamap_sync(txr->txtag, txbuf->map,
2832                             BUS_DMASYNC_POSTWRITE);
2833                         bus_dmamap_unload(txr->txtag, txbuf->map);
2834                         m_freem(txbuf->m_head);
2835                         txbuf->m_head = NULL;
2836                 }
2837 #ifdef DEV_NETMAP
2838                 /*
2839                  * In netmap mode, set the map for the packet buffer.
2840                  * NOTE: Some drivers (not this one) also need to set
2841                  * the physical buffer address in the NIC ring.
2842                  * Slots in the netmap ring (indexed by "si") are
2843                  * kring->nkr_hwofs positions "ahead" wrt the
2844                  * corresponding slot in the NIC ring. In some drivers
2845                  * (not here) nkr_hwofs can be negative. Function
2846                  * netmap_idx_n2k() handles wraparounds properly.
2847                  */
2848                 if (slot) {
2849                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
2850                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
2851                 }
2852 #endif /* DEV_NETMAP */
2853                 /* Clear the EOP index */
2854                 txbuf->eop_index = -1;
2855         }
2856
2857 #ifdef IXGBE_FDIR
2858         /* Set the rate at which we sample packets */
2859         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
2860                 txr->atr_sample = atr_sample_rate;
2861 #endif
2862
2863         /* Set number of descriptors available */
2864         txr->tx_avail = adapter->num_tx_desc;
2865
2866         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2867             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2868         IXGBE_TX_UNLOCK(txr);
2869 }
2870
2871 /*********************************************************************
2872  *
2873  *  Initialize all transmit rings.
2874  *
2875  **********************************************************************/
2876 static int
2877 ixgbe_setup_transmit_structures(struct adapter *adapter)
2878 {
2879         struct tx_ring *txr = adapter->tx_rings;
2880
2881         for (int i = 0; i < adapter->num_queues; i++, txr++)
2882                 ixgbe_setup_transmit_ring(txr);
2883
2884         return (0);
2885 }
2886
2887 /*********************************************************************
2888  *
2889  *  Enable transmit unit.
2890  *
2891  **********************************************************************/
2892 static void
2893 ixgbe_initialize_transmit_units(struct adapter *adapter)
2894 {
2895         struct tx_ring  *txr = adapter->tx_rings;
2896         struct ixgbe_hw *hw = &adapter->hw;
2897
2898         /* Setup the Base and Length of the Tx Descriptor Ring */
2899
2900         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2901                 u64     tdba = txr->txdma.dma_paddr;
2902                 u32     txctrl;
2903
2904                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
2905                        (tdba & 0x00000000ffffffffULL));
2906                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
2907                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
2908                     adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
2909
2910                 /* Setup the HW Tx Head and Tail descriptor pointers */
2911                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
2912                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
2913
2914                 /* Setup Transmit Descriptor Cmd Settings */
2915                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
2916                 txr->queue_status = IXGBE_QUEUE_IDLE;
2917
2918                 /* Disable Head Writeback */
2919                 switch (hw->mac.type) {
2920                 case ixgbe_mac_82598EB:
2921                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
2922                         break;
2923                 case ixgbe_mac_82599EB:
2924                 case ixgbe_mac_X540:
2925                 default:
2926                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
2927                         break;
2928                 }
2929                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2930                 switch (hw->mac.type) {
2931                 case ixgbe_mac_82598EB:
2932                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
2933                         break;
2934                 case ixgbe_mac_82599EB:
2935                 case ixgbe_mac_X540:
2936                 default:
2937                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
2938                         break;
2939                 }
2940
2941         }
2942
2943         if (hw->mac.type != ixgbe_mac_82598EB) {
2944                 u32 dmatxctl, rttdcs;
2945                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2946                 dmatxctl |= IXGBE_DMATXCTL_TE;
2947                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2948                 /* Disable arbiter to set MTQC */
2949                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2950                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
2951                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2952                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2953                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2954                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2955         }
2956
2957         return;
2958 }
2959
2960 /*********************************************************************
2961  *
2962  *  Free all transmit rings.
2963  *
2964  **********************************************************************/
2965 static void
2966 ixgbe_free_transmit_structures(struct adapter *adapter)
2967 {
2968         struct tx_ring *txr = adapter->tx_rings;
2969
2970         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2971                 IXGBE_TX_LOCK(txr);
2972                 ixgbe_free_transmit_buffers(txr);
2973                 ixgbe_dma_free(adapter, &txr->txdma);
2974                 IXGBE_TX_UNLOCK(txr);
2975                 IXGBE_TX_LOCK_DESTROY(txr);
2976         }
2977         kfree(adapter->tx_rings, M_DEVBUF);
2978 }
2979
2980 /*********************************************************************
2981  *
2982  *  Free transmit ring related data structures.
2983  *
2984  **********************************************************************/
2985 static void
2986 ixgbe_free_transmit_buffers(struct tx_ring *txr)
2987 {
2988         struct adapter *adapter = txr->adapter;
2989         struct ixgbe_tx_buf *tx_buffer;
2990         int             i;
2991
2992         INIT_DEBUGOUT("free_transmit_ring: begin");
2993
2994         if (txr->tx_buffers == NULL)
2995                 return;
2996
2997         tx_buffer = txr->tx_buffers;
2998         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2999                 if (tx_buffer->m_head != NULL) {
3000                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3001                             BUS_DMASYNC_POSTWRITE);
3002                         bus_dmamap_unload(txr->txtag,
3003                             tx_buffer->map);
3004                         m_freem(tx_buffer->m_head);
3005                         tx_buffer->m_head = NULL;
3006                         if (tx_buffer->map != NULL) {
3007                                 bus_dmamap_destroy(txr->txtag,
3008                                     tx_buffer->map);
3009                                 tx_buffer->map = NULL;
3010                         }
3011                 } else if (tx_buffer->map != NULL) {
3012                         bus_dmamap_unload(txr->txtag,
3013                             tx_buffer->map);
3014                         bus_dmamap_destroy(txr->txtag,
3015                             tx_buffer->map);
3016                         tx_buffer->map = NULL;
3017                 }
3018         }
3019 #if 0 /* __FreeBSD_version >= 800000 */
3020         if (txr->br != NULL)
3021                 buf_ring_free(txr->br, M_DEVBUF);
3022 #endif
3023         if (txr->tx_buffers != NULL) {
3024                 kfree(txr->tx_buffers, M_DEVBUF);
3025                 txr->tx_buffers = NULL;
3026         }
3027         if (txr->txtag != NULL) {
3028                 bus_dma_tag_destroy(txr->txtag);
3029                 txr->txtag = NULL;
3030         }
3031         return;
3032 }
3033
3034 /*********************************************************************
3035  *
3036  *  Advanced Context Descriptor setup for VLAN or CSUM
3037  *
3038  **********************************************************************/
3039
3040 static bool
3041 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3042 {
3043         struct adapter *adapter = txr->adapter;
3044         struct ixgbe_adv_tx_context_desc *TXD;
3045         struct ixgbe_tx_buf        *tx_buffer;
3046         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3047         struct ether_vlan_header *eh;
3048         struct ip *ip;
3049         struct ip6_hdr *ip6;
3050         int  ehdrlen, ip_hlen = 0;
3051         u16     etype;
3052         u8      ipproto = 0;
3053         bool    offload = TRUE;
3054         int ctxd = txr->next_avail_desc;
3055         u16 vtag = 0;
3056
3057
3058         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3059                 offload = FALSE;
3060
3061         tx_buffer = &txr->tx_buffers[ctxd];
3062         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3063
3064         /*
3065         ** In advanced descriptors the vlan tag must
3066         ** be placed into the descriptor itself.
3067         */
3068         if (mp->m_flags & M_VLANTAG) {
3069                 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3070                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3071         } else if (offload == FALSE)
3072                 return FALSE;
3073
3074         /*
3075          * Determine where frame payload starts.
3076          * Jump over vlan headers if already present,
3077          * helpful for QinQ too.
3078          */
3079         eh = mtod(mp, struct ether_vlan_header *);
3080         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3081                 etype = ntohs(eh->evl_proto);
3082                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3083         } else {
3084                 etype = ntohs(eh->evl_encap_proto);
3085                 ehdrlen = ETHER_HDR_LEN;
3086         }
3087
3088         /* Set the ether header length */
3089         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3090
3091         switch (etype) {
3092                 case ETHERTYPE_IP:
3093                         ip = (struct ip *)(mp->m_data + ehdrlen);
3094                         ip_hlen = ip->ip_hl << 2;
3095                         ipproto = ip->ip_p;
3096                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3097                         break;
3098                 case ETHERTYPE_IPV6:
3099                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3100                         ip_hlen = sizeof(struct ip6_hdr);
3101                         /* XXX-BZ this will go badly in case of ext hdrs. */
3102                         ipproto = ip6->ip6_nxt;
3103                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3104                         break;
3105                 default:
3106                         offload = FALSE;
3107                         break;
3108         }
3109
3110         vlan_macip_lens |= ip_hlen;
3111         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3112
3113         switch (ipproto) {
3114                 case IPPROTO_TCP:
3115                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3116                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3117                         break;
3118
3119                 case IPPROTO_UDP:
3120                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3121                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3122                         break;
3123
3124 #if 0
3125                 case IPPROTO_SCTP:
3126                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3127                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3128                         break;
3129 #endif
3130                 default:
3131                         offload = FALSE;
3132                         break;
3133         }
3134
3135         /* Now copy bits into descriptor */
3136         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3137         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3138         TXD->seqnum_seed = htole32(0);
3139         TXD->mss_l4len_idx = htole32(0);
3140
3141         tx_buffer->m_head = NULL;
3142         tx_buffer->eop_index = -1;
3143
3144         /* We've consumed the first desc, adjust counters */
3145         if (++ctxd == adapter->num_tx_desc)
3146                 ctxd = 0;
3147         txr->next_avail_desc = ctxd;
3148         --txr->tx_avail;
3149
3150         return (offload);
3151 }
3152
3153 /**********************************************************************
3154  *
3155  *  Setup work for hardware segmentation offload (TSO) on
3156  *  adapters using advanced tx descriptors
3157  *
3158  **********************************************************************/
3159 static bool
3160 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3161     u32 *olinfo_status)
3162 {
3163         struct adapter *adapter = txr->adapter;
3164         struct ixgbe_adv_tx_context_desc *TXD;
3165         struct ixgbe_tx_buf        *tx_buffer;
3166         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3167         u16 vtag = 0, eh_type;
3168         u32 mss_l4len_idx = 0, len;
3169         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3170         struct ether_vlan_header *eh;
3171 #if 0 /* IPv6 TSO */
3172 #ifdef INET6
3173         struct ip6_hdr *ip6;
3174 #endif
3175 #endif
3176 #ifdef INET
3177         struct ip *ip;
3178 #endif
3179         struct tcphdr *th;
3180
3181
3182         /*
3183          * Determine where frame payload starts.
3184          * Jump over vlan headers if already present
3185          */
3186         eh = mtod(mp, struct ether_vlan_header *);
3187         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3188                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3189                 eh_type = eh->evl_proto;
3190         } else {
3191                 ehdrlen = ETHER_HDR_LEN;
3192                 eh_type = eh->evl_encap_proto;
3193         }
3194
3195         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3196         len = ehdrlen + sizeof(struct tcphdr);
3197         switch (ntohs(eh_type)) {
3198 #if 0 /* IPv6 TSO */
3199 #ifdef INET6
3200         case ETHERTYPE_IPV6:
3201                 if (mp->m_len < len + sizeof(struct ip6_hdr))
3202                         return FALSE;
3203                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3204                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3205                 if (ip6->ip6_nxt != IPPROTO_TCP)
3206                         return FALSE;
3207                 ip_hlen = sizeof(struct ip6_hdr);
3208                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3209                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3210                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3211                 break;
3212 #endif
3213 #endif
3214 #ifdef INET
3215         case ETHERTYPE_IP:
3216                 if (mp->m_len < len + sizeof(struct ip))
3217                         return FALSE;
3218                 ip = (struct ip *)(mp->m_data + ehdrlen);
3219                 if (ip->ip_p != IPPROTO_TCP)
3220                         return FALSE;
3221                 ip->ip_sum = 0;
3222                 ip_hlen = ip->ip_hl << 2;
3223                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3224                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3225                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3226                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3227                 /* Tell transmit desc to also do IPv4 checksum. */
3228                 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3229                 break;
3230 #endif
3231         default:
3232                 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3233                     __func__, ntohs(eh_type));
3234                 break;
3235         }
3236
3237         ctxd = txr->next_avail_desc;
3238         tx_buffer = &txr->tx_buffers[ctxd];
3239         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3240
3241         tcp_hlen = th->th_off << 2;
3242
3243         /* This is used in the transmit desc in encap */
3244         *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3245
3246         /* VLAN MACLEN IPLEN */
3247         if (mp->m_flags & M_VLANTAG) {
3248                 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3249                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3250         }
3251
3252         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3253         vlan_macip_lens |= ip_hlen;
3254         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3255
3256         /* ADV DTYPE TUCMD */
3257         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3258         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3259         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3260
3261         /* MSS L4LEN IDX */
3262         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3263         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3264         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3265
3266         TXD->seqnum_seed = htole32(0);
3267         tx_buffer->m_head = NULL;
3268         tx_buffer->eop_index = -1;
3269
3270         if (++ctxd == adapter->num_tx_desc)
3271                 ctxd = 0;
3272
3273         txr->tx_avail--;
3274         txr->next_avail_desc = ctxd;
3275         return TRUE;
3276 }
3277
3278 #ifdef IXGBE_FDIR
3279 /*
3280 ** This routine parses packet headers so that Flow
3281 ** Director can make a hashed filter table entry
3282 ** allowing traffic flows to be identified and kept
3283 ** on the same cpu.  This would be a performance
3284 ** hit, but we only do it at IXGBE_FDIR_RATE of
3285 ** packets.
3286 */
3287 static void
3288 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3289 {
3290         struct adapter                  *adapter = txr->adapter;
3291         struct ix_queue                 *que;
3292         struct ip                       *ip;
3293         struct tcphdr                   *th;
3294         struct udphdr                   *uh;
3295         struct ether_vlan_header        *eh;
3296         union ixgbe_atr_hash_dword      input = {.dword = 0};
3297         union ixgbe_atr_hash_dword      common = {.dword = 0};
3298         int                             ehdrlen, ip_hlen;
3299         u16                             etype;
3300
3301         eh = mtod(mp, struct ether_vlan_header *);
3302         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3303                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3304                 etype = eh->evl_proto;
3305         } else {
3306                 ehdrlen = ETHER_HDR_LEN;
3307                 etype = eh->evl_encap_proto;
3308         }
3309
3310         /* Only handling IPv4 */
3311         if (etype != htons(ETHERTYPE_IP))
3312                 return;
3313
3314         ip = (struct ip *)(mp->m_data + ehdrlen);
3315         ip_hlen = ip->ip_hl << 2;
3316
3317         /* check if we're UDP or TCP */
3318         switch (ip->ip_p) {
3319         case IPPROTO_TCP:
3320                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3321                 /* src and dst are inverted */
3322                 common.port.dst ^= th->th_sport;
3323                 common.port.src ^= th->th_dport;
3324                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3325                 break;
3326         case IPPROTO_UDP:
3327                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3328                 /* src and dst are inverted */
3329                 common.port.dst ^= uh->uh_sport;
3330                 common.port.src ^= uh->uh_dport;
3331                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3332                 break;
3333         default:
3334                 return;
3335         }
3336
3337         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3338         if (mp->m_pkthdr.ether_vtag)
3339                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3340         else
3341                 common.flex_bytes ^= etype;
3342         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3343
3344         que = &adapter->queues[txr->me];
3345         /*
3346         ** This assumes the Rx queue and Tx
3347         ** queue are bound to the same CPU
3348         */
3349         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3350             input, common, que->msix);
3351 }
3352 #endif /* IXGBE_FDIR */
3353
3354 /**********************************************************************
3355  *
3356  *  Examine each tx_buffer in the used queue. If the hardware is done
3357  *  processing the packet then free associated resources. The
3358  *  tx_buffer is put back on the free queue.
3359  *
3360  **********************************************************************/
3361 static void
3362 ixgbe_txeof(struct tx_ring *txr)
3363 {
3364         struct adapter  *adapter = txr->adapter;
3365         struct ifnet    *ifp = adapter->ifp;
3366         u32     first, last, done, processed;
3367         struct ixgbe_tx_buf *tx_buffer;
3368         struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3369
3370         IXGBE_TX_LOCK_ASSERT(txr);
3371
3372 #ifdef DEV_NETMAP
3373         if (ifp->if_capenable & IFCAP_NETMAP) {
3374                 struct netmap_adapter *na = NA(ifp);
3375                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3376
3377                 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3378
3379                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3380                     BUS_DMASYNC_POSTREAD);
3381                 /*
3382                  * In netmap mode, all the work is done in the context
3383                  * of the client thread. Interrupt handlers only wake up
3384                  * clients, which may be sleeping on individual rings
3385                  * or on a global resource for all rings.
3386                  * To implement tx interrupt mitigation, we wake up the client
3387                  * thread roughly every half ring, even if the NIC interrupts
3388                  * more frequently. This is implemented as follows:
3389                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3390                  *   the slot that should wake up the thread (nkr_num_slots
3391                  *   means the user thread should not be woken up);
3392                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3393                  *   or the slot has the DD bit set.
3394                  *
3395                  * When the driver has separate locks, we need to
3396                  * release and re-acquire txlock to avoid deadlocks.
3397                  * XXX see if we can find a better way.
3398                  */
3399                 if (!netmap_mitigate ||
3400                     (kring->nr_kflags < kring->nkr_num_slots &&
3401                      tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3402                         kring->nr_kflags = kring->nkr_num_slots;
3403                         selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3404                         IXGBE_TX_UNLOCK(txr);
3405                         IXGBE_CORE_LOCK(adapter);
3406                         selwakeuppri(&na->tx_si, PI_NET);
3407                         IXGBE_CORE_UNLOCK(adapter);
3408                         IXGBE_TX_LOCK(txr);
3409                 }
3410                 return FALSE;
3411         }
3412 #endif /* DEV_NETMAP */
3413
3414         if (txr->tx_avail == adapter->num_tx_desc) {
3415                 txr->queue_status = IXGBE_QUEUE_IDLE;
3416                 return;
3417         }
3418
3419         processed = 0;
3420         first = txr->next_to_clean;
3421         tx_buffer = &txr->tx_buffers[first];
3422         /* For cleanup we just use legacy struct */
3423         tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3424         last = tx_buffer->eop_index;
3425         if (last == -1)
3426                 return;
3427         eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3428
3429         /*
3430         ** Get the index of the first descriptor
3431         ** BEYOND the EOP and call that 'done'.
3432         ** I do this so the comparison in the
3433         ** inner while loop below can be simple
3434         */
3435         if (++last == adapter->num_tx_desc) last = 0;
3436         done = last;
3437
3438         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3439             BUS_DMASYNC_POSTREAD);
3440         /*
3441         ** Only the EOP descriptor of a packet now has the DD
3442         ** bit set, this is what we look for...
3443         */
3444         while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3445                 /* We clean the range of the packet */
3446                 while (first != done) {
3447                         tx_desc->upper.data = 0;
3448                         tx_desc->lower.data = 0;
3449                         tx_desc->buffer_addr = 0;
3450                         ++txr->tx_avail;
3451                         ++processed;
3452
3453                         if (tx_buffer->m_head) {
3454                                 txr->bytes +=
3455                                     tx_buffer->m_head->m_pkthdr.len;
3456                                 bus_dmamap_sync(txr->txtag,
3457                                     tx_buffer->map,
3458                                     BUS_DMASYNC_POSTWRITE);
3459                                 bus_dmamap_unload(txr->txtag,
3460                                     tx_buffer->map);
3461                                 m_freem(tx_buffer->m_head);
3462                                 tx_buffer->m_head = NULL;
3463                                 tx_buffer->map = NULL;
3464                         }
3465                         tx_buffer->eop_index = -1;
3466                         txr->watchdog_time = ticks;
3467
3468                         if (++first == adapter->num_tx_desc)
3469                                 first = 0;
3470
3471                         tx_buffer = &txr->tx_buffers[first];
3472                         tx_desc =
3473                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3474                 }
3475                 ++txr->packets;
3476                 IFNET_STAT_INC(ifp, opackets, 1);
3477                 /* See if there is more work now */
3478                 last = tx_buffer->eop_index;
3479                 if (last != -1) {
3480                         eop_desc =
3481                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3482                         /* Get next done point */
3483                         if (++last == adapter->num_tx_desc) last = 0;
3484                         done = last;
3485                 } else
3486                         break;
3487         }
3488         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3489             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3490
3491         txr->next_to_clean = first;
3492
3493         /*
3494         ** Watchdog calculation, we know there's
3495         ** work outstanding or the first return
3496         ** would have been taken, so none processed
3497         ** for too long indicates a hang.
3498         */
3499         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3500                 txr->queue_status = IXGBE_QUEUE_HUNG;
3501
3502         /* With a minimum free clear the depleted state bit.  */
3503         if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3504                 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3505
3506         if (txr->tx_avail == adapter->num_tx_desc) {
3507                 txr->queue_status = IXGBE_QUEUE_IDLE;
3508         }
3509 }
3510
3511 /*********************************************************************
3512  *
3513  *  Refresh mbuf buffers for RX descriptor rings
3514  *   - now keeps its own state so discards due to resource
3515  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3516  *     it just returns, keeping its placeholder, thus it can simply
3517  *     be recalled to try again.
3518  *
3519  **********************************************************************/
3520 static void
3521 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3522 {
3523         struct adapter          *adapter = rxr->adapter;
3524         bus_dma_segment_t       hseg[1];
3525         bus_dma_segment_t       pseg[1];
3526         struct ixgbe_rx_buf     *rxbuf;
3527         struct mbuf             *mh, *mp;
3528         int                     i, j, nsegs, error;
3529         bool                    refreshed = FALSE;
3530
3531         i = j = rxr->next_to_refresh;
3532         /* Control the loop with one beyond */
3533         if (++j == adapter->num_rx_desc)
3534                 j = 0;
3535
3536         while (j != limit) {
3537                 rxbuf = &rxr->rx_buffers[i];
3538                 if (rxr->hdr_split == FALSE)
3539                         goto no_split;
3540
3541                 if (rxbuf->m_head == NULL) {
3542                         mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3543                         if (mh == NULL)
3544                                 goto update;
3545                 } else
3546                         mh = rxbuf->m_head;
3547
3548                 mh->m_pkthdr.len = mh->m_len = MHLEN;
3549                 mh->m_len = MHLEN;
3550                 mh->m_flags |= M_PKTHDR;
3551                 /* Get the memory mapping */
3552                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3553                     rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3554                 if (error != 0) {
3555                         kprintf("Refresh mbufs: hdr dmamap load"
3556                             " failure - %d\n", error);
3557                         m_free(mh);
3558                         rxbuf->m_head = NULL;
3559                         goto update;
3560                 }
3561                 rxbuf->m_head = mh;
3562                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3563                     BUS_DMASYNC_PREREAD);
3564                 rxr->rx_base[i].read.hdr_addr =
3565                     htole64(hseg[0].ds_addr);
3566
3567 no_split:
3568                 if (rxbuf->m_pack == NULL) {
3569                         mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3570                             M_PKTHDR, adapter->rx_mbuf_sz);
3571                         if (mp == NULL)
3572                                 goto update;
3573                 } else
3574                         mp = rxbuf->m_pack;
3575
3576                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3577                 /* Get the memory mapping */
3578                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3579                     rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3580                 if (error != 0) {
3581                         kprintf("Refresh mbufs: payload dmamap load"
3582                             " failure - %d\n", error);
3583                         m_free(mp);
3584                         rxbuf->m_pack = NULL;
3585                         goto update;
3586                 }
3587                 rxbuf->m_pack = mp;
3588                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3589                     BUS_DMASYNC_PREREAD);
3590                 rxr->rx_base[i].read.pkt_addr =
3591                     htole64(pseg[0].ds_addr);
3592
3593                 refreshed = TRUE;
3594                 /* Next is precalculated */
3595                 i = j;
3596                 rxr->next_to_refresh = i;
3597                 if (++j == adapter->num_rx_desc)
3598                         j = 0;
3599         }
3600 update:
3601         if (refreshed) /* Update hardware tail index */
3602                 IXGBE_WRITE_REG(&adapter->hw,
3603                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3604         return;
3605 }
3606
3607 /*********************************************************************
3608  *
3609  *  Allocate memory for rx_buffer structures. Since we use one
3610  *  rx_buffer per received packet, the maximum number of rx_buffer's
3611  *  that we'll need is equal to the number of receive descriptors
3612  *  that we've allocated.
3613  *
3614  **********************************************************************/
3615 static int
3616 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3617 {
3618         struct  adapter         *adapter = rxr->adapter;
3619         device_t                dev = adapter->dev;
3620         struct ixgbe_rx_buf     *rxbuf;
3621         int                     i, bsize, error;
3622
3623         bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3624         if (!(rxr->rx_buffers =
3625             (struct ixgbe_rx_buf *) kmalloc(bsize,
3626             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3627                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3628                 error = ENOMEM;
3629                 goto fail;
3630         }
3631
3632         if ((error = bus_dma_tag_create(NULL,   /* parent */
3633                                    1, 0,        /* alignment, bounds */
3634                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3635                                    BUS_SPACE_MAXADDR,   /* highaddr */
3636                                    NULL, NULL,          /* filter, filterarg */
3637                                    MSIZE,               /* maxsize */
3638                                    1,                   /* nsegments */
3639                                    MSIZE,               /* maxsegsize */
3640                                    0,                   /* flags */
3641                                    &rxr->htag))) {
3642                 device_printf(dev, "Unable to create RX DMA tag\n");
3643                 goto fail;
3644         }
3645
3646         if ((error = bus_dma_tag_create(NULL,   /* parent */
3647                                    1, 0,        /* alignment, bounds */
3648                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3649                                    BUS_SPACE_MAXADDR,   /* highaddr */
3650                                    NULL, NULL,          /* filter, filterarg */
3651                                    MJUM16BYTES,         /* maxsize */
3652                                    1,                   /* nsegments */
3653                                    MJUM16BYTES,         /* maxsegsize */
3654                                    0,                   /* flags */
3655                                    &rxr->ptag))) {
3656                 device_printf(dev, "Unable to create RX DMA tag\n");
3657                 goto fail;
3658         }
3659
3660         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3661                 rxbuf = &rxr->rx_buffers[i];
3662                 error = bus_dmamap_create(rxr->htag,
3663                     BUS_DMA_NOWAIT, &rxbuf->hmap);
3664                 if (error) {
3665                         device_printf(dev, "Unable to create RX head map\n");
3666                         goto fail;
3667                 }
3668                 error = bus_dmamap_create(rxr->ptag,
3669                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3670                 if (error) {
3671                         device_printf(dev, "Unable to create RX pkt map\n");
3672                         goto fail;
3673                 }
3674         }
3675
3676         return (0);
3677
3678 fail:
3679         /* Frees all, but can handle partial completion */
3680         ixgbe_free_receive_structures(adapter);
3681         return (error);
3682 }
3683
3684 /*
3685 ** Used to detect a descriptor that has
3686 ** been merged by Hardware RSC.
3687 */
3688 static inline u32
3689 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3690 {
3691         return (le32toh(rx->wb.lower.lo_dword.data) &
3692             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3693 }
3694
3695 /*********************************************************************
3696  *
3697  *  Initialize Hardware RSC (LRO) feature on 82599
3698  *  for an RX ring, this is toggled by the LRO capability
3699  *  even though it is transparent to the stack.
3700  *
3701  **********************************************************************/
3702 #if 0   /* NET_LRO */
3703 static void
3704 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3705 {
3706         struct  adapter         *adapter = rxr->adapter;
3707         struct  ixgbe_hw        *hw = &adapter->hw;
3708         u32                     rscctrl, rdrxctl;
3709
3710         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3711         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3712 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3713         if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3714 #endif /* DEV_NETMAP */
3715         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3716         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3717         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3718
3719         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3720         rscctrl |= IXGBE_RSCCTL_RSCEN;
3721         /*
3722         ** Limit the total number of descriptors that
3723         ** can be combined, so it does not exceed 64K
3724         */
3725         if (adapter->rx_mbuf_sz == MCLBYTES)
3726                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3727         else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3728                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3729         else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3730                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3731         else  /* Using 16K cluster */
3732                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3733
3734         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3735
3736         /* Enable TCP header recognition */
3737         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3738             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3739             IXGBE_PSRTYPE_TCPHDR));
3740
3741         /* Disable RSC for ACK packets */
3742         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3743             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3744
3745         rxr->hw_rsc = TRUE;
3746 }
3747 #endif
3748
3749 static void
3750 ixgbe_free_receive_ring(struct rx_ring *rxr)
3751 {
3752         struct  adapter         *adapter;
3753         struct ixgbe_rx_buf       *rxbuf;
3754         int i;
3755
3756         adapter = rxr->adapter;
3757         for (i = 0; i < adapter->num_rx_desc; i++) {
3758                 rxbuf = &rxr->rx_buffers[i];
3759                 if (rxbuf->m_head != NULL) {
3760                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3761                             BUS_DMASYNC_POSTREAD);
3762                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3763                         rxbuf->m_head->m_flags |= M_PKTHDR;
3764                         m_freem(rxbuf->m_head);
3765                 }
3766                 if (rxbuf->m_pack != NULL) {
3767                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3768                             BUS_DMASYNC_POSTREAD);
3769                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3770                         rxbuf->m_pack->m_flags |= M_PKTHDR;
3771                         m_freem(rxbuf->m_pack);
3772                 }
3773                 rxbuf->m_head = NULL;
3774                 rxbuf->m_pack = NULL;
3775         }
3776 }
3777
3778
3779 /*********************************************************************
3780  *
3781  *  Initialize a receive ring and its buffers.
3782  *
3783  **********************************************************************/
3784 static int
3785 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3786 {
3787         struct  adapter         *adapter;
3788 #if 0 /* NET_LRO */
3789         struct ifnet            *ifp;
3790         device_t                dev;
3791 #endif
3792         struct ixgbe_rx_buf     *rxbuf;
3793         bus_dma_segment_t       pseg[1], hseg[1];
3794 #if 0   /* NET_LRO */
3795         struct lro_ctrl         *lro = &rxr->lro;
3796 #endif
3797         int                     rsize, nsegs, error = 0;
3798 #ifdef DEV_NETMAP
3799         struct netmap_adapter *na = NA(rxr->adapter->ifp);
3800         struct netmap_slot *slot;
3801 #endif /* DEV_NETMAP */
3802
3803         adapter = rxr->adapter;
3804 #if 0 /* NET_LRO */
3805         ifp = adapter->ifp;
3806         dev = adapter->dev;
3807 #endif
3808
3809         /* Clear the ring contents */
3810         IXGBE_RX_LOCK(rxr);
3811 #ifdef DEV_NETMAP
3812         /* same as in ixgbe_setup_transmit_ring() */
3813         slot = netmap_reset(na, NR_RX, rxr->me, 0);
3814 #endif /* DEV_NETMAP */
3815         rsize = roundup2(adapter->num_rx_desc *
3816             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3817         bzero((void *)rxr->rx_base, rsize);
3818
3819         /* Free current RX buffer structs and their mbufs */
3820         ixgbe_free_receive_ring(rxr);
3821
3822         /* Configure header split? */
3823         if (ixgbe_header_split)
3824                 rxr->hdr_split = TRUE;
3825
3826         /* Now replenish the mbufs */
3827         for (int j = 0; j != adapter->num_rx_desc; ++j) {
3828                 struct mbuf     *mh, *mp;
3829
3830                 rxbuf = &rxr->rx_buffers[j];
3831 #ifdef DEV_NETMAP
3832                 /*
3833                  * In netmap mode, fill the map and set the buffer
3834                  * address in the NIC ring, considering the offset
3835                  * between the netmap and NIC rings (see comment in
3836                  * ixgbe_setup_transmit_ring() ). No need to allocate
3837                  * an mbuf, so end the block with a continue;
3838                  */
3839                 if (slot) {
3840                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
3841                         uint64_t paddr;
3842                         void *addr;
3843
3844                         addr = PNMB(slot + sj, &paddr);
3845                         netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
3846                         /* Update descriptor */
3847                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
3848                         continue;
3849                 }
3850 #endif /* DEV_NETMAP */
3851                 /*
3852                 ** Don't allocate mbufs if not
3853                 ** doing header split, its wasteful
3854                 */
3855                 if (rxr->hdr_split == FALSE)
3856                         goto skip_head;
3857
3858                 /* First the header */
3859                 rxbuf->m_head = m_gethdr(MB_DONTWAIT, MT_DATA);
3860                 if (rxbuf->m_head == NULL) {
3861                         error = ENOBUFS;
3862                         goto fail;
3863                 }
3864                 m_adj(rxbuf->m_head, ETHER_ALIGN);
3865                 mh = rxbuf->m_head;
3866                 mh->m_len = mh->m_pkthdr.len = MHLEN;
3867                 mh->m_flags |= M_PKTHDR;
3868                 /* Get the memory mapping */
3869                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3870                     rxbuf->hmap, rxbuf->m_head, hseg, 1,
3871                     &nsegs, BUS_DMA_NOWAIT);
3872
3873                 if (error != 0) /* Nothing elegant to do here */
3874                         goto fail;
3875                 bus_dmamap_sync(rxr->htag,
3876                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
3877                 /* Update descriptor */
3878                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
3879
3880 skip_head:
3881                 /* Now the payload cluster */
3882                 rxbuf->m_pack = m_getjcl(MB_DONTWAIT, MT_DATA,
3883                     M_PKTHDR, adapter->rx_mbuf_sz);
3884                 if (rxbuf->m_pack == NULL) {
3885                         error = ENOBUFS;
3886                         goto fail;
3887                 }
3888                 mp = rxbuf->m_pack;
3889                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3890                 /* Get the memory mapping */
3891                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3892                     rxbuf->pmap, mp, hseg, 1,
3893                     &nsegs, BUS_DMA_NOWAIT);
3894                 if (error != 0)
3895                         goto fail;
3896                 bus_dmamap_sync(rxr->ptag,
3897                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
3898                 /* Update descriptor */
3899                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
3900         }
3901
3902
3903         /* Setup our descriptor indices */
3904         rxr->next_to_check = 0;
3905         rxr->next_to_refresh = 0;
3906         rxr->lro_enabled = FALSE;
3907         rxr->rx_split_packets = 0;
3908         rxr->rx_bytes = 0;
3909         rxr->discard = FALSE;
3910         rxr->vtag_strip = FALSE;
3911
3912         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3913             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3914
3915         /*
3916         ** Now set up the LRO interface:
3917         ** 82598 uses software LRO, the
3918         ** 82599 and X540 use a hardware assist.
3919         */
3920 #if 0 /* NET_LRO */
3921         if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
3922             (ifp->if_capenable & IFCAP_RXCSUM) &&
3923             (ifp->if_capenable & IFCAP_LRO))
3924                 ixgbe_setup_hw_rsc(rxr);
3925         else if (ifp->if_capenable & IFCAP_LRO) {
3926                 int err = tcp_lro_init(lro);
3927                 if (err) {
3928                         device_printf(dev, "LRO Initialization failed!\n");
3929                         goto fail;
3930                 }
3931                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
3932                 rxr->lro_enabled = TRUE;
3933                 lro->ifp = adapter->ifp;
3934         }
3935 #endif
3936
3937         IXGBE_RX_UNLOCK(rxr);
3938         return (0);
3939
3940 fail:
3941         ixgbe_free_receive_ring(rxr);
3942         IXGBE_RX_UNLOCK(rxr);
3943         return (error);
3944 }
3945
3946 /*********************************************************************
3947  *
3948  *  Initialize all receive rings.
3949  *
3950  **********************************************************************/
3951 static int
3952 ixgbe_setup_receive_structures(struct adapter *adapter)
3953 {
3954         struct rx_ring *rxr = adapter->rx_rings;
3955         int j;
3956
3957         for (j = 0; j < adapter->num_queues; j++, rxr++)
3958                 if (ixgbe_setup_receive_ring(rxr))
3959                         goto fail;
3960
3961         return (0);
3962 fail:
3963         /*
3964          * Free RX buffers allocated so far, we will only handle
3965          * the rings that completed, the failing case will have
3966          * cleaned up for itself. 'j' failed, so its the terminus.
3967          */
3968         for (int i = 0; i < j; ++i) {
3969                 rxr = &adapter->rx_rings[i];
3970                 ixgbe_free_receive_ring(rxr);
3971         }
3972
3973         return (ENOBUFS);
3974 }
3975
3976 /*********************************************************************
3977  *
3978  *  Setup receive registers and features.
3979  *
3980  **********************************************************************/
3981 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
3982
3983 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
3984
3985 static void
3986 ixgbe_initialize_receive_units(struct adapter *adapter)
3987 {
3988         struct  rx_ring *rxr = adapter->rx_rings;
3989         struct ixgbe_hw *hw = &adapter->hw;
3990         struct ifnet   *ifp = adapter->ifp;
3991         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
3992         u32             reta, mrqc = 0, hlreg, random[10];
3993
3994
3995         /*
3996          * Make sure receives are disabled while
3997          * setting up the descriptor ring
3998          */
3999         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4000         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4001             rxctrl & ~IXGBE_RXCTRL_RXEN);
4002
4003         /* Enable broadcasts */
4004         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4005         fctrl |= IXGBE_FCTRL_BAM;
4006         fctrl |= IXGBE_FCTRL_DPF;
4007         fctrl |= IXGBE_FCTRL_PMCF;
4008         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4009
4010         /* Set for Jumbo Frames? */
4011         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4012         if (ifp->if_mtu > ETHERMTU)
4013                 hlreg |= IXGBE_HLREG0_JUMBOEN;
4014         else
4015                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4016 #ifdef DEV_NETMAP
4017         /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4018         if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4019                 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4020         else
4021                 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4022 #endif /* DEV_NETMAP */
4023         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4024
4025         bufsz = (adapter->rx_mbuf_sz +
4026             BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4027
4028         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4029                 u64 rdba = rxr->rxdma.dma_paddr;
4030
4031                 /* Setup the Base and Length of the Rx Descriptor Ring */
4032                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4033                                (rdba & 0x00000000ffffffffULL));
4034                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4035                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4036                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4037
4038                 /* Set up the SRRCTL register */
4039                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4040                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4041                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4042                 srrctl |= bufsz;
4043                 if (rxr->hdr_split) {
4044                         /* Use a standard mbuf for the header */
4045                         srrctl |= ((IXGBE_RX_HDR <<
4046                             IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
4047                             & IXGBE_SRRCTL_BSIZEHDR_MASK);
4048                         srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4049                 } else
4050                         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4051                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4052
4053                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4054                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4055                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4056         }
4057
4058         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4059                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4060                               IXGBE_PSRTYPE_UDPHDR |
4061                               IXGBE_PSRTYPE_IPV4HDR |
4062                               IXGBE_PSRTYPE_IPV6HDR;
4063                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4064         }
4065
4066         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4067
4068         /* Setup RSS */
4069         if (adapter->num_queues > 1) {
4070                 int i, j;
4071                 reta = 0;
4072
4073                 /* set up random bits */
4074                 karc4rand(&random, sizeof(random));
4075
4076                 /* Set up the redirection table */
4077                 for (i = 0, j = 0; i < 128; i++, j++) {
4078                         if (j == adapter->num_queues) j = 0;
4079                         reta = (reta << 8) | (j * 0x11);
4080                         if ((i & 3) == 3)
4081                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4082                 }
4083
4084                 /* Now fill our hash function seeds */
4085                 for (int i = 0; i < 10; i++)
4086                         IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4087
4088                 /* Perform hash on these packet types */
4089                 mrqc = IXGBE_MRQC_RSSEN
4090                      | IXGBE_MRQC_RSS_FIELD_IPV4
4091                      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4092                      | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4093                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4094                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4095                      | IXGBE_MRQC_RSS_FIELD_IPV6
4096                      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4097                      | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4098                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4099                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4100
4101                 /* RSS and RX IPP Checksum are mutually exclusive */
4102                 rxcsum |= IXGBE_RXCSUM_PCSD;
4103         }
4104
4105         if (ifp->if_capenable & IFCAP_RXCSUM)
4106                 rxcsum |= IXGBE_RXCSUM_PCSD;
4107
4108         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4109                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4110
4111         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4112
4113         return;
4114 }
4115
4116 /*********************************************************************
4117  *
4118  *  Free all receive rings.
4119  *
4120  **********************************************************************/
4121 static void
4122 ixgbe_free_receive_structures(struct adapter *adapter)
4123 {
4124         struct rx_ring *rxr = adapter->rx_rings;
4125
4126         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4127 #if 0   /* NET_LRO */
4128                 struct lro_ctrl         *lro = &rxr->lro;
4129 #endif
4130                 ixgbe_free_receive_buffers(rxr);
4131                 /* Free LRO memory */
4132 #if 0   /* NET_LRO */
4133                 tcp_lro_free(lro);
4134 #endif
4135                 /* Free the ring memory as well */
4136                 ixgbe_dma_free(adapter, &rxr->rxdma);
4137         }
4138
4139         kfree(adapter->rx_rings, M_DEVBUF);
4140 }
4141
4142
4143 /*********************************************************************
4144  *
4145  *  Free receive ring data structures
4146  *
4147  **********************************************************************/
4148 static void
4149 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4150 {
4151         struct adapter          *adapter = rxr->adapter;
4152         struct ixgbe_rx_buf     *rxbuf;
4153
4154         INIT_DEBUGOUT("free_receive_structures: begin");
4155
4156         /* Cleanup any existing buffers */
4157         if (rxr->rx_buffers != NULL) {
4158                 for (int i = 0; i < adapter->num_rx_desc; i++) {
4159                         rxbuf = &rxr->rx_buffers[i];
4160                         if (rxbuf->m_head != NULL) {
4161                                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4162                                     BUS_DMASYNC_POSTREAD);
4163                                 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4164                                 rxbuf->m_head->m_flags |= M_PKTHDR;
4165                                 m_freem(rxbuf->m_head);
4166                         }
4167                         if (rxbuf->m_pack != NULL) {
4168                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4169                                     BUS_DMASYNC_POSTREAD);
4170                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4171                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
4172                                 m_freem(rxbuf->m_pack);
4173                         }
4174                         rxbuf->m_head = NULL;
4175                         rxbuf->m_pack = NULL;
4176                         if (rxbuf->hmap != NULL) {
4177                                 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4178                                 rxbuf->hmap = NULL;
4179                         }
4180                         if (rxbuf->pmap != NULL) {
4181                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4182                                 rxbuf->pmap = NULL;
4183                         }
4184                 }
4185                 if (rxr->rx_buffers != NULL) {
4186                         kfree(rxr->rx_buffers, M_DEVBUF);
4187                         rxr->rx_buffers = NULL;
4188                 }
4189         }
4190
4191         if (rxr->htag != NULL) {
4192                 bus_dma_tag_destroy(rxr->htag);
4193                 rxr->htag = NULL;
4194         }
4195         if (rxr->ptag != NULL) {
4196                 bus_dma_tag_destroy(rxr->ptag);
4197                 rxr->ptag = NULL;
4198         }
4199
4200         return;
4201 }
4202
4203 static __inline void
4204 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4205 {
4206
4207         /*
4208          * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4209          * should be computed by hardware. Also it should not have VLAN tag in
4210          * ethernet header.  In case of IPv6 we do not yet support ext. hdrs.
4211          */
4212 #if 0   /* NET_LRO */
4213         if (rxr->lro_enabled &&
4214             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4215             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4216             ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4217             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4218             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4219             (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4220             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4221             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4222                 /*
4223                  * Send to the stack if:
4224                  **  - LRO not enabled, or
4225                  **  - no LRO resources, or
4226                  **  - lro enqueue fails
4227                  */
4228                 if (rxr->lro.lro_cnt != 0)
4229                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4230                                 return;
4231         }
4232 #endif
4233         IXGBE_RX_UNLOCK(rxr);
4234         (*ifp->if_input)(ifp, m);
4235         IXGBE_RX_LOCK(rxr);
4236 }
4237
4238 static __inline void
4239 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4240 {
4241         struct ixgbe_rx_buf     *rbuf;
4242
4243         rbuf = &rxr->rx_buffers[i];
4244
4245         if (rbuf->fmp != NULL) {/* Partial chain ? */
4246                 rbuf->fmp->m_flags |= M_PKTHDR;
4247                 m_freem(rbuf->fmp);
4248                 rbuf->fmp = NULL;
4249         }
4250
4251         /*
4252         ** With advanced descriptors the writeback
4253         ** clobbers the buffer addrs, so its easier
4254         ** to just free the existing mbufs and take
4255         ** the normal refresh path to get new buffers
4256         ** and mapping.
4257         */
4258         if (rbuf->m_head) {
4259                 m_free(rbuf->m_head);
4260                 rbuf->m_head = NULL;
4261         }
4262
4263         if (rbuf->m_pack) {
4264                 m_free(rbuf->m_pack);
4265                 rbuf->m_pack = NULL;
4266         }
4267
4268         return;
4269 }
4270
4271 static void
4272 ixgbe_add_sysctl(struct adapter *adapter)
4273 {
4274         sysctl_ctx_init(&adapter->sysctl_ctx);
4275         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
4276             SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
4277             device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
4278         if (adapter->sysctl_tree == NULL) {
4279                 device_printf(adapter->dev, "can't add sysctl node\n");
4280                 return;
4281         }
4282         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4283                         SYSCTL_CHILDREN(adapter->sysctl_tree),
4284                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
4285                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
4286
4287         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4288             SYSCTL_CHILDREN(adapter->sysctl_tree),
4289             OID_AUTO, "intr_rate", CTLTYPE_INT | CTLFLAG_RW,
4290             adapter, 0, ixgbe_sysctl_intr_rate, "I", "interrupt rate");
4291
4292         /*
4293         ** Allow a kind of speed control by forcing the autoneg
4294         ** advertised speed list to only a certain value, this
4295         ** supports 1G on 82599 devices, and 100Mb on x540.
4296         */
4297         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4298                         SYSCTL_CHILDREN(adapter->sysctl_tree),
4299                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
4300                         adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
4301
4302         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4303                         SYSCTL_CHILDREN(adapter->sysctl_tree),
4304                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
4305                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
4306
4307         /* Sysctl for limiting the amount of work done in the taskqueue */
4308         ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
4309             "max number of rx packets to process", &adapter->rx_process_limit,
4310             ixgbe_rx_process_limit);
4311 }
4312
4313 /*********************************************************************
4314  *
4315  *  This routine executes in interrupt context. It replenishes
4316  *  the mbufs in the descriptor and sends data which has been
4317  *  dma'ed into host memory to upper layer.
4318  *
4319  *  We loop at most count times if count is > 0, or until done if
4320  *  count < 0.
4321  *
4322  *  Return TRUE for more work, FALSE for all clean.
4323  *********************************************************************/
4324 static void
4325 ixgbe_rxeof(struct ix_queue *que, int count)
4326 {
4327         struct adapter          *adapter = que->adapter;
4328         struct rx_ring          *rxr = que->rxr;
4329         struct ifnet            *ifp = adapter->ifp;
4330 #if 0   /* NET_LRO */
4331         struct lro_ctrl         *lro = &rxr->lro;
4332         struct lro_entry        *queued;
4333 #endif
4334         int                     i, nextp, processed = 0;
4335         u32                     staterr = 0;
4336         union ixgbe_adv_rx_desc *cur;
4337         struct ixgbe_rx_buf     *rbuf, *nbuf;
4338
4339         IXGBE_RX_LOCK(rxr);
4340
4341 #ifdef DEV_NETMAP
4342         if (ifp->if_capenable & IFCAP_NETMAP) {
4343                 /*
4344                  * Same as the txeof routine: only wakeup clients on intr.
4345                  * NKR_PENDINTR in nr_kflags is used to implement interrupt
4346                  * mitigation (ixgbe_rxsync() will not look for new packets
4347                  * unless NKR_PENDINTR is set).
4348                  */
4349                 struct netmap_adapter *na = NA(ifp);
4350
4351                 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4352                 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4353                 IXGBE_RX_UNLOCK(rxr);
4354                 IXGBE_CORE_LOCK(adapter);
4355                 selwakeuppri(&na->rx_si, PI_NET);
4356                 IXGBE_CORE_UNLOCK(adapter);
4357                 return (FALSE);
4358         }
4359 #endif /* DEV_NETMAP */
4360         for (i = rxr->next_to_check; count != 0;) {
4361                 struct mbuf     *sendmp, *mh, *mp;
4362                 u32             rsc, ptype;
4363                 u16             hlen, plen, hdr;
4364                 u16             vtag = 0;
4365                 bool            eop;
4366
4367                 /* Sync the ring. */
4368                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4369                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4370
4371                 cur = &rxr->rx_base[i];
4372                 staterr = le32toh(cur->wb.upper.status_error);
4373
4374                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4375                         break;
4376                 if ((ifp->if_flags & IFF_RUNNING) == 0)
4377                         break;
4378
4379                 count--;
4380                 sendmp = NULL;
4381                 nbuf = NULL;
4382                 rsc = 0;
4383                 cur->wb.upper.status_error = 0;
4384                 rbuf = &rxr->rx_buffers[i];
4385                 mh = rbuf->m_head;
4386                 mp = rbuf->m_pack;
4387
4388                 plen = le16toh(cur->wb.upper.length);
4389                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4390                     IXGBE_RXDADV_PKTTYPE_MASK;
4391                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4392                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4393
4394                 /* Process vlan info */
4395                 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
4396                         vtag = le16toh(cur->wb.upper.vlan);
4397
4398                 /* Make sure bad packets are discarded */
4399                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4400                     (rxr->discard)) {
4401                         IFNET_STAT_INC(ifp, ierrors, 1);
4402                         rxr->rx_discarded++;
4403                         if (eop)
4404                                 rxr->discard = FALSE;
4405                         else
4406                                 rxr->discard = TRUE;
4407                         ixgbe_rx_discard(rxr, i);
4408                         goto next_desc;
4409                 }
4410
4411                 /*
4412                 ** On 82599 which supports a hardware
4413                 ** LRO (called HW RSC), packets need
4414                 ** not be fragmented across sequential
4415                 ** descriptors, rather the next descriptor
4416                 ** is indicated in bits of the descriptor.
4417                 ** This also means that we might proceses
4418                 ** more than one packet at a time, something
4419                 ** that has never been true before, it
4420                 ** required eliminating global chain pointers
4421                 ** in favor of what we are doing here.  -jfv
4422                 */
4423                 if (!eop) {
4424                         /*
4425                         ** Figure out the next descriptor
4426                         ** of this frame.
4427                         */
4428                         if (rxr->hw_rsc == TRUE) {
4429                                 rsc = ixgbe_rsc_count(cur);
4430                                 rxr->rsc_num += (rsc - 1);
4431                         }
4432                         if (rsc) { /* Get hardware index */
4433                                 nextp = ((staterr &
4434                                     IXGBE_RXDADV_NEXTP_MASK) >>
4435                                     IXGBE_RXDADV_NEXTP_SHIFT);
4436                         } else { /* Just sequential */
4437                                 nextp = i + 1;
4438                                 if (nextp == adapter->num_rx_desc)
4439                                         nextp = 0;
4440                         }
4441                         nbuf = &rxr->rx_buffers[nextp];
4442                         prefetch(nbuf);
4443                 }
4444                 /*
4445                 ** The header mbuf is ONLY used when header
4446                 ** split is enabled, otherwise we get normal
4447                 ** behavior, ie, both header and payload
4448                 ** are DMA'd into the payload buffer.
4449                 **
4450                 ** Rather than using the fmp/lmp global pointers
4451                 ** we now keep the head of a packet chain in the
4452                 ** buffer struct and pass this along from one
4453                 ** descriptor to the next, until we get EOP.
4454                 */
4455                 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4456                         /* This must be an initial descriptor */
4457                         hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4458                             IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4459                         if (hlen > IXGBE_RX_HDR)
4460                                 hlen = IXGBE_RX_HDR;
4461                         mh->m_len = hlen;
4462                         mh->m_flags |= M_PKTHDR;
4463                         mh->m_next = NULL;
4464                         mh->m_pkthdr.len = mh->m_len;
4465                         /* Null buf pointer so it is refreshed */
4466                         rbuf->m_head = NULL;
4467                         /*
4468                         ** Check the payload length, this
4469                         ** could be zero if its a small
4470                         ** packet.
4471                         */
4472                         if (plen > 0) {
4473                                 mp->m_len = plen;
4474                                 mp->m_next = NULL;
4475                                 mp->m_flags &= ~M_PKTHDR;
4476                                 mh->m_next = mp;
4477                                 mh->m_pkthdr.len += mp->m_len;
4478                                 /* Null buf pointer so it is refreshed */
4479                                 rbuf->m_pack = NULL;
4480                                 rxr->rx_split_packets++;
4481                         }
4482                         /*
4483                         ** Now create the forward
4484                         ** chain so when complete
4485                         ** we wont have to.
4486                         */
4487                         if (eop == 0) {
4488                                 /* stash the chain head */
4489                                 nbuf->fmp = mh;
4490                                 /* Make forward chain */
4491                                 if (plen)
4492                                         mp->m_next = nbuf->m_pack;
4493                                 else
4494                                         mh->m_next = nbuf->m_pack;
4495                         } else {
4496                                 /* Singlet, prepare to send */
4497                                 sendmp = mh;
4498                                 /* If hardware handled vtag */
4499                                 if (vtag) {
4500                                         sendmp->m_pkthdr.ether_vlantag = vtag;
4501                                         sendmp->m_flags |= M_VLANTAG;
4502                                 }
4503                         }
4504                 } else {
4505                         /*
4506                         ** Either no header split, or a
4507                         ** secondary piece of a fragmented
4508                         ** split packet.
4509                         */
4510                         mp->m_len = plen;
4511                         /*
4512                         ** See if there is a stored head
4513                         ** that determines what we are
4514                         */
4515                         sendmp = rbuf->fmp;
4516                         rbuf->m_pack = rbuf->fmp = NULL;
4517
4518                         if (sendmp != NULL) {  /* secondary frag */
4519                                 mp->m_flags &= ~M_PKTHDR;
4520                                 sendmp->m_pkthdr.len += mp->m_len;
4521                         } else {
4522                                 /* first desc of a non-ps chain */
4523                                 sendmp = mp;
4524                                 sendmp->m_flags |= M_PKTHDR;
4525                                 sendmp->m_pkthdr.len = mp->m_len;
4526                                 if (staterr & IXGBE_RXD_STAT_VP) {
4527                                         sendmp->m_pkthdr.ether_vlantag = vtag;
4528                                         sendmp->m_flags |= M_VLANTAG;
4529                                 }
4530                         }
4531                         /* Pass the head pointer on */
4532                         if (eop == 0) {
4533                                 nbuf->fmp = sendmp;
4534                                 sendmp = NULL;
4535                                 mp->m_next = nbuf->m_pack;
4536                         }
4537                 }
4538                 ++processed;
4539                 /* Sending this frame? */
4540                 if (eop) {
4541                         sendmp->m_pkthdr.rcvif = ifp;
4542                         IFNET_STAT_INC(ifp, ipackets, 1);
4543                         rxr->rx_packets++;
4544                         /* capture data for AIM */
4545                         rxr->bytes += sendmp->m_pkthdr.len;
4546                         rxr->rx_bytes += sendmp->m_pkthdr.len;
4547                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4548                                 ixgbe_rx_checksum(staterr, sendmp, ptype);
4549 #if 0 /* __FreeBSD_version >= 800000 */
4550                         sendmp->m_pkthdr.flowid = que->msix;
4551                         sendmp->m_flags |= M_FLOWID;
4552 #endif
4553                 }
4554 next_desc:
4555                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4556                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4557
4558                 /* Advance our pointers to the next descriptor. */
4559                 if (++i == adapter->num_rx_desc)
4560                         i = 0;
4561
4562                 /* Now send to the stack or do LRO */
4563                 if (sendmp != NULL) {
4564                         rxr->next_to_check = i;
4565                         ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4566                         i = rxr->next_to_check;
4567                 }
4568
4569                /* Every 8 descriptors we go to refresh mbufs */
4570                 if (processed == 8) {
4571                         ixgbe_refresh_mbufs(rxr, i);
4572                         processed = 0;
4573                 }
4574         }
4575
4576         /* Refresh any remaining buf structs */
4577         if (ixgbe_rx_unrefreshed(rxr))
4578                 ixgbe_refresh_mbufs(rxr, i);
4579
4580         rxr->next_to_check = i;
4581
4582         /*
4583          * Flush any outstanding LRO work
4584          */
4585 #if 0   /* NET_LRO */
4586         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4587                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4588                 tcp_lro_flush(lro, queued);
4589         }
4590 #endif
4591
4592         IXGBE_RX_UNLOCK(rxr);
4593
4594         /*
4595         ** We still have cleaning to do?
4596         ** Schedule another interrupt if so.
4597         */
4598         if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4599                 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4600         }
4601 }
4602
4603
4604 /*********************************************************************
4605  *
4606  *  Verify that the hardware indicated that the checksum is valid.
4607  *  Inform the stack about the status of checksum so that stack
4608  *  doesn't spend time verifying the checksum.
4609  *
4610  *********************************************************************/
4611 static void
4612 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4613 {
4614         u16     status = (u16) staterr;
4615         u8      errors = (u8) (staterr >> 24);
4616         bool    sctp = FALSE;
4617
4618         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4619             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4620                 sctp = TRUE;
4621
4622         if (status & IXGBE_RXD_STAT_IPCS) {
4623                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4624                         /* IP Checksum Good */
4625                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4626                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4627
4628                 } else
4629                         mp->m_pkthdr.csum_flags = 0;
4630         }
4631         if (status & IXGBE_RXD_STAT_L4CS) {
4632                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4633 #if 0
4634                 if (sctp)
4635                         type = CSUM_SCTP_VALID;
4636 #endif
4637                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4638                         mp->m_pkthdr.csum_flags |= type;
4639                         if (!sctp)
4640                                 mp->m_pkthdr.csum_data = htons(0xffff);
4641                 }
4642         }
4643         return;
4644 }
4645
4646
4647 /*
4648 ** This routine is run via an vlan config EVENT,
4649 ** it enables us to use the HW Filter table since
4650 ** we can get the vlan id. This just creates the
4651 ** entry in the soft version of the VFTA, init will
4652 ** repopulate the real table.
4653 */
4654 static void
4655 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4656 {
4657         struct adapter  *adapter = ifp->if_softc;
4658         u16             index, bit;
4659
4660         if (ifp->if_softc !=  arg)   /* Not our event */
4661                 return;
4662
4663         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4664                 return;
4665
4666         IXGBE_CORE_LOCK(adapter);
4667         index = (vtag >> 5) & 0x7F;
4668         bit = vtag & 0x1F;
4669         adapter->shadow_vfta[index] |= (1 << bit);
4670         ++adapter->num_vlans;
4671         ixgbe_init_locked(adapter);
4672         IXGBE_CORE_UNLOCK(adapter);
4673 }
4674
4675 /*
4676 ** This routine is run via an vlan
4677 ** unconfig EVENT, remove our entry
4678 ** in the soft vfta.
4679 */
4680 static void
4681 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4682 {
4683         struct adapter  *adapter = ifp->if_softc;
4684         u16             index, bit;
4685
4686         if (ifp->if_softc !=  arg)
4687                 return;
4688
4689         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4690                 return;
4691
4692         IXGBE_CORE_LOCK(adapter);
4693         index = (vtag >> 5) & 0x7F;
4694         bit = vtag & 0x1F;
4695         adapter->shadow_vfta[index] &= ~(1 << bit);
4696         --adapter->num_vlans;
4697         /* Re-init to load the changes */
4698         ixgbe_init_locked(adapter);
4699         IXGBE_CORE_UNLOCK(adapter);
4700 }
4701
4702 static void
4703 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4704 {
4705         struct ifnet    *ifp = adapter->ifp;
4706         struct ixgbe_hw *hw = &adapter->hw;
4707         struct rx_ring  *rxr;
4708         u32             ctrl;
4709
4710         /*
4711         ** We get here thru init_locked, meaning
4712         ** a soft reset, this has already cleared
4713         ** the VFTA and other state, so if there
4714         ** have been no vlan's registered do nothing.
4715         */
4716         if (adapter->num_vlans == 0)
4717                 return;
4718
4719         /*
4720         ** A soft reset zero's out the VFTA, so
4721         ** we need to repopulate it now.
4722         */
4723         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4724                 if (adapter->shadow_vfta[i] != 0)
4725                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4726                             adapter->shadow_vfta[i]);
4727
4728         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4729         /* Enable the Filter Table if enabled */
4730         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4731                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4732                 ctrl |= IXGBE_VLNCTRL_VFE;
4733         }
4734         if (hw->mac.type == ixgbe_mac_82598EB)
4735                 ctrl |= IXGBE_VLNCTRL_VME;
4736         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4737
4738         /* Setup the queues for vlans */
4739         for (int i = 0; i < adapter->num_queues; i++) {
4740                 rxr = &adapter->rx_rings[i];
4741                 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4742                 if (hw->mac.type != ixgbe_mac_82598EB) {
4743                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4744                         ctrl |= IXGBE_RXDCTL_VME;
4745                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4746                 }
4747                 rxr->vtag_strip = TRUE;
4748         }
4749 }
4750
4751 static void
4752 ixgbe_enable_intr(struct adapter *adapter)
4753 {
4754         struct ixgbe_hw *hw = &adapter->hw;
4755         struct ix_queue *que = adapter->queues;
4756         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4757
4758
4759         /* Enable Fan Failure detection */
4760         if (hw->device_id == IXGBE_DEV_ID_82598AT)
4761                     mask |= IXGBE_EIMS_GPI_SDP1;
4762         else {
4763                     mask |= IXGBE_EIMS_ECC;
4764                     mask |= IXGBE_EIMS_GPI_SDP0;
4765                     mask |= IXGBE_EIMS_GPI_SDP1;
4766                     mask |= IXGBE_EIMS_GPI_SDP2;
4767 #ifdef IXGBE_FDIR
4768                     mask |= IXGBE_EIMS_FLOW_DIR;
4769 #endif
4770         }
4771
4772         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4773
4774         /* With RSS we use auto clear */
4775         if (adapter->msix_mem) {
4776                 mask = IXGBE_EIMS_ENABLE_MASK;
4777                 /* Don't autoclear Link */
4778                 mask &= ~IXGBE_EIMS_OTHER;
4779                 mask &= ~IXGBE_EIMS_LSC;
4780                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4781         }
4782
4783         /*
4784         ** Now enable all queues, this is done separately to
4785         ** allow for handling the extended (beyond 32) MSIX
4786         ** vectors that can be used by 82599
4787         */
4788         for (int i = 0; i < adapter->num_queues; i++, que++)
4789                 ixgbe_enable_queue(adapter, que->msix);
4790
4791         IXGBE_WRITE_FLUSH(hw);
4792
4793         return;
4794 }
4795
4796 static void
4797 ixgbe_disable_intr(struct adapter *adapter)
4798 {
4799         if (adapter->msix_mem)
4800                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4801         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4802                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4803         } else {
4804                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4805                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4806                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4807         }
4808         IXGBE_WRITE_FLUSH(&adapter->hw);
4809         return;
4810 }
4811
4812 u16
4813 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4814 {
4815         u16 value;
4816
4817         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4818             reg, 2);
4819
4820         return (value);
4821 }
4822
4823 void
4824 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4825 {
4826         pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4827             reg, value, 2);
4828
4829         return;
4830 }
4831
4832 /*
4833 ** Setup the correct IVAR register for a particular MSIX interrupt
4834 **   (yes this is all very magic and confusing :)
4835 **  - entry is the register array entry
4836 **  - vector is the MSIX vector for this queue
4837 **  - type is RX/TX/MISC
4838 */
4839 static void
4840 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4841 {
4842         struct ixgbe_hw *hw = &adapter->hw;
4843         u32 ivar, index;
4844
4845         vector |= IXGBE_IVAR_ALLOC_VAL;
4846
4847         switch (hw->mac.type) {
4848
4849         case ixgbe_mac_82598EB:
4850                 if (type == -1)
4851                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4852                 else
4853                         entry += (type * 64);
4854                 index = (entry >> 2) & 0x1F;
4855                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4856                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4857                 ivar |= (vector << (8 * (entry & 0x3)));
4858                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
4859                 break;
4860
4861         case ixgbe_mac_82599EB:
4862         case ixgbe_mac_X540:
4863                 if (type == -1) { /* MISC IVAR */
4864                         index = (entry & 1) * 8;
4865                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4866                         ivar &= ~(0xFF << index);
4867                         ivar |= (vector << index);
4868                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4869                 } else {        /* RX/TX IVARS */
4870                         index = (16 * (entry & 1)) + (8 * type);
4871                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4872                         ivar &= ~(0xFF << index);
4873                         ivar |= (vector << index);
4874                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4875                 }
4876
4877         default:
4878                 break;
4879         }
4880 }
4881
4882 static void
4883 ixgbe_configure_ivars(struct adapter *adapter)
4884 {
4885         struct  ix_queue *que = adapter->queues;
4886
4887         for (int i = 0; i < adapter->num_queues; i++, que++) {
4888                 /* First the RX queue entry */
4889                 ixgbe_set_ivar(adapter, i, que->msix, 0);
4890                 /* ... and the TX */
4891                 ixgbe_set_ivar(adapter, i, que->msix, 1);
4892                 /* Set an Initial EITR value */
4893                 ixgbe_set_eitr(adapter, que->msix, IXGBE_INTR_RATE);
4894         }
4895
4896         /* For the Link interrupt */
4897         ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
4898 }
4899
4900 /*
4901 ** ixgbe_sfp_probe - called in the local timer to
4902 ** determine if a port had optics inserted.
4903 */
4904 static bool ixgbe_sfp_probe(struct adapter *adapter)
4905 {
4906         struct ixgbe_hw *hw = &adapter->hw;
4907         device_t        dev = adapter->dev;
4908         bool            result = FALSE;
4909
4910         if ((hw->phy.type == ixgbe_phy_nl) &&
4911             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
4912                 s32 ret = hw->phy.ops.identify_sfp(hw);
4913                 if (ret)
4914                         goto out;
4915                 ret = hw->phy.ops.reset(hw);
4916                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4917                         device_printf(dev,"Unsupported SFP+ module detected!");
4918                         kprintf(" Reload driver with supported module.\n");
4919                         adapter->sfp_probe = FALSE;
4920                         goto out;
4921                 } else
4922                         device_printf(dev,"SFP+ module detected!\n");
4923                 /* We now have supported optics */
4924                 adapter->sfp_probe = FALSE;
4925                 /* Set the optics type so system reports correctly */
4926                 ixgbe_setup_optics(adapter);
4927                 result = TRUE;
4928         }
4929 out:
4930         return (result);
4931 }
4932
4933 /*
4934 ** Tasklet for handling SFP module interrupts
4935 */
4936 static void
4937 ixgbe_handle_mod(void *context, int pending)
4938 {
4939         struct adapter  *adapter = context;
4940         struct ixgbe_hw *hw = &adapter->hw;
4941         device_t        dev = adapter->dev;
4942         u32 err;
4943
4944         err = hw->phy.ops.identify_sfp(hw);
4945         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4946                 device_printf(dev,
4947                     "Unsupported SFP+ module type was detected.\n");
4948                 return;
4949         }
4950         err = hw->mac.ops.setup_sfp(hw);
4951         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4952                 device_printf(dev,
4953                     "Setup failure - unsupported SFP+ module type.\n");
4954                 return;
4955         }
4956         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
4957         return;
4958 }
4959
4960
4961 /*
4962 ** Tasklet for handling MSF (multispeed fiber) interrupts
4963 */
4964 static void
4965 ixgbe_handle_msf(void *context, int pending)
4966 {
4967         struct adapter  *adapter = context;
4968         struct ixgbe_hw *hw = &adapter->hw;
4969         u32 autoneg;
4970         bool negotiate;
4971
4972         autoneg = hw->phy.autoneg_advertised;
4973         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
4974                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
4975         if (hw->mac.ops.setup_link)
4976                 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
4977         return;
4978 }
4979
4980 #ifdef IXGBE_FDIR
4981 /*
4982 ** Tasklet for reinitializing the Flow Director filter table
4983 */
4984 static void
4985 ixgbe_reinit_fdir(void *context, int pending)
4986 {
4987         struct adapter  *adapter = context;
4988         struct ifnet   *ifp = adapter->ifp;
4989
4990         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
4991                 return;
4992         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
4993         adapter->fdir_reinit = 0;
4994         /* re-enable flow director interrupts */
4995         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
4996         /* Restart the interface */
4997         ifp->if_drv_flags |= IFF_DRV_RUNNING;
4998         return;
4999 }
5000 #endif
5001
5002 /**********************************************************************
5003  *
5004  *  Update the board statistics counters.
5005  *
5006  **********************************************************************/
5007 static void
5008 ixgbe_update_stats_counters(struct adapter *adapter)
5009 {
5010         struct ifnet   *ifp = adapter->ifp;
5011         struct ixgbe_hw *hw = &adapter->hw;
5012         u32  missed_rx = 0, bprc, lxon, lxoff, total;
5013         u64  total_missed_rx = 0;
5014
5015         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5016         adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5017         adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5018         adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5019
5020         for (int i = 0; i < 8; i++) {
5021                 u32 mp;
5022                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5023                 /* missed_rx tallies misses for the gprc workaround */
5024                 missed_rx += mp;
5025                 /* global total per queue */
5026                 adapter->stats.mpc[i] += mp;
5027                 /* Running comprehensive total for stats display */
5028                 total_missed_rx += adapter->stats.mpc[i];
5029                 if (hw->mac.type == ixgbe_mac_82598EB)
5030                         adapter->stats.rnbc[i] +=
5031                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5032                 adapter->stats.pxontxc[i] +=
5033                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5034                 adapter->stats.pxonrxc[i] +=
5035                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5036                 adapter->stats.pxofftxc[i] +=
5037                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5038                 adapter->stats.pxoffrxc[i] +=
5039                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5040                 adapter->stats.pxon2offc[i] +=
5041                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5042         }
5043         for (int i = 0; i < 16; i++) {
5044                 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5045                 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5046                 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5047                 adapter->stats.qbrc[i] +=
5048                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5049                 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5050                 adapter->stats.qbtc[i] +=
5051                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5052                 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5053         }
5054         adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5055         adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5056         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5057
5058         /* Hardware workaround, gprc counts missed packets */
5059         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5060         adapter->stats.gprc -= missed_rx;
5061
5062         if (hw->mac.type != ixgbe_mac_82598EB) {
5063                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5064                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5065                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5066                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5067                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5068                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5069                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5070                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5071         } else {
5072                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5073                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5074                 /* 82598 only has a counter in the high register */
5075                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5076                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5077                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5078         }
5079
5080         /*
5081          * Workaround: mprc hardware is incorrectly counting
5082          * broadcasts, so for now we subtract those.
5083          */
5084         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5085         adapter->stats.bprc += bprc;
5086         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5087         if (hw->mac.type == ixgbe_mac_82598EB)
5088                 adapter->stats.mprc -= bprc;
5089
5090         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5091         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5092         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5093         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5094         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5095         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5096
5097         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5098         adapter->stats.lxontxc += lxon;
5099         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5100         adapter->stats.lxofftxc += lxoff;
5101         total = lxon + lxoff;
5102
5103         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5104         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5105         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5106         adapter->stats.gptc -= total;
5107         adapter->stats.mptc -= total;
5108         adapter->stats.ptc64 -= total;
5109         adapter->stats.gotc -= total * ETHER_MIN_LEN;
5110
5111         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5112         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5113         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5114         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5115         adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5116         adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5117         adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5118         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5119         adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5120         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5121         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5122         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5123         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5124         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5125         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5126         adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5127         adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5128         adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5129         /* Only read FCOE on 82599 */
5130         if (hw->mac.type != ixgbe_mac_82598EB) {
5131                 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5132                 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5133                 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5134                 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5135                 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5136         }
5137
5138         /* Fill out the OS statistics structure */
5139         IFNET_STAT_SET(ifp, ipackets, adapter->stats.gprc);
5140         IFNET_STAT_SET(ifp, opackets, adapter->stats.gptc);
5141         IFNET_STAT_SET(ifp, ibytes, adapter->stats.gorc);
5142         IFNET_STAT_SET(ifp, obytes, adapter->stats.gotc);
5143         IFNET_STAT_SET(ifp, imcasts, adapter->stats.mprc);
5144
5145         /* Rx Errors */
5146         IFNET_STAT_SET(ifp, ierrors, total_missed_rx + adapter->stats.crcerrs +
5147                 adapter->stats.rlec);
5148 }
5149
5150 /** ixgbe_sysctl_tdh_handler - Handler function
5151  *  Retrieves the TDH value from the hardware
5152  */
5153 static int
5154 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5155 {
5156         int error;
5157
5158         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5159         if (!txr) return 0;
5160
5161         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5162         error = sysctl_handle_int(oidp, &val, 0, req);
5163         if (error || !req->newptr)
5164                 return error;
5165         return 0;
5166 }
5167
5168 /** ixgbe_sysctl_tdt_handler - Handler function
5169  *  Retrieves the TDT value from the hardware
5170  */
5171 static int
5172 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5173 {
5174         int error;
5175
5176         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5177         if (!txr) return 0;
5178
5179         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5180         error = sysctl_handle_int(oidp, &val, 0, req);
5181         if (error || !req->newptr)
5182                 return error;
5183         return 0;
5184 }
5185
5186 /** ixgbe_sysctl_rdh_handler - Handler function
5187  *  Retrieves the RDH value from the hardware
5188  */
5189 static int
5190 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5191 {
5192         int error;
5193
5194         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5195         if (!rxr) return 0;
5196
5197         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5198         error = sysctl_handle_int(oidp, &val, 0, req);
5199         if (error || !req->newptr)
5200                 return error;
5201         return 0;
5202 }
5203
5204 /** ixgbe_sysctl_rdt_handler - Handler function
5205  *  Retrieves the RDT value from the hardware
5206  */
5207 static int
5208 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5209 {
5210         int error;
5211
5212         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5213         if (!rxr) return 0;
5214
5215         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5216         error = sysctl_handle_int(oidp, &val, 0, req);
5217         if (error || !req->newptr)
5218                 return error;
5219         return 0;
5220 }
5221
5222 /*
5223  * Add sysctl variables, one per statistic, to the system.
5224  */
5225 static void
5226 ixgbe_add_hw_stats(struct adapter *adapter)
5227 {
5228         struct tx_ring *txr = adapter->tx_rings;
5229         struct rx_ring *rxr = adapter->rx_rings;
5230
5231         struct sysctl_ctx_list *ctx = &adapter->sysctl_ctx;
5232         struct sysctl_oid *tree = adapter->sysctl_tree;
5233         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5234         struct ixgbe_hw_stats *stats = &adapter->stats;
5235
5236         struct sysctl_oid *stat_node, *queue_node;
5237         struct sysctl_oid_list *stat_list, *queue_list;
5238
5239 #define QUEUE_NAME_LEN 32
5240         char namebuf[QUEUE_NAME_LEN];
5241
5242         /* Driver Statistics */
5243         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5244                         CTLFLAG_RD, &adapter->dropped_pkts,
5245                         "Driver dropped packets");
5246         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5247                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5248                         "m_defrag() failed");
5249         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup",
5250                         CTLFLAG_RD, &adapter->no_tx_dma_setup,
5251                         "Driver tx dma failure in xmit");
5252         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5253                         CTLFLAG_RD, &adapter->watchdog_events,
5254                         "Watchdog timeouts");
5255         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx",
5256                         CTLFLAG_RD, &adapter->tso_tx,
5257                         "TSO");
5258         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5259                         CTLFLAG_RD, &adapter->link_irq,
5260                         "Link MSIX IRQ Handled");
5261
5262         for (int i = 0; i < adapter->num_queues; i++, txr++) {
5263         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5264                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5265                                             CTLFLAG_RD, NULL, "Queue Name");
5266                 queue_list = SYSCTL_CHILDREN(queue_node);
5267                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5268                                 CTLFLAG_RD, &(adapter->queues[i].irqs), 0,
5269                                 "irqs on this queue");
5270                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5271                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5272                                 ixgbe_sysctl_tdh_handler, "IU",
5273                                 "Transmit Descriptor Head");
5274                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5275                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5276                                 ixgbe_sysctl_tdt_handler, "IU",
5277                                 "Transmit Descriptor Tail");
5278                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5279                                 CTLFLAG_RD, &txr->no_desc_avail, 0,
5280                                 "Queue No Descriptor Available");
5281                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5282                                 CTLFLAG_RD, &txr->total_packets, 0,
5283                                 "Queue Packets Transmitted");
5284         }
5285
5286         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5287         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5288                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5289                                             CTLFLAG_RD, NULL, "Queue Name");
5290                 queue_list = SYSCTL_CHILDREN(queue_node);
5291
5292 #if 0   /* NET_LRO */
5293                 struct lro_ctrl *lro = &rxr->lro;
5294 #endif
5295
5296         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5297                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5298                                             CTLFLAG_RD, NULL, "Queue Name");
5299                 queue_list = SYSCTL_CHILDREN(queue_node);
5300
5301                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5302                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5303                                 ixgbe_sysctl_rdh_handler, "IU",
5304                                 "Receive Descriptor Head");
5305                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5306                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5307                                 ixgbe_sysctl_rdt_handler, "IU",
5308                                 "Receive Descriptor Tail");
5309                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5310                                 CTLFLAG_RD, &rxr->rx_packets, 0,
5311                                 "Queue Packets Received");
5312                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5313                                 CTLFLAG_RD, &rxr->rx_bytes, 0,
5314                                 "Queue Bytes Received");
5315 #if 0   /* NET_LRO */
5316                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5317                                 CTLFLAG_RD, &lro->lro_queued, 0,
5318                                 "LRO Queued");
5319                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5320                                 CTLFLAG_RD, &lro->lro_flushed, 0,
5321                                 "LRO Flushed");
5322 #endif
5323         }
5324
5325         /* MAC stats get the own sub node */
5326
5327         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5328                                     CTLFLAG_RD, NULL, "MAC Statistics");
5329         stat_list = SYSCTL_CHILDREN(stat_node);
5330
5331         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5332                         CTLFLAG_RD, &stats->crcerrs, 0,
5333                         "CRC Errors");
5334         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5335                         CTLFLAG_RD, &stats->illerrc, 0,
5336                         "Illegal Byte Errors");
5337         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5338                         CTLFLAG_RD, &stats->errbc, 0,
5339                         "Byte Errors");
5340         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5341                         CTLFLAG_RD, &stats->mspdc, 0,
5342                         "MAC Short Packets Discarded");
5343         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5344                         CTLFLAG_RD, &stats->mlfc, 0,
5345                         "MAC Local Faults");
5346         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5347                         CTLFLAG_RD, &stats->mrfc, 0,
5348                         "MAC Remote Faults");
5349         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5350                         CTLFLAG_RD, &stats->rlec, 0,
5351                         "Receive Length Errors");
5352         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd",
5353                         CTLFLAG_RD, &stats->lxontxc, 0,
5354                         "Link XON Transmitted");
5355         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd",
5356                         CTLFLAG_RD, &stats->lxonrxc, 0,
5357                         "Link XON Received");
5358         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd",
5359                         CTLFLAG_RD, &stats->lxofftxc, 0,
5360                         "Link XOFF Transmitted");
5361         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd",
5362                         CTLFLAG_RD, &stats->lxoffrxc, 0,
5363                         "Link XOFF Received");
5364
5365         /* Packet Reception Stats */
5366         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5367                         CTLFLAG_RD, &stats->tor, 0,
5368                         "Total Octets Received");
5369         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5370                         CTLFLAG_RD, &stats->gorc, 0,
5371                         "Good Octets Received");
5372         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5373                         CTLFLAG_RD, &stats->tpr, 0,
5374                         "Total Packets Received");
5375         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5376                         CTLFLAG_RD, &stats->gprc, 0,
5377                         "Good Packets Received");
5378         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5379                         CTLFLAG_RD, &stats->mprc, 0,
5380                         "Multicast Packets Received");
5381         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5382                         CTLFLAG_RD, &stats->bprc, 0,
5383                         "Broadcast Packets Received");
5384         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5385                         CTLFLAG_RD, &stats->prc64, 0,
5386                         "64 byte frames received ");
5387         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5388                         CTLFLAG_RD, &stats->prc127, 0,
5389                         "65-127 byte frames received");
5390         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5391                         CTLFLAG_RD, &stats->prc255, 0,
5392                         "128-255 byte frames received");
5393         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5394                         CTLFLAG_RD, &stats->prc511, 0,
5395                         "256-511 byte frames received");
5396         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5397                         CTLFLAG_RD, &stats->prc1023, 0,
5398                         "512-1023 byte frames received");
5399         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5400                         CTLFLAG_RD, &stats->prc1522, 0,
5401                         "1023-1522 byte frames received");
5402         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5403                         CTLFLAG_RD, &stats->ruc, 0,
5404                         "Receive Undersized");
5405         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5406                         CTLFLAG_RD, &stats->rfc, 0,
5407                         "Fragmented Packets Received ");
5408         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5409                         CTLFLAG_RD, &stats->roc, 0,
5410                         "Oversized Packets Received");
5411         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5412                         CTLFLAG_RD, &stats->rjc, 0,
5413                         "Received Jabber");
5414         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5415                         CTLFLAG_RD, &stats->mngprc, 0,
5416                         "Management Packets Received");
5417         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5418                         CTLFLAG_RD, &stats->mngptc, 0,
5419                         "Management Packets Dropped");
5420         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5421                         CTLFLAG_RD, &stats->xec, 0,
5422                         "Checksum Errors");
5423
5424         /* Packet Transmission Stats */
5425         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5426                         CTLFLAG_RD, &stats->gotc, 0,
5427                         "Good Octets Transmitted");
5428         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5429                         CTLFLAG_RD, &stats->tpt, 0,
5430                         "Total Packets Transmitted");
5431         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5432                         CTLFLAG_RD, &stats->gptc, 0,
5433                         "Good Packets Transmitted");
5434         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5435                         CTLFLAG_RD, &stats->bptc, 0,
5436                         "Broadcast Packets Transmitted");
5437         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5438                         CTLFLAG_RD, &stats->mptc, 0,
5439                         "Multicast Packets Transmitted");
5440         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5441                         CTLFLAG_RD, &stats->mngptc, 0,
5442                         "Management Packets Transmitted");
5443         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5444                         CTLFLAG_RD, &stats->ptc64, 0,
5445                         "64 byte frames transmitted ");
5446         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5447                         CTLFLAG_RD, &stats->ptc127, 0,
5448                         "65-127 byte frames transmitted");
5449         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5450                         CTLFLAG_RD, &stats->ptc255, 0,
5451                         "128-255 byte frames transmitted");
5452         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5453                         CTLFLAG_RD, &stats->ptc511, 0,
5454                         "256-511 byte frames transmitted");
5455         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5456                         CTLFLAG_RD, &stats->ptc1023, 0,
5457                         "512-1023 byte frames transmitted");
5458         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5459                         CTLFLAG_RD, &stats->ptc1522, 0,
5460                         "1024-1522 byte frames transmitted");
5461
5462         /* FC Stats */
5463         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc",
5464                 CTLFLAG_RD, &stats->fccrc, 0,
5465                 "FC CRC Errors");
5466         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last",
5467                 CTLFLAG_RD, &stats->fclast, 0,
5468                 "FC Last Error");
5469         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd",
5470                 CTLFLAG_RD, &stats->fcoerpdc, 0,
5471                 "FCoE Packets Dropped");
5472         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd",
5473                 CTLFLAG_RD, &stats->fcoeprc, 0,
5474                 "FCoE Packets Received");
5475         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd",
5476                 CTLFLAG_RD, &stats->fcoeptc, 0,
5477                 "FCoE Packets Transmitted");
5478         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd",
5479                 CTLFLAG_RD, &stats->fcoedwrc, 0,
5480                 "FCoE DWords Received");
5481         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd",
5482                 CTLFLAG_RD, &stats->fcoedwtc, 0,
5483                 "FCoE DWords Transmitted");
5484 }
5485
5486 /*
5487 ** Set flow control using sysctl:
5488 ** Flow control values:
5489 **      0 - off
5490 **      1 - rx pause
5491 **      2 - tx pause
5492 **      3 - full
5493 */
5494 static int
5495 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5496 {
5497         int error, last;
5498         struct adapter *adapter = (struct adapter *) arg1;
5499
5500         last = adapter->fc;
5501         error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5502         if ((error) || (req->newptr == NULL))
5503                 return (error);
5504
5505         /* Don't bother if it's not changed */
5506         if (adapter->fc == last)
5507                 return (0);
5508
5509         switch (adapter->fc) {
5510                 case ixgbe_fc_rx_pause:
5511                 case ixgbe_fc_tx_pause:
5512                 case ixgbe_fc_full:
5513                         adapter->hw.fc.requested_mode = adapter->fc;
5514                         break;
5515                 case ixgbe_fc_none:
5516                 default:
5517                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
5518         }
5519         /* Don't autoneg if forcing a value */
5520         adapter->hw.fc.disable_fc_autoneg = TRUE;
5521         ixgbe_fc_enable(&adapter->hw);
5522         return error;
5523 }
5524
5525 static void
5526 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5527         const char *description, int *limit, int value)
5528 {
5529         *limit = value;
5530         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
5531             SYSCTL_CHILDREN(adapter->sysctl_tree),
5532             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5533 }
5534
5535 /*
5536 ** Control link advertise speed:
5537 **      1 - advertise only 1G
5538 **      2 - advertise 100Mb
5539 **      3 - advertise normal
5540 */
5541 static int
5542 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5543 {
5544         int                     error = 0;
5545         struct adapter          *adapter;
5546         device_t                dev;
5547         struct ixgbe_hw         *hw;
5548         ixgbe_link_speed        speed, last;
5549
5550         adapter = (struct adapter *) arg1;
5551         dev = adapter->dev;
5552         hw = &adapter->hw;
5553         last = adapter->advertise;
5554
5555         error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5556         if ((error) || (adapter->advertise == -1))
5557                 return (error);
5558
5559         if (adapter->advertise == last) /* no change */
5560                 return (0);
5561
5562         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5563             (hw->phy.multispeed_fiber)))
5564                 return (error);
5565
5566         if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5567                 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5568                 return (error);
5569         }
5570
5571         if (adapter->advertise == 1)
5572                 speed = IXGBE_LINK_SPEED_1GB_FULL;
5573         else if (adapter->advertise == 2)
5574                 speed = IXGBE_LINK_SPEED_100_FULL;
5575         else if (adapter->advertise == 3)
5576                 speed = IXGBE_LINK_SPEED_1GB_FULL |
5577                         IXGBE_LINK_SPEED_10GB_FULL;
5578         else /* bogus value */
5579                 return (error);
5580
5581         hw->mac.autotry_restart = TRUE;
5582         hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5583
5584         return (error);
5585 }
5586
5587 /*
5588 ** Thermal Shutdown Trigger
5589 **   - cause a Thermal Overtemp IRQ
5590 */
5591 static int
5592 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5593 {
5594         int             error, fire = 0;
5595         struct adapter  *adapter = (struct adapter *) arg1;
5596         struct ixgbe_hw *hw = &adapter->hw;
5597
5598
5599         if (hw->mac.type != ixgbe_mac_X540)
5600                 return (0);
5601
5602         error = sysctl_handle_int(oidp, &fire, 0, req);
5603         if ((error) || (req->newptr == NULL))
5604                 return (error);
5605
5606         if (fire) {
5607                 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5608                 reg |= IXGBE_EICR_TS;
5609                 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5610         }
5611
5612         return (0);
5613 }
5614
5615 static void
5616 ixgbe_set_eitr(struct adapter *sc, int idx, int rate)
5617 {
5618         uint32_t eitr = 0;
5619
5620         /* convert rate in intr/s to hw representation */
5621         if (rate > 0) {
5622                 eitr = 1000000 / rate;
5623                 eitr <<= IXGBE_EITR_INTVL_SHIFT;
5624
5625                 if (eitr == 0) {
5626                         /* Don't disable it */
5627                         eitr = 1 << IXGBE_EITR_INTVL_SHIFT;
5628                 } else if (eitr > IXGBE_EITR_INTVL_MASK) {
5629                         /* Don't allow it to be too large */
5630                         eitr = IXGBE_EITR_INTVL_MASK;
5631                 }
5632         }
5633         IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(idx), eitr);
5634 }
5635
5636 static int
5637 ixgbe_sysctl_intr_rate(SYSCTL_HANDLER_ARGS)
5638 {
5639         struct adapter *sc = (void *)arg1;
5640         struct ifnet *ifp = sc->ifp;
5641         int error, intr_rate, running;
5642         struct ix_queue *que = sc->queues;
5643
5644         intr_rate = sc->intr_rate;
5645         error = sysctl_handle_int(oidp, &intr_rate, 0, req);
5646         if (error || req->newptr == NULL)
5647                 return error;
5648         if (intr_rate < 0)
5649                 return EINVAL;
5650
5651         ifnet_serialize_all(ifp);
5652
5653         sc->intr_rate = intr_rate;
5654         running = ifp->if_flags & IFF_RUNNING;
5655         if (running)
5656                 ixgbe_set_eitr(sc, 0, sc->intr_rate);
5657
5658         if (running && (sc->intr_type == PCI_INTR_TYPE_MSIX)) {
5659                 for (int i = 0; i < sc->num_queues; i++, que++)
5660                         ixgbe_set_eitr(sc, que->msix, sc->intr_rate);
5661         }
5662
5663         if (bootverbose)
5664                 if_printf(ifp, "interrupt rate set to %d/sec\n", sc->intr_rate);
5665
5666         ifnet_deserialize_all(ifp);
5667
5668         return 0;
5669 }
5670
5671 /* rearrange mbuf chain to get contiguous bytes */
5672 static int
5673 ixgbe_tso_pullup(struct tx_ring *txr, struct mbuf **mp)
5674 {
5675         int hoff, iphlen, thoff;
5676         struct mbuf *m;
5677
5678         m = *mp;
5679         KASSERT(M_WRITABLE(m), ("TSO mbuf not writable"));
5680
5681         iphlen = m->m_pkthdr.csum_iphlen;
5682         thoff = m->m_pkthdr.csum_thlen;
5683         hoff = m->m_pkthdr.csum_lhlen;
5684
5685         KASSERT(iphlen > 0, ("invalid ip hlen"));
5686         KASSERT(thoff > 0, ("invalid tcp hlen"));
5687         KASSERT(hoff > 0, ("invalid ether hlen"));
5688
5689         if (__predict_false(m->m_len < hoff + iphlen + thoff)) {
5690                 m = m_pullup(m, hoff + iphlen + thoff);
5691                 if (m == NULL) {
5692                         *mp = NULL;
5693                         return ENOBUFS;
5694                 }
5695                 *mp = m;
5696         }
5697
5698         return 0;
5699 }